From ee9fa38554f8427f5361708b28551e70ff0e1256 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu@rochester.edu>
Date: Fri, 22 Mar 2024 15:20:00 -0400
Subject: [PATCH 001/404] [libc] fix missing macro dependency in bazel (#86298)

![image](https://github.com/llvm/llvm-project/assets/20108837/94cb5718-3526-4bae-8a79-f5b1d19b352d)
---
 .../llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel      | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel
index 803010e8b3ad..5f59d70ecc16 100644
--- a/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel
@@ -46,6 +46,7 @@ libc_support_library(
         "//libc:__support_cpp_type_traits",
         "//libc:__support_fputil_fp_bits",
         "//libc:__support_fputil_fpbits_str",
+        "//libc:llvm_libc_macros_math_macros",
         "//libc/test/UnitTest:LibcUnitTest",
         "//libc/test/UnitTest:fp_test_helpers",
         "//libc/utils/MPFRWrapper:mpfr_impl",
-- 
GitLab


From d394f3a162b871668d0c8e8bf6a94922fa8698ae Mon Sep 17 00:00:00 2001
From: Xing Xue <xingxue@outlook.com>
Date: Fri, 22 Mar 2024 15:25:08 -0400
Subject: [PATCH 002/404] [OpenMP][AIX] Affinity implementation for AIX
 (#84984)

This patch implements `affinity` for AIX, which is quite different from
platforms such as Linux.
- Setting CPU affinity through masks and related functions are not
supported. System call `bindprocessor()` is used to bind a thread to one
CPU per call.
- There are no system routines to get the affinity info of a thread. The
implementation of `get_system_affinity()` for AIX gets the mask of all
available CPUs, to be used as the full mask only.
- Topology is not available from the file system. It is obtained through
system SRAD (Scheduler Resource Allocation Domain).

This patch has run through the libomp LIT tests successfully with
`affinity` enabled.
---
 openmp/runtime/src/kmp.h            |   5 +-
 openmp/runtime/src/kmp_affinity.cpp | 130 ++++++++++++++++++++++++++--
 openmp/runtime/src/kmp_affinity.h   |  74 +++++++++++++++-
 openmp/runtime/src/kmp_os.h         |   2 +-
 openmp/runtime/src/z_Linux_util.cpp |  39 +++++++--
 openmp/runtime/test/lit.cfg         |   2 +-
 6 files changed, 233 insertions(+), 19 deletions(-)

diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index 885d6636abe4..18ccf10fe17d 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -819,6 +819,7 @@ private:
 typedef KMPAffinity::Mask kmp_affin_mask_t;
 extern KMPAffinity *__kmp_affinity_dispatch;
 
+#ifndef KMP_OS_AIX
 class kmp_affinity_raii_t {
   kmp_affin_mask_t *mask;
   bool restored;
@@ -843,6 +844,7 @@ public:
   }
   ~kmp_affinity_raii_t() { restore(); }
 };
+#endif // !KMP_OS_AIX
 
 // Declare local char buffers with this size for printing debug and info
 // messages, using __kmp_affinity_print_mask().
@@ -3910,7 +3912,8 @@ extern void __kmp_balanced_affinity(kmp_info_t *th, int team_size);
 #if KMP_WEIGHTED_ITERATIONS_SUPPORTED
 extern int __kmp_get_first_osid_with_ecore(void);
 #endif
-#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY
+#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY ||     \
+    KMP_OS_AIX
 extern int kmp_set_thread_affinity_mask_initial(void);
 #endif
 static inline void __kmp_assign_root_init_mask() {
diff --git a/openmp/runtime/src/kmp_affinity.cpp b/openmp/runtime/src/kmp_affinity.cpp
index 048bd174fc95..b574dbbaf54f 100644
--- a/openmp/runtime/src/kmp_affinity.cpp
+++ b/openmp/runtime/src/kmp_affinity.cpp
@@ -2910,12 +2910,17 @@ static inline const char *__kmp_cpuinfo_get_envvar() {
 }
 
 // Parse /proc/cpuinfo (or an alternate file in the same format) to obtain the
-// affinity map.
+// affinity map. On AIX, the map is obtained through system SRAD (Scheduler
+// Resource Allocation Domain).
 static bool __kmp_affinity_create_cpuinfo_map(int *line,
                                               kmp_i18n_id_t *const msg_id) {
+  *msg_id = kmp_i18n_null;
+
+#if KMP_OS_AIX
+  unsigned num_records = __kmp_xproc;
+#else
   const char *filename = __kmp_cpuinfo_get_filename();
   const char *envvar = __kmp_cpuinfo_get_envvar();
-  *msg_id = kmp_i18n_null;
 
   if (__kmp_affinity.flags.verbose) {
     KMP_INFORM(AffParseFilename, "KMP_AFFINITY", filename);
@@ -2974,6 +2979,7 @@ static bool __kmp_affinity_create_cpuinfo_map(int *line,
     *msg_id = kmp_i18n_str_CantRewindCpuinfo;
     return false;
   }
+#endif // KMP_OS_AIX
 
   // Allocate the array of records to store the proc info in.  The dummy
   // element at the end makes the logic in filling them out easier to code.
@@ -3003,6 +3009,99 @@ static bool __kmp_affinity_create_cpuinfo_map(int *line,
     INIT_PROC_INFO(threadInfo[i]);
   }
 
+#if KMP_OS_AIX
+  int smt_threads;
+  lpar_info_format1_t cpuinfo;
+  unsigned num_avail = __kmp_xproc;
+
+  if (__kmp_affinity.flags.verbose)
+    KMP_INFORM(AffParseFilename, "KMP_AFFINITY", "system info for topology");
+
+  // Get the number of SMT threads per core.
+  int retval =
+      lpar_get_info(LPAR_INFO_FORMAT1, &cpuinfo, sizeof(lpar_info_format1_t));
+  if (!retval)
+    smt_threads = cpuinfo.smt_threads;
+  else {
+    CLEANUP_THREAD_INFO;
+    *msg_id = kmp_i18n_str_UnknownTopology;
+    return false;
+  }
+
+  // Allocate a resource set containing available system resourses.
+  rsethandle_t sys_rset = rs_alloc(RS_SYSTEM);
+  if (sys_rset == NULL) {
+    CLEANUP_THREAD_INFO;
+    *msg_id = kmp_i18n_str_UnknownTopology;
+    return false;
+  }
+  // Allocate a resource set for the SRAD info.
+  rsethandle_t srad = rs_alloc(RS_EMPTY);
+  if (srad == NULL) {
+    rs_free(sys_rset);
+    CLEANUP_THREAD_INFO;
+    *msg_id = kmp_i18n_str_UnknownTopology;
+    return false;
+  }
+
+  // Get the SRAD system detail level.
+  int sradsdl = rs_getinfo(NULL, R_SRADSDL, 0);
+  if (sradsdl < 0) {
+    rs_free(sys_rset);
+    rs_free(srad);
+    CLEANUP_THREAD_INFO;
+    *msg_id = kmp_i18n_str_UnknownTopology;
+    return false;
+  }
+  // Get the number of RADs at that SRAD SDL.
+  int num_rads = rs_numrads(sys_rset, sradsdl, 0);
+  if (num_rads < 0) {
+    rs_free(sys_rset);
+    rs_free(srad);
+    CLEANUP_THREAD_INFO;
+    *msg_id = kmp_i18n_str_UnknownTopology;
+    return false;
+  }
+
+  // Get the maximum number of procs that may be contained in a resource set.
+  int max_procs = rs_getinfo(NULL, R_MAXPROCS, 0);
+  if (max_procs < 0) {
+    rs_free(sys_rset);
+    rs_free(srad);
+    CLEANUP_THREAD_INFO;
+    *msg_id = kmp_i18n_str_UnknownTopology;
+    return false;
+  }
+
+  int cur_rad = 0;
+  int num_set = 0;
+  for (int srad_idx = 0; cur_rad < num_rads && srad_idx < VMI_MAXRADS;
+       ++srad_idx) {
+    // Check if the SRAD is available in the RSET.
+    if (rs_getrad(sys_rset, srad, sradsdl, srad_idx, 0) < 0)
+      continue;
+
+    for (int cpu = 0; cpu < max_procs; cpu++) {
+      // Set the info for the cpu if it is in the SRAD.
+      if (rs_op(RS_TESTRESOURCE, srad, NULL, R_PROCS, cpu)) {
+        threadInfo[cpu][osIdIndex] = cpu;
+        threadInfo[cpu][pkgIdIndex] = cur_rad;
+        threadInfo[cpu][coreIdIndex] = cpu / smt_threads;
+        ++num_set;
+        if (num_set >= num_avail) {
+          // Done if all available CPUs have been set.
+          break;
+        }
+      }
+    }
+    ++cur_rad;
+  }
+  rs_free(sys_rset);
+  rs_free(srad);
+
+  // The topology is already sorted.
+
+#else // !KMP_OS_AIX
   unsigned num_avail = 0;
   *line = 0;
 #if KMP_ARCH_S390X
@@ -3250,6 +3349,8 @@ static bool __kmp_affinity_create_cpuinfo_map(int *line,
   qsort(threadInfo, num_avail, sizeof(*threadInfo),
         __kmp_affinity_cmp_ProcCpuInfo_phys_id);
 
+#endif // KMP_OS_AIX
+
   // The table is now sorted by pkgId / coreId / threadId, but we really don't
   // know the radix of any of the fields. pkgId's may be sparsely assigned among
   // the chips on a system. Although coreId's are usually assigned
@@ -4445,7 +4546,7 @@ static bool __kmp_aux_affinity_initialize_topology(kmp_affinity_t &affinity) {
     }
 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
 
-#if KMP_OS_LINUX
+#if KMP_OS_LINUX || KMP_OS_AIX
     if (!success) {
       int line = 0;
       success = __kmp_affinity_create_cpuinfo_map(&line, &msg_id);
@@ -4841,7 +4942,12 @@ void __kmp_affinity_uninitialize(void) {
   }
   if (__kmp_affin_origMask != NULL) {
     if (KMP_AFFINITY_CAPABLE()) {
+#if KMP_OS_AIX
+      // Uninitialize by unbinding the thread.
+      bindprocessor(BINDTHREAD, thread_self(), PROCESSOR_CLASS_ANY);
+#else
       __kmp_set_system_affinity(__kmp_affin_origMask, FALSE);
+#endif
     }
     KMP_CPU_FREE(__kmp_affin_origMask);
     __kmp_affin_origMask = NULL;
@@ -5015,7 +5121,10 @@ void __kmp_affinity_bind_init_mask(int gtid) {
     __kmp_set_system_affinity(th->th.th_affin_mask, FALSE);
   } else
 #endif
+#ifndef KMP_OS_AIX
+    // Do not set the full mask as the init mask on AIX.
     __kmp_set_system_affinity(th->th.th_affin_mask, TRUE);
+#endif
 }
 
 void __kmp_affinity_bind_place(int gtid) {
@@ -5128,7 +5237,7 @@ int __kmp_aux_set_affinity(void **mask) {
 int __kmp_aux_get_affinity(void **mask) {
   int gtid;
   int retval;
-#if KMP_OS_WINDOWS || KMP_DEBUG
+#if KMP_OS_WINDOWS || KMP_OS_AIX || KMP_DEBUG
   kmp_info_t *th;
 #endif
   if (!KMP_AFFINITY_CAPABLE()) {
@@ -5136,7 +5245,7 @@ int __kmp_aux_get_affinity(void **mask) {
   }
 
   gtid = __kmp_entry_gtid();
-#if KMP_OS_WINDOWS || KMP_DEBUG
+#if KMP_OS_WINDOWS || KMP_OS_AIX || KMP_DEBUG
   th = __kmp_threads[gtid];
 #else
   (void)gtid; // unused variable
@@ -5159,7 +5268,7 @@ int __kmp_aux_get_affinity(void **mask) {
     }
   }
 
-#if !KMP_OS_WINDOWS
+#if !KMP_OS_WINDOWS && !KMP_OS_AIX
 
   retval = __kmp_get_system_affinity((kmp_affin_mask_t *)(*mask), FALSE);
   KA_TRACE(
@@ -5179,7 +5288,7 @@ int __kmp_aux_get_affinity(void **mask) {
   KMP_CPU_COPY((kmp_affin_mask_t *)(*mask), th->th.th_affin_mask);
   return 0;
 
-#endif /* KMP_OS_WINDOWS */
+#endif /* !KMP_OS_WINDOWS && !KMP_OS_AIX */
 }
 
 int __kmp_aux_get_affinity_max_proc() {
@@ -5561,7 +5670,8 @@ void __kmp_balanced_affinity(kmp_info_t *th, int nthreads) {
   }
 }
 
-#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY
+#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY ||     \
+    KMP_OS_AIX
 // We don't need this entry for Windows because
 // there is GetProcessAffinityMask() api
 //
@@ -5596,7 +5706,11 @@ extern "C"
                 "set full mask for thread %d\n",
                 gtid));
   KMP_DEBUG_ASSERT(__kmp_affin_fullMask != NULL);
+#if KMP_OS_AIX
+  return bindprocessor(BINDTHREAD, thread_self(), PROCESSOR_CLASS_ANY);
+#else
   return __kmp_set_system_affinity(__kmp_affin_fullMask, FALSE);
+#endif
 }
 #endif
 
diff --git a/openmp/runtime/src/kmp_affinity.h b/openmp/runtime/src/kmp_affinity.h
index 1c7db2f59943..7efc090f8863 100644
--- a/openmp/runtime/src/kmp_affinity.h
+++ b/openmp/runtime/src/kmp_affinity.h
@@ -191,7 +191,8 @@ public:
 };
 #endif /* KMP_USE_HWLOC */
 
-#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY
+#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY ||     \
+    KMP_OS_AIX
 #if KMP_OS_LINUX
 /* On some of the older OS's that we build on, these constants aren't present
    in <asm/unistd.h> #included from <sys.syscall.h>. They must be the same on
@@ -317,6 +318,10 @@ public:
 #elif KMP_OS_NETBSD
 #include <pthread.h>
 #include <sched.h>
+#elif KMP_OS_AIX
+#include <sys/dr.h>
+#include <sys/rset.h>
+#define VMI_MAXRADS 64 // Maximum number of RADs allowed by AIX.
 #endif
 class KMPNativeAffinity : public KMPAffinity {
   class Mask : public KMPAffinity::Mask {
@@ -404,6 +409,70 @@ class KMPNativeAffinity : public KMPAffinity {
         ++retval;
       return retval;
     }
+#if KMP_OS_AIX
+    // On AIX, we don't have a way to get CPU(s) a thread is bound to.
+    // This routine is only used to get the full mask.
+    int get_system_affinity(bool abort_on_error) override {
+      KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
+                  "Illegal get affinity operation when not capable");
+
+      (void)abort_on_error;
+
+      // Set the mask with all CPUs that are available.
+      for (int i = 0; i < __kmp_xproc; ++i)
+        KMP_CPU_SET(i, this);
+      return 0;
+    }
+    int set_system_affinity(bool abort_on_error) const override {
+      KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
+
+                  "Illegal set affinity operation when not capable");
+
+      int location;
+      int gtid = __kmp_entry_gtid();
+      int tid = thread_self();
+
+      // Unbind the thread if it was bound to any processors before so that
+      // we can bind the thread to CPUs specified by the mask not others.
+      int retval = bindprocessor(BINDTHREAD, tid, PROCESSOR_CLASS_ANY);
+
+      // On AIX, we can only bind to one instead of a set of CPUs with the
+      // bindprocessor() system call.
+      KMP_CPU_SET_ITERATE(location, this) {
+        if (KMP_CPU_ISSET(location, this)) {
+          retval = bindprocessor(BINDTHREAD, tid, location);
+          if (retval == -1 && errno == 1) {
+            rsid_t rsid;
+            rsethandle_t rsh;
+            // Put something in rsh to prevent compiler warning
+            // about uninitalized use
+            rsh = rs_alloc(RS_EMPTY);
+            rsid.at_pid = getpid();
+            if (RS_DEFAULT_RSET != ra_getrset(R_PROCESS, rsid, 0, rsh)) {
+              retval = ra_detachrset(R_PROCESS, rsid, 0);
+              retval = bindprocessor(BINDTHREAD, tid, location);
+            }
+          }
+          if (retval == 0) {
+            KA_TRACE(10, ("__kmp_set_system_affinity:  Done binding "
+                          "T#%d to cpu=%d.\n",
+                          gtid, location));
+            continue;
+          }
+          int error = errno;
+          if (abort_on_error) {
+            __kmp_fatal(KMP_MSG(FunctionError, "bindprocessor()"),
+                        KMP_ERR(error), __kmp_msg_null);
+            KA_TRACE(10, ("__kmp_set_system_affinity:  Error binding "
+                          "T#%d to cpu=%d, errno=%d.\n",
+                          gtid, location, error));
+            return error;
+          }
+        }
+      }
+      return 0;
+    }
+#else // !KMP_OS_AIX
     int get_system_affinity(bool abort_on_error) override {
       KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
                   "Illegal get affinity operation when not capable");
@@ -446,6 +515,7 @@ class KMPNativeAffinity : public KMPAffinity {
       }
       return error;
     }
+#endif // KMP_OS_AIX
   };
   void determine_capable(const char *env_var) override {
     __kmp_affinity_determine_capable(env_var);
@@ -475,7 +545,7 @@ class KMPNativeAffinity : public KMPAffinity {
   api_type get_api_type() const override { return NATIVE_OS; }
 };
 #endif /* KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY  \
-        */
+          || KMP_OS_AIX */
 
 #if KMP_OS_WINDOWS
 class KMPNativeAffinity : public KMPAffinity {
diff --git a/openmp/runtime/src/kmp_os.h b/openmp/runtime/src/kmp_os.h
index 63da9e5fa15d..a628070c882a 100644
--- a/openmp/runtime/src/kmp_os.h
+++ b/openmp/runtime/src/kmp_os.h
@@ -76,7 +76,7 @@
 #endif
 
 #if (KMP_OS_LINUX || KMP_OS_WINDOWS || KMP_OS_FREEBSD || KMP_OS_NETBSD ||      \
-     KMP_OS_DRAGONFLY) &&                                                      \
+     KMP_OS_DRAGONFLY || KMP_OS_AIX) &&                                        \
     !KMP_OS_WASI
 #define KMP_AFFINITY_SUPPORTED 1
 #if KMP_OS_WINDOWS && KMP_ARCH_X86_64
diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp
index d751a417331c..29db9d008a49 100644
--- a/openmp/runtime/src/z_Linux_util.cpp
+++ b/openmp/runtime/src/z_Linux_util.cpp
@@ -125,7 +125,8 @@ static void __kmp_print_cond(char *buffer, kmp_cond_align_t *cond) {
 }
 #endif
 
-#if ((KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY) &&  \
+#if ((KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY ||   \
+      KMP_OS_AIX) &&                                                           \
      KMP_AFFINITY_SUPPORTED)
 
 /* Affinity support */
@@ -142,6 +143,29 @@ void __kmp_affinity_bind_thread(int which) {
   KMP_CPU_FREE_FROM_STACK(mask);
 }
 
+#if KMP_OS_AIX
+void __kmp_affinity_determine_capable(const char *env_var) {
+  // All versions of AIX support bindprocessor().
+
+  size_t mask_size = __kmp_xproc / CHAR_BIT;
+  // Round up to byte boundary.
+  if (__kmp_xproc % CHAR_BIT)
+    ++mask_size;
+
+  // Round up to the mask_size_type boundary.
+  if (mask_size % sizeof(__kmp_affin_mask_size))
+    mask_size += sizeof(__kmp_affin_mask_size) -
+                 mask_size % sizeof(__kmp_affin_mask_size);
+  KMP_AFFINITY_ENABLE(mask_size);
+  KA_TRACE(10,
+           ("__kmp_affinity_determine_capable: "
+            "AIX OS affinity interface bindprocessor functional (mask size = "
+            "%" KMP_SIZE_T_SPEC ").\n",
+            __kmp_affin_mask_size));
+}
+
+#else // !KMP_OS_AIX
+
 /* Determine if we can access affinity functionality on this version of
  * Linux* OS by checking __NR_sched_{get,set}affinity system calls, and set
  * __kmp_affin_mask_size to the appropriate value (0 means not capable). */
@@ -271,8 +295,9 @@ void __kmp_affinity_determine_capable(const char *env_var) {
     KMP_WARNING(AffCantGetMaskSize, env_var);
   }
 }
-
-#endif // KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
+#endif // KMP_OS_AIX
+#endif // (KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD ||                  \
+           KMP_OS_DRAGONFLY || KMP_OS_AIX) && KMP_AFFINITY_SUPPORTED
 
 #if KMP_USE_FUTEX
 
@@ -501,7 +526,7 @@ static void *__kmp_launch_worker(void *thr) {
 #endif /* KMP_BLOCK_SIGNALS */
   void *exit_val;
 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||     \
-    KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS
+    KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX
   void *volatile padding = 0;
 #endif
   int gtid;
@@ -550,7 +575,7 @@ static void *__kmp_launch_worker(void *thr) {
 #endif /* KMP_BLOCK_SIGNALS */
 
 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||     \
-    KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS
+    KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX
   if (__kmp_stkoffset > 0 && gtid > 0) {
     padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
     (void)padding;
@@ -1268,7 +1293,8 @@ static void __kmp_atfork_child(void) {
   ++__kmp_fork_count;
 
 #if KMP_AFFINITY_SUPPORTED
-#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY
+#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY ||     \
+    KMP_OS_AIX
   // reset the affinity in the child to the initial thread
   // affinity in the parent
   kmp_set_thread_affinity_mask_initial();
@@ -2325,6 +2351,7 @@ int __kmp_is_address_mapped(void *addr) {
   found = (int)addr < (__builtin_wasm_memory_size(0) * PAGESIZE);
 #elif KMP_OS_AIX
 
+  (void)rc;
   // FIXME(AIX): Implement this
   found = 1;
 
diff --git a/openmp/runtime/test/lit.cfg b/openmp/runtime/test/lit.cfg
index a3456063c10f..e27e52bb4289 100644
--- a/openmp/runtime/test/lit.cfg
+++ b/openmp/runtime/test/lit.cfg
@@ -129,7 +129,7 @@ if config.operating_system == 'NetBSD':
 if config.operating_system == 'Darwin':
     config.available_features.add("darwin")
 
-if config.operating_system in ['Windows', 'Linux', 'FreeBSD', 'NetBSD', 'DragonFly']:
+if config.operating_system in ['Windows', 'Linux', 'FreeBSD', 'NetBSD', 'DragonFly', 'AIX']:
     config.available_features.add('affinity')
 
 if config.operating_system in ['Linux']:
-- 
GitLab


From dfe4ca9b7f4a422500d78280dc5eefd1979939e6 Mon Sep 17 00:00:00 2001
From: Paul Kirth <paulkirth@google.com>
Date: Fri, 22 Mar 2024 12:27:41 -0700
Subject: [PATCH 003/404] [RISCV][lld] Set the type of TLSDESC relocation's
 referenced local symbol to STT_NOTYPE

When adding fixups for RISCV_TLSDESC_ADD_LO and RISCV_TLSDESC_LOAD_LO,
the local label added for RISCV TLSDESC relocations have STT_TLS set,
which is incorrect. Instead, these labels should have `STT_NOTYPE`.

This patch stops adding such fixups and avoid setting the STT_TLS on
these symbols. Failing to do so can cause LLD to emit an error `has an
STT_TLS symbol but doesn't have an SHF_TLS section`. We additionally,
adjust how LLD services these relocations to avoid errors with
incompatible relocation and symbol types.

Reviewers: topperc, MaskRay

Reviewed By: MaskRay

Pull Request: https://github.com/llvm/llvm-project/pull/85817
---
 lld/ELF/Relocations.cpp                       |  5 +++-
 lld/test/ELF/riscv-tlsdesc-relax.s            |  8 ++++++
 lld/test/ELF/riscv-tlsdesc.s                  | 27 +++++++++++--------
 .../Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp |  2 --
 4 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index 619fbaf5dc54..92a1b9baaca3 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -1480,7 +1480,10 @@ template <class ELFT, class RelTy> void RelocationScanner::scanOne(RelTy *&i) {
 
   // Process TLS relocations, including TLS optimizations. Note that
   // R_TPREL and R_TPREL_NEG relocations are resolved in processAux.
-  if (sym.isTls()) {
+  //
+  // Some RISCV TLSDESC relocations reference a local NOTYPE symbol,
+  // but we need to process them in handleTlsRelocation.
+  if (sym.isTls() || oneof<R_TLSDESC_PC, R_TLSDESC_CALL>(expr)) {
     if (unsigned processed =
             handleTlsRelocation(type, sym, *sec, offset, addend, expr)) {
       i += processed - 1;
diff --git a/lld/test/ELF/riscv-tlsdesc-relax.s b/lld/test/ELF/riscv-tlsdesc-relax.s
index fb24317e6535..5718d4175be1 100644
--- a/lld/test/ELF/riscv-tlsdesc-relax.s
+++ b/lld/test/ELF/riscv-tlsdesc-relax.s
@@ -33,12 +33,14 @@
 # GD64-NEXT:         c.add   a0, tp
 # GD64-NEXT:         jal     {{.*}} <foo>
 ## &.got[c]-. = 0x20c0+8 - 0x1020 = 0x10a8
+# GD64-LABEL: <.Ltlsdesc_hi1>:
 # GD64-NEXT:   1020: auipc   a4, 0x1
 # GD64-NEXT:         ld      a5, 0xa8(a4)
 # GD64-NEXT:         addi    a0, a4, 0xa8
 # GD64-NEXT:         jalr    t0, 0x0(a5)
 # GD64-NEXT:         c.add   a0, tp
 ## &.got[c]-. = 0x20c0+8 - 0x1032 = 0x1096
+# GD64-LABEL: <.Ltlsdesc_hi2>:
 # GD64-NEXT:   1032: auipc   a6, 0x1
 # GD64-NEXT:         ld      a7, 0x96(a6)
 # GD64-NEXT:         addi    a0, a6, 0x96
@@ -64,6 +66,7 @@
 # LE64-NEXT:         jal     {{.*}} <foo>
 # LE64-NEXT:                 R_RISCV_JAL foo
 # LE64-NEXT:                 R_RISCV_RELAX *ABS*
+# LE64-LABEL: <.Ltlsdesc_hi1>:
 # LE64-NEXT:         addi    a0, zero, 0x7ff
 # LE64-NEXT:                 R_RISCV_TLSDESC_HI20 b
 # LE64-NEXT:                 R_RISCV_RELAX *ABS*
@@ -71,6 +74,7 @@
 # LE64-NEXT:                 R_RISCV_TLSDESC_ADD_LO12 .Ltlsdesc_hi1
 # LE64-NEXT:                 R_RISCV_TLSDESC_CALL .Ltlsdesc_hi1
 # LE64-NEXT:         c.add   a0, tp
+# LE64-LABEL: <.Ltlsdesc_hi2>:
 # LE64-NEXT:         addi    zero, zero, 0x0
 # LE64-NEXT:                 R_RISCV_TLSDESC_HI20 b
 # LE64-NEXT:         addi    zero, zero, 0x0
@@ -93,9 +97,11 @@
 # LE64A-NEXT:         addi    a0, a0, -0x479
 # LE64A-NEXT:         c.add   a0, tp
 # LE64A-NEXT:         jal     {{.*}} <foo>
+# LE64A-LABEL: <.Ltlsdesc_hi1>:
 # LE64A-NEXT:         lui     a0, 0x2
 # LE64A-NEXT:         addi    a0, a0, -0x479
 # LE64A-NEXT:         c.add   a0, tp
+# LE64A-LABEL: <.Ltlsdesc_hi2>:
 # LE64A-NEXT:         addi    zero, zero, 0x0
 # LE64A-NEXT:         addi    zero, zero, 0x0
 # LE64A-NEXT:         lui     a0, 0x2
@@ -115,10 +121,12 @@
 # IE64-NEXT:         c.add   a0, tp
 # IE64-NEXT:         jal     {{.*}} <foo>
 ## &.got[c]-. = 0x120e0+8 - 0x11018 = 0x10d0
+# IE64-LABEL: <.Ltlsdesc_hi1>:
 # IE64-NEXT:  11018: auipc   a0, 0x1
 # IE64-NEXT:         ld      a0, 0xd0(a0)
 # IE64-NEXT:         c.add   a0, tp
 ## &.got[c]-. = 0x120e0+8 - 0x1102a = 0x10be
+# IE64-LABEL: <.Ltlsdesc_hi2>:
 # IE64-NEXT:         addi    zero, zero, 0x0
 # IE64-NEXT:         addi    zero, zero, 0x0
 # IE64-NEXT:  1102a: auipc   a0, 0x1
diff --git a/lld/test/ELF/riscv-tlsdesc.s b/lld/test/ELF/riscv-tlsdesc.s
index c583e15cf30c..935ecbddfbff 100644
--- a/lld/test/ELF/riscv-tlsdesc.s
+++ b/lld/test/ELF/riscv-tlsdesc.s
@@ -29,11 +29,13 @@
 # RUN: ld.lld -e 0 -z now a.32.o c.32.so -o a.32.ie
 # RUN: llvm-objdump --no-show-raw-insn -M no-aliases -h -d a.32.ie | FileCheck %s --check-prefix=IE32
 
-# RUN: llvm-mc -triple=riscv64 -filetype=obj d.s -o d.64.o
-# RUN: not ld.lld -shared -soname=d.64.so -o d.64.so d.64.o 2>&1 | FileCheck %s --check-prefix=BADTLSLABEL
+## Prior to https://github.com/llvm/llvm-project/pull/85817 the local TLSDESC
+## labels would be marked STT_TLS, resulting in an error "has an STT_TLS symbol but doesn't have an SHF_TLS section"
 
+# RUN: llvm-mc -triple=riscv64 -filetype=obj d.s -o d.64.o
+# RUN: ld.lld -shared -soname=d.64.so -o d.64.so d.64.o --fatal-warnings
 # RUN: llvm-mc -triple=riscv32 -filetype=obj d.s -o d.32.o --defsym ELF32=1
-# RUN: not ld.lld -shared -soname=d.32.so -o d.32.so d.32.o 2>&1 | FileCheck %s --check-prefix=BADTLSLABEL
+# RUN: ld.lld -shared -soname=d.32.so -o d.32.so d.32.o --fatal-warnings
 
 # GD64-RELA:      .rela.dyn {
 # GD64-RELA-NEXT:   0x2408 R_RISCV_TLSDESC - 0x7FF
@@ -74,14 +76,14 @@
 # GD64-NEXT:         add     a0, a0, tp
 
 ## &.got[b]-. = 0x23e0+40 - 0x12f4 = 0x1114
-# GD64-NEXT:   12f4: auipc   a2, 0x1
+# GD64:        12f4: auipc   a2, 0x1
 # GD64-NEXT:         ld      a3, 0x114(a2)
 # GD64-NEXT:         addi    a0, a2, 0x114
 # GD64-NEXT:         jalr    t0, 0x0(a3)
 # GD64-NEXT:         add     a0, a0, tp
 
 ## &.got[c]-. = 0x23e0+24 - 0x1308 = 0x10f0
-# GD64-NEXT:   1308: auipc   a4, 0x1
+# GD64:        1308: auipc   a4, 0x1
 # GD64-NEXT:         ld      a5, 0xf0(a4)
 # GD64-NEXT:         addi    a0, a4, 0xf0
 # GD64-NEXT:         jalr    t0, 0x0(a5)
@@ -89,7 +91,7 @@
 
 # NOREL: no relocations
 
-# LE64-LABEL: <.text>:
+# LE64-LABEL: <.Ltlsdesc_hi0>:
 ## st_value(a) = 8
 # LE64-NEXT:         addi    zero, zero, 0x0
 # LE64-NEXT:         addi    zero, zero, 0x0
@@ -97,12 +99,14 @@
 # LE64-NEXT:         addi    a0, zero, 0x8
 # LE64-NEXT:         add     a0, a0, tp
 ## st_value(b) = 2047
+# LE64-LABEL: <.Ltlsdesc_hi1>:
 # LE64-NEXT:         addi    zero, zero, 0x0
 # LE64-NEXT:         addi    zero, zero, 0x0
 # LE64-NEXT:         addi    zero, zero, 0x0
 # LE64-NEXT:         addi    a0, zero, 0x7ff
 # LE64-NEXT:         add     a0, a0, tp
 ## st_value(c) = 2048
+# LE64-LABEL: <.Ltlsdesc_hi2>:
 # LE64-NEXT:         addi    zero, zero, 0x0
 # LE64-NEXT:         addi    zero, zero, 0x0
 # LE64-NEXT:         lui     a0, 0x1
@@ -116,18 +120,20 @@
 # IE64:       .got     00000010 00000000000123a8
 
 ## a and b are optimized to use LE. c is optimized to IE.
-# IE64-LABEL: <.text>:
+# IE64-LABEL: <.Ltlsdesc_hi0>:
 # IE64-NEXT:         addi    zero, zero, 0x0
 # IE64-NEXT:         addi    zero, zero, 0x0
 # IE64-NEXT:         addi    zero, zero, 0x0
 # IE64-NEXT:         addi    a0, zero, 0x8
 # IE64-NEXT:         add     a0, a0, tp
+# IE64-LABEL: <.Ltlsdesc_hi1>:
 # IE64-NEXT:         addi    zero, zero, 0x0
 # IE64-NEXT:         addi    zero, zero, 0x0
 # IE64-NEXT:         addi    zero, zero, 0x0
 # IE64-NEXT:         addi    a0, zero, 0x7ff
 # IE64-NEXT:         add     a0, a0, tp
 ## &.got[c]-. = 0x123a8+8 - 0x112b8 = 0x10f8
+# IE64-LABEL: <.Ltlsdesc_hi2>:
 # IE64-NEXT:         addi    zero, zero, 0x0
 # IE64-NEXT:         addi    zero, zero, 0x0
 # IE64-NEXT:  112b8: auipc   a0, 0x1
@@ -136,7 +142,7 @@
 
 # IE32:       .got     00000008 00012248
 
-# IE32-LABEL: <.text>:
+# IE32-LABEL: <.Ltlsdesc_hi0>:
 ## st_value(a) = 8
 # IE32-NEXT:         addi    zero, zero, 0x0
 # IE32-NEXT:         addi    zero, zero, 0x0
@@ -144,21 +150,20 @@
 # IE32-NEXT:         addi    a0, zero, 0x8
 # IE32-NEXT:         add     a0, a0, tp
 ## st_value(b) = 2047
+# IE32-LABEL: <.Ltlsdesc_hi1>:
 # IE32-NEXT:         addi    zero, zero, 0x0
 # IE32-NEXT:         addi    zero, zero, 0x0
 # IE32-NEXT:         addi    zero, zero, 0x0
 # IE32-NEXT:         addi    a0, zero, 0x7ff
 # IE32-NEXT:         add     a0, a0, tp
 ## &.got[c]-. = 0x12248+4 - 0x111cc = 0x1080
+# IE32-LABEL: <.Ltlsdesc_hi2>:
 # IE32-NEXT:         addi    zero, zero, 0x0
 # IE32-NEXT:         addi    zero, zero, 0x0
 # IE32-NEXT:  111cc: auipc   a0, 0x1
 # IE32-NEXT:         lw      a0, 0x80(a0)
 # IE32-NEXT:         add     a0, a0, tp
 
-## FIXME This should not pass, but the code MC layer needs a fix to prevent this.
-# BADTLSLABEL: error: d.{{.*}}.o has an STT_TLS symbol but doesn't have an SHF_TLS section
-
 #--- a.s
 .macro load dst, src
 .ifdef ELF32
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
index 254a9a4bc0ef..b8e0f3a867f4 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
@@ -207,8 +207,6 @@ void RISCVMCExpr::fixELFSymbolsInTLSFixups(MCAssembler &Asm) const {
   case VK_RISCV_TLS_GOT_HI:
   case VK_RISCV_TLS_GD_HI:
   case VK_RISCV_TLSDESC_HI:
-  case VK_RISCV_TLSDESC_ADD_LO:
-  case VK_RISCV_TLSDESC_LOAD_LO:
     break;
   }
 
-- 
GitLab


From e9639e9c0636d9e2b9591c2cdac5cac75e363e77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Fri, 22 Mar 2024 12:56:45 -0700
Subject: [PATCH 004/404] [flang][NFC] Extract FIROpConversion to its own files
 (#86213)

This PR extracts `FIROpConversion` and `FIROpAndTypeConversion`
templated base patterns to a header file. All the functions from
FIROpConversion that do not require the template argument are moved to a
base class named `ConvertFIRToLLVMPattern`.
This move is done so the `FIROpConversion` pattern and all its utility
functions can be reused outside of the codegen pass.

For the most part the code is only moved to the new files and not
modified. The only update is that addition of the PatternBenefit
argument with a default value to the constructor so it can be forwarded
to the `ConversionPattern` ctor.

This split is done in a similar way for the `ConvertOpToLLVMPattern`
base pattern that is based on the `ConvertToLLVMPattern` base class in
`mlir/include/mlir/Conversion/LLVMCommon/Pattern.h`.
---
 .../flang/Optimizer/CodeGen/FIROpPatterns.h   | 248 +++++++++
 .../flang/Optimizer/CodeGen/TypeConverter.h   |   4 +-
 flang/lib/Optimizer/CodeGen/CMakeLists.txt    |   1 +
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       | 486 +++---------------
 flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp | 315 ++++++++++++
 5 files changed, 636 insertions(+), 418 deletions(-)
 create mode 100644 flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
 create mode 100644 flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp

diff --git a/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
new file mode 100644
index 000000000000..06a44f188565
--- /dev/null
+++ b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
@@ -0,0 +1,248 @@
+//===-- FIROpPatterns.h -- FIR operation conversion patterns ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_OPTIMIZER_CODEGEN_FIROPPATTERNS_H
+#define FORTRAN_OPTIMIZER_CODEGEN_FIROPPATTERNS_H
+
+#include "flang/Optimizer/CodeGen/TypeConverter.h"
+#include "mlir/Conversion/LLVMCommon/Pattern.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+
+namespace fir {
+
+struct FIRToLLVMPassOptions;
+
+static constexpr unsigned defaultAddressSpace = 0u;
+
+class ConvertFIRToLLVMPattern : public mlir::ConvertToLLVMPattern {
+public:
+  ConvertFIRToLLVMPattern(llvm::StringRef rootOpName,
+                          mlir::MLIRContext *context,
+                          const fir::LLVMTypeConverter &typeConverter,
+                          const fir::FIRToLLVMPassOptions &options,
+                          mlir::PatternBenefit benefit = 1);
+
+protected:
+  mlir::Type convertType(mlir::Type ty) const {
+    return lowerTy().convertType(ty);
+  }
+
+  // Convert FIR type to LLVM without turning fir.box<T> into memory
+  // reference.
+  mlir::Type convertObjectType(mlir::Type firType) const;
+
+  mlir::LLVM::ConstantOp
+  genI32Constant(mlir::Location loc, mlir::ConversionPatternRewriter &rewriter,
+                 int value) const;
+
+  mlir::LLVM::ConstantOp
+  genConstantOffset(mlir::Location loc,
+                    mlir::ConversionPatternRewriter &rewriter,
+                    int offset) const;
+
+  /// Perform an extension or truncation as needed on an integer value. Lowering
+  /// to the specific target may involve some sign-extending or truncation of
+  /// values, particularly to fit them from abstract box types to the
+  /// appropriate reified structures.
+  mlir::Value integerCast(mlir::Location loc,
+                          mlir::ConversionPatternRewriter &rewriter,
+                          mlir::Type ty, mlir::Value val) const;
+  struct TypePair {
+    mlir::Type fir;
+    mlir::Type llvm;
+  };
+
+  TypePair getBoxTypePair(mlir::Type firBoxTy) const;
+
+  /// Construct code sequence to extract the specific value from a `fir.box`.
+  mlir::Value getValueFromBox(mlir::Location loc, TypePair boxTy,
+                              mlir::Value box, mlir::Type resultTy,
+                              mlir::ConversionPatternRewriter &rewriter,
+                              int boxValue) const;
+
+  /// Method to construct code sequence to get the triple for dimension `dim`
+  /// from a box.
+  llvm::SmallVector<mlir::Value, 3>
+  getDimsFromBox(mlir::Location loc, llvm::ArrayRef<mlir::Type> retTys,
+                 TypePair boxTy, mlir::Value box, mlir::Value dim,
+                 mlir::ConversionPatternRewriter &rewriter) const;
+
+  llvm::SmallVector<mlir::Value, 3>
+  getDimsFromBox(mlir::Location loc, llvm::ArrayRef<mlir::Type> retTys,
+                 TypePair boxTy, mlir::Value box, int dim,
+                 mlir::ConversionPatternRewriter &rewriter) const;
+
+  mlir::Value
+  loadDimFieldFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box,
+                      mlir::Value dim, int off, mlir::Type ty,
+                      mlir::ConversionPatternRewriter &rewriter) const;
+
+  mlir::Value
+  getDimFieldFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box,
+                     int dim, int off, mlir::Type ty,
+                     mlir::ConversionPatternRewriter &rewriter) const;
+
+  mlir::Value getStrideFromBox(mlir::Location loc, TypePair boxTy,
+                               mlir::Value box, unsigned dim,
+                               mlir::ConversionPatternRewriter &rewriter) const;
+
+  /// Read base address from a fir.box. Returned address has type ty.
+  mlir::Value
+  getBaseAddrFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box,
+                     mlir::ConversionPatternRewriter &rewriter) const;
+
+  mlir::Value
+  getElementSizeFromBox(mlir::Location loc, mlir::Type resultTy, TypePair boxTy,
+                        mlir::Value box,
+                        mlir::ConversionPatternRewriter &rewriter) const;
+
+  // Get the element type given an LLVM type that is of the form
+  // (array|struct|vector)+ and the provided indexes.
+  mlir::Type getBoxEleTy(mlir::Type type,
+                         llvm::ArrayRef<std::int64_t> indexes) const;
+
+  // Return LLVM type of the object described by a fir.box of \p boxType.
+  mlir::Type getLlvmObjectTypeFromBoxType(mlir::Type boxType) const;
+
+  /// Read the address of the type descriptor from a box.
+  mlir::Value
+  loadTypeDescAddress(mlir::Location loc, TypePair boxTy, mlir::Value box,
+                      mlir::ConversionPatternRewriter &rewriter) const;
+
+  // Load the attribute from the \p box and perform a check against \p maskValue
+  // The final comparison is implemented as `(attribute & maskValue) != 0`.
+  mlir::Value genBoxAttributeCheck(mlir::Location loc, TypePair boxTy,
+                                   mlir::Value box,
+                                   mlir::ConversionPatternRewriter &rewriter,
+                                   unsigned maskValue) const;
+
+  template <typename... ARGS>
+  mlir::LLVM::GEPOp genGEP(mlir::Location loc, mlir::Type ty,
+                           mlir::ConversionPatternRewriter &rewriter,
+                           mlir::Value base, ARGS... args) const {
+    llvm::SmallVector<mlir::LLVM::GEPArg> cv = {args...};
+    auto llvmPtrTy =
+        mlir::LLVM::LLVMPointerType::get(ty.getContext(), /*addressSpace=*/0);
+    return rewriter.create<mlir::LLVM::GEPOp>(loc, llvmPtrTy, ty, base, cv);
+  }
+
+  // Find the Block in which the alloca should be inserted.
+  // The order to recursively find the proper block:
+  // 1. An OpenMP Op that will be outlined.
+  // 2. A LLVMFuncOp
+  // 3. The first ancestor that is an OpenMP Op or a LLVMFuncOp
+  mlir::Block *getBlockForAllocaInsert(mlir::Operation *op) const;
+
+  // Generate an alloca of size 1 for an object of type \p llvmObjectTy in the
+  // allocation address space provided for the architecture in the DataLayout
+  // specification. If the address space is different from the devices
+  // program address space we perform a cast. In the case of most architectures
+  // the program and allocation address space will be the default of 0 and no
+  // cast will be emitted.
+  mlir::Value
+  genAllocaAndAddrCastWithType(mlir::Location loc, mlir::Type llvmObjectTy,
+                               unsigned alignment,
+                               mlir::ConversionPatternRewriter &rewriter) const;
+
+  const fir::LLVMTypeConverter &lowerTy() const {
+    return *static_cast<const fir::LLVMTypeConverter *>(
+        this->getTypeConverter());
+  }
+
+  void attachTBAATag(mlir::LLVM::AliasAnalysisOpInterface op,
+                     mlir::Type baseFIRType, mlir::Type accessFIRType,
+                     mlir::LLVM::GEPOp gep) const {
+    lowerTy().attachTBAATag(op, baseFIRType, accessFIRType, gep);
+  }
+
+  unsigned
+  getAllocaAddressSpace(mlir::ConversionPatternRewriter &rewriter) const;
+
+  unsigned
+  getProgramAddressSpace(mlir::ConversionPatternRewriter &rewriter) const;
+
+  const fir::FIRToLLVMPassOptions &options;
+
+  using ConvertToLLVMPattern::match;
+  using ConvertToLLVMPattern::matchAndRewrite;
+};
+
+template <typename SourceOp>
+class FIROpConversion : public ConvertFIRToLLVMPattern {
+public:
+  using OpAdaptor = typename SourceOp::Adaptor;
+
+  explicit FIROpConversion(const LLVMTypeConverter &typeConverter,
+                           const fir::FIRToLLVMPassOptions &options,
+                           mlir::PatternBenefit benefit = 1)
+      : ConvertFIRToLLVMPattern(SourceOp::getOperationName(),
+                                &typeConverter.getContext(), typeConverter,
+                                options, benefit) {}
+
+  /// Wrappers around the RewritePattern methods that pass the derived op type.
+  void rewrite(mlir::Operation *op, mlir::ArrayRef<mlir::Value> operands,
+               mlir::ConversionPatternRewriter &rewriter) const final {
+    rewrite(mlir::cast<SourceOp>(op),
+            OpAdaptor(operands, mlir::cast<SourceOp>(op)), rewriter);
+  }
+  mlir::LogicalResult match(mlir::Operation *op) const final {
+    return match(mlir::cast<SourceOp>(op));
+  }
+  mlir::LogicalResult
+  matchAndRewrite(mlir::Operation *op, mlir::ArrayRef<mlir::Value> operands,
+                  mlir::ConversionPatternRewriter &rewriter) const final {
+    return matchAndRewrite(mlir::cast<SourceOp>(op),
+                           OpAdaptor(operands, mlir::cast<SourceOp>(op)),
+                           rewriter);
+  }
+
+  /// Rewrite and Match methods that operate on the SourceOp type. These must be
+  /// overridden by the derived pattern class.
+  virtual mlir::LogicalResult match(SourceOp op) const {
+    llvm_unreachable("must override match or matchAndRewrite");
+  }
+  virtual void rewrite(SourceOp op, OpAdaptor adaptor,
+                       mlir::ConversionPatternRewriter &rewriter) const {
+    llvm_unreachable("must override rewrite or matchAndRewrite");
+  }
+  virtual mlir::LogicalResult
+  matchAndRewrite(SourceOp op, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const {
+    if (mlir::failed(match(op)))
+      return mlir::failure();
+    rewrite(op, adaptor, rewriter);
+    return mlir::success();
+  }
+
+private:
+  using ConvertFIRToLLVMPattern::matchAndRewrite;
+  using ConvertToLLVMPattern::match;
+};
+
+/// FIR conversion pattern template
+template <typename FromOp>
+class FIROpAndTypeConversion : public FIROpConversion<FromOp> {
+public:
+  using FIROpConversion<FromOp>::FIROpConversion;
+  using OpAdaptor = typename FromOp::Adaptor;
+
+  mlir::LogicalResult
+  matchAndRewrite(FromOp op, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const final {
+    mlir::Type ty = this->convertType(op.getType());
+    return doRewrite(op, ty, adaptor, rewriter);
+  }
+
+  virtual mlir::LogicalResult
+  doRewrite(FromOp addr, mlir::Type ty, OpAdaptor adaptor,
+            mlir::ConversionPatternRewriter &rewriter) const = 0;
+};
+
+} // namespace fir
+
+#endif // FORTRAN_OPTIMIZER_CODEGEN_FIROPPATTERNS_H
diff --git a/flang/include/flang/Optimizer/CodeGen/TypeConverter.h b/flang/include/flang/Optimizer/CodeGen/TypeConverter.h
index 396c13639255..79b3bfe4e80e 100644
--- a/flang/include/flang/Optimizer/CodeGen/TypeConverter.h
+++ b/flang/include/flang/Optimizer/CodeGen/TypeConverter.h
@@ -94,8 +94,8 @@ public:
   // to LLVM IR dialect here.
   //
   // fir.complex<T> | std.complex<T>    --> llvm<"{t,t}">
-  template <typename C> mlir::Type convertComplexType(C cmplx) const {
-    LLVM_DEBUG(llvm::dbgs() << "type convert: " << cmplx << '\n');
+  template <typename C>
+  mlir::Type convertComplexType(C cmplx) const {
     auto eleTy = cmplx.getElementType();
     return convertType(specifics->complexMemoryType(eleTy));
   }
diff --git a/flang/lib/Optimizer/CodeGen/CMakeLists.txt b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
index 175ab9fefda2..879bc28d017a 100644
--- a/flang/lib/Optimizer/CodeGen/CMakeLists.txt
+++ b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
@@ -3,6 +3,7 @@ add_flang_library(FIRCodeGen
   CGOps.cpp
   CodeGen.cpp
   CodeGenOpenMP.cpp
+  FIROpPatterns.cpp
   PreCGRewrite.cpp
   TBAABuilder.cpp
   Target.cpp
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index faf90ef6b50a..06ce84f1543a 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -14,6 +14,8 @@
 
 #include "CGOps.h"
 #include "flang/Optimizer/CodeGen/CodeGenOpenMP.h"
+#include "flang/Optimizer/CodeGen/FIROpPatterns.h"
+#include "flang/Optimizer/CodeGen/TypeConverter.h"
 #include "flang/Optimizer/Dialect/FIRAttr.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
@@ -58,42 +60,14 @@ namespace fir {
 
 #define DEBUG_TYPE "flang-codegen"
 
-// fir::LLVMTypeConverter for converting to LLVM IR dialect types.
-#include "flang/Optimizer/CodeGen/TypeConverter.h"
-
 // TODO: This should really be recovered from the specified target.
 static constexpr unsigned defaultAlign = 8;
-static constexpr unsigned defaultAddressSpace = 0u;
 
 /// `fir.box` attribute values as defined for CFI_attribute_t in
 /// flang/ISO_Fortran_binding.h.
 static constexpr unsigned kAttrPointer = CFI_attribute_pointer;
 static constexpr unsigned kAttrAllocatable = CFI_attribute_allocatable;
 
-static inline unsigned
-getAllocaAddressSpace(mlir::ConversionPatternRewriter &rewriter) {
-  mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp();
-  assert(parentOp != nullptr &&
-         "expected insertion block to have parent operation");
-  if (auto module = parentOp->getParentOfType<mlir::ModuleOp>())
-    if (mlir::Attribute addrSpace =
-            mlir::DataLayout(module).getAllocaMemorySpace())
-      return llvm::cast<mlir::IntegerAttr>(addrSpace).getUInt();
-  return defaultAddressSpace;
-}
-
-static inline unsigned
-getProgramAddressSpace(mlir::ConversionPatternRewriter &rewriter) {
-  mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp();
-  assert(parentOp != nullptr &&
-         "expected insertion block to have parent operation");
-  if (auto module = parentOp->getParentOfType<mlir::ModuleOp>())
-    if (mlir::Attribute addrSpace =
-            mlir::DataLayout(module).getProgramMemorySpace())
-      return llvm::cast<mlir::IntegerAttr>(addrSpace).getUInt();
-  return defaultAddressSpace;
-}
-
 static inline mlir::Type getLlvmPtrType(mlir::MLIRContext *context,
                                         unsigned addressSpace = 0) {
   return mlir::LLVM::LLVMPointerType::get(context, addressSpace);
@@ -151,333 +125,9 @@ static unsigned getLenParamFieldId(mlir::Type ty) {
   return getTypeDescFieldId(ty) + 1;
 }
 
-namespace {
-/// FIR conversion pattern template
-template <typename FromOp>
-class FIROpConversion : public mlir::ConvertOpToLLVMPattern<FromOp> {
-public:
-  explicit FIROpConversion(const fir::LLVMTypeConverter &lowering,
-                           const fir::FIRToLLVMPassOptions &options)
-      : mlir::ConvertOpToLLVMPattern<FromOp>(lowering), options(options) {}
-
-protected:
-  mlir::Type convertType(mlir::Type ty) const {
-    return lowerTy().convertType(ty);
-  }
-
-  // Convert FIR type to LLVM without turning fir.box<T> into memory
-  // reference.
-  mlir::Type convertObjectType(mlir::Type firType) const {
-    if (auto boxTy = firType.dyn_cast<fir::BaseBoxType>())
-      return lowerTy().convertBoxTypeAsStruct(boxTy);
-    return lowerTy().convertType(firType);
-  }
-
-  mlir::LLVM::ConstantOp
-  genI32Constant(mlir::Location loc, mlir::ConversionPatternRewriter &rewriter,
-                 int value) const {
-    mlir::Type i32Ty = rewriter.getI32Type();
-    mlir::IntegerAttr attr = rewriter.getI32IntegerAttr(value);
-    return rewriter.create<mlir::LLVM::ConstantOp>(loc, i32Ty, attr);
-  }
-
-  mlir::LLVM::ConstantOp
-  genConstantOffset(mlir::Location loc,
-                    mlir::ConversionPatternRewriter &rewriter,
-                    int offset) const {
-    mlir::Type ity = lowerTy().offsetType();
-    mlir::IntegerAttr cattr = rewriter.getI32IntegerAttr(offset);
-    return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
-  }
-
-  /// Perform an extension or truncation as needed on an integer value. Lowering
-  /// to the specific target may involve some sign-extending or truncation of
-  /// values, particularly to fit them from abstract box types to the
-  /// appropriate reified structures.
-  mlir::Value integerCast(mlir::Location loc,
-                          mlir::ConversionPatternRewriter &rewriter,
-                          mlir::Type ty, mlir::Value val) const {
-    auto valTy = val.getType();
-    // If the value was not yet lowered, lower its type so that it can
-    // be used in getPrimitiveTypeSizeInBits.
-    if (!valTy.isa<mlir::IntegerType>())
-      valTy = convertType(valTy);
-    auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
-    auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy);
-    if (toSize < fromSize)
-      return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
-    if (toSize > fromSize)
-      return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
-    return val;
-  }
-
-  struct TypePair {
-    mlir::Type fir;
-    mlir::Type llvm;
-  };
-
-  TypePair getBoxTypePair(mlir::Type firBoxTy) const {
-    mlir::Type llvmBoxTy = lowerTy().convertBoxTypeAsStruct(
-        mlir::cast<fir::BaseBoxType>(firBoxTy));
-    return TypePair{firBoxTy, llvmBoxTy};
-  }
-
-  /// Construct code sequence to extract the specific value from a `fir.box`.
-  mlir::Value getValueFromBox(mlir::Location loc, TypePair boxTy,
-                              mlir::Value box, mlir::Type resultTy,
-                              mlir::ConversionPatternRewriter &rewriter,
-                              int boxValue) const {
-    if (box.getType().isa<mlir::LLVM::LLVMPointerType>()) {
-      auto pty = ::getLlvmPtrType(resultTy.getContext());
-      auto p = rewriter.create<mlir::LLVM::GEPOp>(
-          loc, pty, boxTy.llvm, box,
-          llvm::ArrayRef<mlir::LLVM::GEPArg>{0, boxValue});
-      auto loadOp = rewriter.create<mlir::LLVM::LoadOp>(loc, resultTy, p);
-      attachTBAATag(loadOp, boxTy.fir, nullptr, p);
-      return loadOp;
-    }
-    return rewriter.create<mlir::LLVM::ExtractValueOp>(loc, box, boxValue);
-  }
-
-  /// Method to construct code sequence to get the triple for dimension `dim`
-  /// from a box.
-  llvm::SmallVector<mlir::Value, 3>
-  getDimsFromBox(mlir::Location loc, llvm::ArrayRef<mlir::Type> retTys,
-                 TypePair boxTy, mlir::Value box, mlir::Value dim,
-                 mlir::ConversionPatternRewriter &rewriter) const {
-    mlir::Value l0 =
-        loadDimFieldFromBox(loc, boxTy, box, dim, 0, retTys[0], rewriter);
-    mlir::Value l1 =
-        loadDimFieldFromBox(loc, boxTy, box, dim, 1, retTys[1], rewriter);
-    mlir::Value l2 =
-        loadDimFieldFromBox(loc, boxTy, box, dim, 2, retTys[2], rewriter);
-    return {l0, l1, l2};
-  }
-
-  llvm::SmallVector<mlir::Value, 3>
-  getDimsFromBox(mlir::Location loc, llvm::ArrayRef<mlir::Type> retTys,
-                 TypePair boxTy, mlir::Value box, int dim,
-                 mlir::ConversionPatternRewriter &rewriter) const {
-    mlir::Value l0 =
-        getDimFieldFromBox(loc, boxTy, box, dim, 0, retTys[0], rewriter);
-    mlir::Value l1 =
-        getDimFieldFromBox(loc, boxTy, box, dim, 1, retTys[1], rewriter);
-    mlir::Value l2 =
-        getDimFieldFromBox(loc, boxTy, box, dim, 2, retTys[2], rewriter);
-    return {l0, l1, l2};
-  }
-
-  mlir::Value
-  loadDimFieldFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box,
-                      mlir::Value dim, int off, mlir::Type ty,
-                      mlir::ConversionPatternRewriter &rewriter) const {
-    assert(box.getType().isa<mlir::LLVM::LLVMPointerType>() &&
-           "descriptor inquiry with runtime dim can only be done on descriptor "
-           "in memory");
-    mlir::LLVM::GEPOp p = genGEP(loc, boxTy.llvm, rewriter, box, 0,
-                                 static_cast<int>(kDimsPosInBox), dim, off);
-    auto loadOp = rewriter.create<mlir::LLVM::LoadOp>(loc, ty, p);
-    attachTBAATag(loadOp, boxTy.fir, nullptr, p);
-    return loadOp;
-  }
-
-  mlir::Value
-  getDimFieldFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box,
-                     int dim, int off, mlir::Type ty,
-                     mlir::ConversionPatternRewriter &rewriter) const {
-    if (box.getType().isa<mlir::LLVM::LLVMPointerType>()) {
-      mlir::LLVM::GEPOp p = genGEP(loc, boxTy.llvm, rewriter, box, 0,
-                                   static_cast<int>(kDimsPosInBox), dim, off);
-      auto loadOp = rewriter.create<mlir::LLVM::LoadOp>(loc, ty, p);
-      attachTBAATag(loadOp, boxTy.fir, nullptr, p);
-      return loadOp;
-    }
-    return rewriter.create<mlir::LLVM::ExtractValueOp>(
-        loc, box, llvm::ArrayRef<std::int64_t>{kDimsPosInBox, dim, off});
-  }
-
-  mlir::Value
-  getStrideFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box,
-                   unsigned dim,
-                   mlir::ConversionPatternRewriter &rewriter) const {
-    auto idxTy = lowerTy().indexType();
-    return getDimFieldFromBox(loc, boxTy, box, dim, kDimStridePos, idxTy,
-                              rewriter);
-  }
-
-  /// Read base address from a fir.box. Returned address has type ty.
-  mlir::Value
-  getBaseAddrFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box,
-                     mlir::ConversionPatternRewriter &rewriter) const {
-    mlir::Type resultTy = ::getLlvmPtrType(boxTy.llvm.getContext());
-    return getValueFromBox(loc, boxTy, box, resultTy, rewriter, kAddrPosInBox);
-  }
-
-  mlir::Value
-  getElementSizeFromBox(mlir::Location loc, mlir::Type resultTy, TypePair boxTy,
-                        mlir::Value box,
-                        mlir::ConversionPatternRewriter &rewriter) const {
-    return getValueFromBox(loc, boxTy, box, resultTy, rewriter,
-                           kElemLenPosInBox);
-  }
-
-  // Get the element type given an LLVM type that is of the form
-  // (array|struct|vector)+ and the provided indexes.
-  static mlir::Type getBoxEleTy(mlir::Type type,
-                                llvm::ArrayRef<std::int64_t> indexes) {
-    for (unsigned i : indexes) {
-      if (auto t = type.dyn_cast<mlir::LLVM::LLVMStructType>()) {
-        assert(!t.isOpaque() && i < t.getBody().size());
-        type = t.getBody()[i];
-      } else if (auto t = type.dyn_cast<mlir::LLVM::LLVMArrayType>()) {
-        type = t.getElementType();
-      } else if (auto t = type.dyn_cast<mlir::VectorType>()) {
-        type = t.getElementType();
-      } else {
-        fir::emitFatalError(mlir::UnknownLoc::get(type.getContext()),
-                            "request for invalid box element type");
-      }
-    }
-    return type;
-  }
-
-  // Return LLVM type of the object described by a fir.box of \p boxType.
-  mlir::Type getLlvmObjectTypeFromBoxType(mlir::Type boxType) const {
-    mlir::Type objectType = fir::dyn_cast_ptrOrBoxEleTy(boxType);
-    assert(objectType && "boxType must be a box type");
-    return this->convertType(objectType);
-  }
-
-  /// Read the address of the type descriptor from a box.
-  mlir::Value
-  loadTypeDescAddress(mlir::Location loc, TypePair boxTy, mlir::Value box,
-                      mlir::ConversionPatternRewriter &rewriter) const {
-    unsigned typeDescFieldId = getTypeDescFieldId(boxTy.fir);
-    mlir::Type tdescType = lowerTy().convertTypeDescType(rewriter.getContext());
-    return getValueFromBox(loc, boxTy, box, tdescType, rewriter,
-                           typeDescFieldId);
-  }
-
-  // Load the attribute from the \p box and perform a check against \p maskValue
-  // The final comparison is implemented as `(attribute & maskValue) != 0`.
-  mlir::Value genBoxAttributeCheck(mlir::Location loc, TypePair boxTy,
-                                   mlir::Value box,
-                                   mlir::ConversionPatternRewriter &rewriter,
-                                   unsigned maskValue) const {
-    mlir::Type attrTy = rewriter.getI32Type();
-    mlir::Value attribute =
-        getValueFromBox(loc, boxTy, box, attrTy, rewriter, kAttributePosInBox);
-    mlir::LLVM::ConstantOp attrMask =
-        genConstantOffset(loc, rewriter, maskValue);
-    auto maskRes =
-        rewriter.create<mlir::LLVM::AndOp>(loc, attrTy, attribute, attrMask);
-    mlir::LLVM::ConstantOp c0 = genConstantOffset(loc, rewriter, 0);
-    return rewriter.create<mlir::LLVM::ICmpOp>(
-        loc, mlir::LLVM::ICmpPredicate::ne, maskRes, c0);
-  }
-
-  template <typename... ARGS>
-  mlir::LLVM::GEPOp genGEP(mlir::Location loc, mlir::Type ty,
-                           mlir::ConversionPatternRewriter &rewriter,
-                           mlir::Value base, ARGS... args) const {
-    llvm::SmallVector<mlir::LLVM::GEPArg> cv = {args...};
-    auto llvmPtrTy = ::getLlvmPtrType(ty.getContext());
-    return rewriter.create<mlir::LLVM::GEPOp>(loc, llvmPtrTy, ty, base, cv);
-  }
-
-  // Find the Block in which the alloca should be inserted.
-  // The order to recursively find the proper block:
-  // 1. An OpenMP Op that will be outlined.
-  // 2. A LLVMFuncOp
-  // 3. The first ancestor that is an OpenMP Op or a LLVMFuncOp
-  static mlir::Block *getBlockForAllocaInsert(mlir::Operation *op) {
-    if (auto iface =
-            mlir::dyn_cast<mlir::omp::OutlineableOpenMPOpInterface>(op))
-      return iface.getAllocaBlock();
-    if (auto llvmFuncOp = mlir::dyn_cast<mlir::LLVM::LLVMFuncOp>(op))
-      return &llvmFuncOp.front();
-    return getBlockForAllocaInsert(op->getParentOp());
-  }
-
-  // Generate an alloca of size 1 for an object of type \p llvmObjectTy in the
-  // allocation address space provided for the architecture in the DataLayout
-  // specification. If the address space is different from the devices
-  // program address space we perform a cast. In the case of most architectures
-  // the program and allocation address space will be the default of 0 and no
-  // cast will be emitted.
-  mlir::Value genAllocaAndAddrCastWithType(
-      mlir::Location loc, mlir::Type llvmObjectTy, unsigned alignment,
-      mlir::ConversionPatternRewriter &rewriter) const {
-    auto thisPt = rewriter.saveInsertionPoint();
-    mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp();
-    if (mlir::isa<mlir::omp::DeclareReductionOp>(parentOp)) {
-      // DeclareReductionOp has multiple child regions. We want to get the first
-      // block of whichever of those regions we are currently in
-      mlir::Region *parentRegion = rewriter.getInsertionBlock()->getParent();
-      rewriter.setInsertionPointToStart(&parentRegion->front());
-    } else {
-      mlir::Block *insertBlock = getBlockForAllocaInsert(parentOp);
-      rewriter.setInsertionPointToStart(insertBlock);
-    }
-    auto size = genI32Constant(loc, rewriter, 1);
-    unsigned allocaAs = getAllocaAddressSpace(rewriter);
-    unsigned programAs = getProgramAddressSpace(rewriter);
-
-    mlir::Value al = rewriter.create<mlir::LLVM::AllocaOp>(
-        loc, ::getLlvmPtrType(llvmObjectTy.getContext(), allocaAs),
-        llvmObjectTy, size, alignment);
-
-    // if our allocation address space, is not the same as the program address
-    // space, then we must emit a cast to the program address space before use.
-    // An example case would be on AMDGPU, where the allocation address space is
-    // the numeric value 5 (private), and the program address space is 0
-    // (generic).
-    if (allocaAs != programAs) {
-      al = rewriter.create<mlir::LLVM::AddrSpaceCastOp>(
-          loc, ::getLlvmPtrType(llvmObjectTy.getContext(), programAs), al);
-    }
-
-    rewriter.restoreInsertionPoint(thisPt);
-    return al;
-  }
-
-  const fir::LLVMTypeConverter &lowerTy() const {
-    return *static_cast<const fir::LLVMTypeConverter *>(
-        this->getTypeConverter());
-  }
-
-  void attachTBAATag(mlir::LLVM::AliasAnalysisOpInterface op,
-                     mlir::Type baseFIRType, mlir::Type accessFIRType,
-                     mlir::LLVM::GEPOp gep) const {
-    lowerTy().attachTBAATag(op, baseFIRType, accessFIRType, gep);
-  }
-
-  const fir::FIRToLLVMPassOptions &options;
-};
-
-/// FIR conversion pattern template
-template <typename FromOp>
-class FIROpAndTypeConversion : public FIROpConversion<FromOp> {
-public:
-  using FIROpConversion<FromOp>::FIROpConversion;
-  using OpAdaptor = typename FromOp::Adaptor;
-
-  mlir::LogicalResult
-  matchAndRewrite(FromOp op, OpAdaptor adaptor,
-                  mlir::ConversionPatternRewriter &rewriter) const final {
-    mlir::Type ty = this->convertType(op.getType());
-    return doRewrite(op, ty, adaptor, rewriter);
-  }
-
-  virtual mlir::LogicalResult
-  doRewrite(FromOp addr, mlir::Type ty, OpAdaptor adaptor,
-            mlir::ConversionPatternRewriter &rewriter) const = 0;
-};
-} // namespace
-
 namespace {
 /// Lower `fir.address_of` operation to `llvm.address_of` operation.
-struct AddrOfOpConversion : public FIROpConversion<fir::AddrOfOp> {
+struct AddrOfOpConversion : public fir::FIROpConversion<fir::AddrOfOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -539,7 +189,7 @@ genAllocationScaleSize(OP op, mlir::Type ity,
 
 namespace {
 /// convert to LLVM IR dialect `alloca`
-struct AllocaOpConversion : public FIROpConversion<fir::AllocaOp> {
+struct AllocaOpConversion : public fir::FIROpConversion<fir::AllocaOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -624,7 +274,7 @@ struct AllocaOpConversion : public FIROpConversion<fir::AllocaOp> {
 namespace {
 /// Lower `fir.box_addr` to the sequence of operations to extract the first
 /// element of the box.
-struct BoxAddrOpConversion : public FIROpConversion<fir::BoxAddrOp> {
+struct BoxAddrOpConversion : public fir::FIROpConversion<fir::BoxAddrOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -645,7 +295,7 @@ struct BoxAddrOpConversion : public FIROpConversion<fir::BoxAddrOp> {
 
 /// Convert `!fir.boxchar_len` to  `!llvm.extractvalue` for the 2nd part of the
 /// boxchar.
-struct BoxCharLenOpConversion : public FIROpConversion<fir::BoxCharLenOp> {
+struct BoxCharLenOpConversion : public fir::FIROpConversion<fir::BoxCharLenOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -668,7 +318,7 @@ struct BoxCharLenOpConversion : public FIROpConversion<fir::BoxCharLenOp> {
 /// Lower `fir.box_dims` to a sequence of operations to extract the requested
 /// dimension information from the boxed value.
 /// Result in a triple set of GEPs and loads.
-struct BoxDimsOpConversion : public FIROpConversion<fir::BoxDimsOp> {
+struct BoxDimsOpConversion : public fir::FIROpConversion<fir::BoxDimsOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -690,7 +340,7 @@ struct BoxDimsOpConversion : public FIROpConversion<fir::BoxDimsOp> {
 
 /// Lower `fir.box_elesize` to a sequence of operations ro extract the size of
 /// an element in the boxed value.
-struct BoxEleSizeOpConversion : public FIROpConversion<fir::BoxEleSizeOp> {
+struct BoxEleSizeOpConversion : public fir::FIROpConversion<fir::BoxEleSizeOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -708,7 +358,7 @@ struct BoxEleSizeOpConversion : public FIROpConversion<fir::BoxEleSizeOp> {
 
 /// Lower `fir.box_isalloc` to a sequence of operations to determine if the
 /// boxed value was from an ALLOCATABLE entity.
-struct BoxIsAllocOpConversion : public FIROpConversion<fir::BoxIsAllocOp> {
+struct BoxIsAllocOpConversion : public fir::FIROpConversion<fir::BoxIsAllocOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -726,7 +376,7 @@ struct BoxIsAllocOpConversion : public FIROpConversion<fir::BoxIsAllocOp> {
 
 /// Lower `fir.box_isarray` to a sequence of operations to determine if the
 /// boxed is an array.
-struct BoxIsArrayOpConversion : public FIROpConversion<fir::BoxIsArrayOp> {
+struct BoxIsArrayOpConversion : public fir::FIROpConversion<fir::BoxIsArrayOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -746,7 +396,7 @@ struct BoxIsArrayOpConversion : public FIROpConversion<fir::BoxIsArrayOp> {
 
 /// Lower `fir.box_isptr` to a sequence of operations to determined if the
 /// boxed value was from a POINTER entity.
-struct BoxIsPtrOpConversion : public FIROpConversion<fir::BoxIsPtrOp> {
+struct BoxIsPtrOpConversion : public fir::FIROpConversion<fir::BoxIsPtrOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -764,7 +414,7 @@ struct BoxIsPtrOpConversion : public FIROpConversion<fir::BoxIsPtrOp> {
 
 /// Lower `fir.box_rank` to the sequence of operation to extract the rank from
 /// the box.
-struct BoxRankOpConversion : public FIROpConversion<fir::BoxRankOp> {
+struct BoxRankOpConversion : public fir::FIROpConversion<fir::BoxRankOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -784,7 +434,8 @@ struct BoxRankOpConversion : public FIROpConversion<fir::BoxRankOp> {
 /// Lower `fir.boxproc_host` operation. Extracts the host pointer from the
 /// boxproc.
 /// TODO: Part of supporting Fortran 2003 procedure pointers.
-struct BoxProcHostOpConversion : public FIROpConversion<fir::BoxProcHostOp> {
+struct BoxProcHostOpConversion
+    : public fir::FIROpConversion<fir::BoxProcHostOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -797,7 +448,8 @@ struct BoxProcHostOpConversion : public FIROpConversion<fir::BoxProcHostOp> {
 
 /// Lower `fir.box_tdesc` to the sequence of operations to extract the type
 /// descriptor from the box.
-struct BoxTypeDescOpConversion : public FIROpConversion<fir::BoxTypeDescOp> {
+struct BoxTypeDescOpConversion
+    : public fir::FIROpConversion<fir::BoxTypeDescOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -814,7 +466,8 @@ struct BoxTypeDescOpConversion : public FIROpConversion<fir::BoxTypeDescOp> {
 
 /// Lower `fir.box_typecode` to a sequence of operations to extract the type
 /// code in the boxed value.
-struct BoxTypeCodeOpConversion : public FIROpConversion<fir::BoxTypeCodeOp> {
+struct BoxTypeCodeOpConversion
+    : public fir::FIROpConversion<fir::BoxTypeCodeOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -832,7 +485,7 @@ struct BoxTypeCodeOpConversion : public FIROpConversion<fir::BoxTypeCodeOp> {
 };
 
 /// Lower `fir.string_lit` to LLVM IR dialect operation.
-struct StringLitOpConversion : public FIROpConversion<fir::StringLitOp> {
+struct StringLitOpConversion : public fir::FIROpConversion<fir::StringLitOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -872,7 +525,7 @@ struct StringLitOpConversion : public FIROpConversion<fir::StringLitOp> {
 };
 
 /// `fir.call` -> `llvm.call`
-struct CallOpConversion : public FIROpConversion<fir::CallOp> {
+struct CallOpConversion : public fir::FIROpConversion<fir::CallOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -903,7 +556,7 @@ namespace {
 /// Per 10.1, the only comparisons available are .EQ. (oeq) and .NE. (une).
 ///
 /// For completeness, all other comparison are done on the real component only.
-struct CmpcOpConversion : public FIROpConversion<fir::CmpcOp> {
+struct CmpcOpConversion : public fir::FIROpConversion<fir::CmpcOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -941,7 +594,7 @@ struct CmpcOpConversion : public FIROpConversion<fir::CmpcOp> {
 };
 
 /// Lower complex constants
-struct ConstcOpConversion : public FIROpConversion<fir::ConstcOp> {
+struct ConstcOpConversion : public fir::FIROpConversion<fir::ConstcOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -968,7 +621,7 @@ struct ConstcOpConversion : public FIROpConversion<fir::ConstcOp> {
 };
 
 /// convert value of from-type to value of to-type
-struct ConvertOpConversion : public FIROpConversion<fir::ConvertOp> {
+struct ConvertOpConversion : public fir::FIROpConversion<fir::ConvertOp> {
   using FIROpConversion::FIROpConversion;
 
   static bool isFloatingPointTy(mlir::Type ty) {
@@ -1137,7 +790,7 @@ struct ConvertOpConversion : public FIROpConversion<fir::ConvertOp> {
 /// only used to carry information during FIR to FIR passes. It may be used
 /// in the future to generate the runtime type info data structures instead
 /// of generating them in lowering.
-struct TypeInfoOpConversion : public FIROpConversion<fir::TypeInfoOp> {
+struct TypeInfoOpConversion : public fir::FIROpConversion<fir::TypeInfoOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -1150,7 +803,7 @@ struct TypeInfoOpConversion : public FIROpConversion<fir::TypeInfoOp> {
 
 /// `fir.dt_entry` operation has no specific CodeGen. The operation is only used
 /// to carry information during FIR to FIR passes.
-struct DTEntryOpConversion : public FIROpConversion<fir::DTEntryOp> {
+struct DTEntryOpConversion : public fir::FIROpConversion<fir::DTEntryOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -1162,7 +815,7 @@ struct DTEntryOpConversion : public FIROpConversion<fir::DTEntryOp> {
 };
 
 /// Lower `fir.global_len` operation.
-struct GlobalLenOpConversion : public FIROpConversion<fir::GlobalLenOp> {
+struct GlobalLenOpConversion : public fir::FIROpConversion<fir::GlobalLenOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -1175,7 +828,7 @@ struct GlobalLenOpConversion : public FIROpConversion<fir::GlobalLenOp> {
 
 /// Lower fir.len_param_index
 struct LenParamIndexOpConversion
-    : public FIROpConversion<fir::LenParamIndexOp> {
+    : public fir::FIROpConversion<fir::LenParamIndexOp> {
   using FIROpConversion::FIROpConversion;
 
   // FIXME: this should be specialized by the runtime target
@@ -1190,7 +843,7 @@ struct LenParamIndexOpConversion
 /// instructions that generate `!llvm.struct<(ptr<ik>, i64)>`. The 1st element
 /// in this struct is a pointer. Its type is determined from `KIND`. The 2nd
 /// element is the length of the character buffer (`#n`).
-struct EmboxCharOpConversion : public FIROpConversion<fir::EmboxCharOp> {
+struct EmboxCharOpConversion : public fir::FIROpConversion<fir::EmboxCharOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -1283,7 +936,7 @@ genTypeStrideInBytes(mlir::Location loc, mlir::Type idxTy,
 
 namespace {
 /// Lower a `fir.allocmem` instruction into `llvm.call @malloc`
-struct AllocMemOpConversion : public FIROpConversion<fir::AllocMemOp> {
+struct AllocMemOpConversion : public fir::FIROpConversion<fir::AllocMemOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -1352,7 +1005,7 @@ static unsigned getDimension(mlir::LLVM::LLVMArrayType ty) {
 
 namespace {
 /// Lower a `fir.freemem` instruction into `llvm.call @free`
-struct FreeMemOpConversion : public FIROpConversion<fir::FreeMemOp> {
+struct FreeMemOpConversion : public fir::FIROpConversion<fir::FreeMemOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -1410,9 +1063,9 @@ convertSubcomponentIndices(mlir::Location loc, mlir::Type eleTy,
 
 /// Common base class for embox to descriptor conversion.
 template <typename OP>
-struct EmboxCommonConversion : public FIROpConversion<OP> {
-  using FIROpConversion<OP>::FIROpConversion;
-  using TypePair = typename FIROpConversion<OP>::TypePair;
+struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
+  using fir::FIROpConversion<OP>::FIROpConversion;
+  using TypePair = typename fir::FIROpConversion<OP>::TypePair;
 
   static int getCFIAttr(fir::BaseBoxType boxTy) {
     auto eleTy = boxTy.getEleTy();
@@ -1434,8 +1087,8 @@ struct EmboxCommonConversion : public FIROpConversion<OP> {
       return size; // Length accounted for in the genTypeStrideInBytes GEP.
     // Otherwise,  multiply the single character size by the length.
     assert(!lenParams.empty());
-    auto len64 = FIROpConversion<OP>::integerCast(loc, rewriter, i64Ty,
-                                                  lenParams.back());
+    auto len64 = fir::FIROpConversion<OP>::integerCast(loc, rewriter, i64Ty,
+                                                       lenParams.back());
     return rewriter.create<mlir::LLVM::MulOp>(loc, i64Ty, size, len64);
   }
 
@@ -2280,7 +1933,7 @@ private:
 
 /// Lower `fir.emboxproc` operation. Creates a procedure box.
 /// TODO: Part of supporting Fortran 2003 procedure pointers.
-struct EmboxProcOpConversion : public FIROpConversion<fir::EmboxProcOp> {
+struct EmboxProcOpConversion : public fir::FIROpConversion<fir::EmboxProcOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -2346,7 +1999,7 @@ private:
 namespace {
 /// Extract a subobject value from an ssa-value of aggregate type
 struct ExtractValueOpConversion
-    : public FIROpAndTypeConversion<fir::ExtractValueOp>,
+    : public fir::FIROpAndTypeConversion<fir::ExtractValueOp>,
       public ValueOpCommon {
   using FIROpAndTypeConversion::FIROpAndTypeConversion;
 
@@ -2365,7 +2018,7 @@ struct ExtractValueOpConversion
 /// InsertValue is the generalized instruction for the composition of new
 /// aggregate type values.
 struct InsertValueOpConversion
-    : public FIROpAndTypeConversion<fir::InsertValueOp>,
+    : public fir::FIROpAndTypeConversion<fir::InsertValueOp>,
       public ValueOpCommon {
   using FIROpAndTypeConversion::FIROpAndTypeConversion;
 
@@ -2383,7 +2036,7 @@ struct InsertValueOpConversion
 
 /// InsertOnRange inserts a value into a sequence over a range of offsets.
 struct InsertOnRangeOpConversion
-    : public FIROpAndTypeConversion<fir::InsertOnRangeOp> {
+    : public fir::FIROpAndTypeConversion<fir::InsertOnRangeOp> {
   using FIROpAndTypeConversion::FIROpAndTypeConversion;
 
   // Increments an array of subscripts in a row major fasion.
@@ -2447,7 +2100,7 @@ namespace {
 /// (See the static restriction on coordinate_of.) array_coor determines the
 /// coordinate (location) of a specific element.
 struct XArrayCoorOpConversion
-    : public FIROpAndTypeConversion<fir::cg::XArrayCoorOp> {
+    : public fir::FIROpAndTypeConversion<fir::cg::XArrayCoorOp> {
   using FIROpAndTypeConversion::FIROpAndTypeConversion;
 
   mlir::LogicalResult
@@ -2615,7 +2268,7 @@ struct XArrayCoorOpConversion
 /// With unboxed arrays, there is the restriction that the array have a static
 /// shape in all but the last column.
 struct CoordinateOpConversion
-    : public FIROpAndTypeConversion<fir::CoordinateOp> {
+    : public fir::FIROpAndTypeConversion<fir::CoordinateOp> {
   using FIROpAndTypeConversion::FIROpAndTypeConversion;
 
   mlir::LogicalResult
@@ -2910,7 +2563,7 @@ private:
 
 /// Convert `fir.field_index`. The conversion depends on whether the size of
 /// the record is static or dynamic.
-struct FieldIndexOpConversion : public FIROpConversion<fir::FieldIndexOp> {
+struct FieldIndexOpConversion : public fir::FIROpConversion<fir::FieldIndexOp> {
   using FIROpConversion::FIROpConversion;
 
   // NB: most field references should be resolved by this point
@@ -2951,7 +2604,7 @@ struct FieldIndexOpConversion : public FIROpConversion<fir::FieldIndexOp> {
 };
 
 /// Convert `fir.end`
-struct FirEndOpConversion : public FIROpConversion<fir::FirEndOp> {
+struct FirEndOpConversion : public fir::FIROpConversion<fir::FirEndOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -2963,7 +2616,7 @@ struct FirEndOpConversion : public FIROpConversion<fir::FirEndOp> {
 };
 
 /// Lower `fir.type_desc` to a global addr.
-struct TypeDescOpConversion : public FIROpConversion<fir::TypeDescOp> {
+struct TypeDescOpConversion : public fir::FIROpConversion<fir::TypeDescOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -2990,7 +2643,7 @@ struct TypeDescOpConversion : public FIROpConversion<fir::TypeDescOp> {
 };
 
 /// Lower `fir.has_value` operation to `llvm.return` operation.
-struct HasValueOpConversion : public FIROpConversion<fir::HasValueOp> {
+struct HasValueOpConversion : public fir::FIROpConversion<fir::HasValueOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3043,7 +2696,7 @@ static inline bool attributeTypeIsCompatible(mlir::MLIRContext *ctx,
 /// Lower `fir.global` operation to `llvm.global` operation.
 /// `fir.insert_on_range` operations are replaced with constant dense attribute
 /// if they are applied on the full range.
-struct GlobalOpConversion : public FIROpConversion<fir::GlobalOp> {
+struct GlobalOpConversion : public fir::FIROpConversion<fir::GlobalOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3167,7 +2820,7 @@ private:
 };
 
 /// `fir.load` --> `llvm.load`
-struct LoadOpConversion : public FIROpConversion<fir::LoadOp> {
+struct LoadOpConversion : public fir::FIROpConversion<fir::LoadOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3216,7 +2869,7 @@ struct LoadOpConversion : public FIROpConversion<fir::LoadOp> {
 /// Lower `fir.no_reassoc` to LLVM IR dialect.
 /// TODO: how do we want to enforce this in LLVM-IR? Can we manipulate the fast
 /// math flags?
-struct NoReassocOpConversion : public FIROpConversion<fir::NoReassocOp> {
+struct NoReassocOpConversion : public fir::FIROpConversion<fir::NoReassocOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3277,7 +2930,7 @@ static void genCaseLadderStep(mlir::Location loc, mlir::Value cmp,
 /// upper bound in the same case condition.
 ///
 /// TODO: lowering of CHARACTER type cases is not handled yet.
-struct SelectCaseOpConversion : public FIROpConversion<fir::SelectCaseOp> {
+struct SelectCaseOpConversion : public fir::FIROpConversion<fir::SelectCaseOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3391,7 +3044,7 @@ static void selectMatchAndRewrite(const fir::LLVMTypeConverter &lowering,
 }
 
 /// conversion of fir::SelectOp to an if-then-else ladder
-struct SelectOpConversion : public FIROpConversion<fir::SelectOp> {
+struct SelectOpConversion : public fir::FIROpConversion<fir::SelectOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3403,7 +3056,7 @@ struct SelectOpConversion : public FIROpConversion<fir::SelectOp> {
 };
 
 /// conversion of fir::SelectRankOp to an if-then-else ladder
-struct SelectRankOpConversion : public FIROpConversion<fir::SelectRankOp> {
+struct SelectRankOpConversion : public fir::FIROpConversion<fir::SelectRankOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3415,7 +3068,7 @@ struct SelectRankOpConversion : public FIROpConversion<fir::SelectRankOp> {
 };
 
 /// Lower `fir.select_type` to LLVM IR dialect.
-struct SelectTypeOpConversion : public FIROpConversion<fir::SelectTypeOp> {
+struct SelectTypeOpConversion : public fir::FIROpConversion<fir::SelectTypeOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3428,7 +3081,7 @@ struct SelectTypeOpConversion : public FIROpConversion<fir::SelectTypeOp> {
 };
 
 /// `fir.store` --> `llvm.store`
-struct StoreOpConversion : public FIROpConversion<fir::StoreOp> {
+struct StoreOpConversion : public fir::FIROpConversion<fir::StoreOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3462,7 +3115,7 @@ namespace {
 
 /// Convert `fir.unboxchar` into two `llvm.extractvalue` instructions. One for
 /// the character buffer and one for the buffer length.
-struct UnboxCharOpConversion : public FIROpConversion<fir::UnboxCharOp> {
+struct UnboxCharOpConversion : public fir::FIROpConversion<fir::UnboxCharOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3487,7 +3140,7 @@ struct UnboxCharOpConversion : public FIROpConversion<fir::UnboxCharOp> {
 /// Lower `fir.unboxproc` operation. Unbox a procedure box value, yielding its
 /// components.
 /// TODO: Part of supporting Fortran 2003 procedure pointers.
-struct UnboxProcOpConversion : public FIROpConversion<fir::UnboxProcOp> {
+struct UnboxProcOpConversion : public fir::FIROpConversion<fir::UnboxProcOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3499,7 +3152,7 @@ struct UnboxProcOpConversion : public FIROpConversion<fir::UnboxProcOp> {
 };
 
 /// convert to LLVM IR dialect `undef`
-struct UndefOpConversion : public FIROpConversion<fir::UndefOp> {
+struct UndefOpConversion : public fir::FIROpConversion<fir::UndefOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3511,7 +3164,7 @@ struct UndefOpConversion : public FIROpConversion<fir::UndefOp> {
   }
 };
 
-struct ZeroOpConversion : public FIROpConversion<fir::ZeroOp> {
+struct ZeroOpConversion : public fir::FIROpConversion<fir::ZeroOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3524,7 +3177,8 @@ struct ZeroOpConversion : public FIROpConversion<fir::ZeroOp> {
 };
 
 /// `fir.unreachable` --> `llvm.unreachable`
-struct UnreachableOpConversion : public FIROpConversion<fir::UnreachableOp> {
+struct UnreachableOpConversion
+    : public fir::FIROpConversion<fir::UnreachableOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3541,7 +3195,7 @@ struct UnreachableOpConversion : public FIROpConversion<fir::UnreachableOp> {
 ///  %1 = llvm.ptrtoint %0
 ///  %2 = llvm.icmp "ne" %1, %0 : i64
 /// ```
-struct IsPresentOpConversion : public FIROpConversion<fir::IsPresentOp> {
+struct IsPresentOpConversion : public fir::FIROpConversion<fir::IsPresentOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3570,7 +3224,7 @@ struct IsPresentOpConversion : public FIROpConversion<fir::IsPresentOp> {
 
 /// Create value signaling an absent optional argument in a call, e.g.
 /// `fir.absent !fir.ref<i64>` -->  `llvm.mlir.zero : !llvm.ptr<i64>`
-struct AbsentOpConversion : public FIROpConversion<fir::AbsentOp> {
+struct AbsentOpConversion : public fir::FIROpConversion<fir::AbsentOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3630,7 +3284,7 @@ complexSum(OPTY sumop, mlir::ValueRange opnds,
 } // namespace
 
 namespace {
-struct AddcOpConversion : public FIROpConversion<fir::AddcOp> {
+struct AddcOpConversion : public fir::FIROpConversion<fir::AddcOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3645,7 +3299,7 @@ struct AddcOpConversion : public FIROpConversion<fir::AddcOp> {
   }
 };
 
-struct SubcOpConversion : public FIROpConversion<fir::SubcOp> {
+struct SubcOpConversion : public fir::FIROpConversion<fir::SubcOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3661,7 +3315,7 @@ struct SubcOpConversion : public FIROpConversion<fir::SubcOp> {
 };
 
 /// Inlined complex multiply
-struct MulcOpConversion : public FIROpConversion<fir::MulcOp> {
+struct MulcOpConversion : public fir::FIROpConversion<fir::MulcOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3695,7 +3349,7 @@ struct MulcOpConversion : public FIROpConversion<fir::MulcOp> {
 };
 
 /// Inlined complex division
-struct DivcOpConversion : public FIROpConversion<fir::DivcOp> {
+struct DivcOpConversion : public fir::FIROpConversion<fir::DivcOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3735,7 +3389,7 @@ struct DivcOpConversion : public FIROpConversion<fir::DivcOp> {
 };
 
 /// Inlined complex negation
-struct NegcOpConversion : public FIROpConversion<fir::NegcOp> {
+struct NegcOpConversion : public fir::FIROpConversion<fir::NegcOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3756,7 +3410,7 @@ struct NegcOpConversion : public FIROpConversion<fir::NegcOp> {
   }
 };
 
-struct BoxOffsetOpConversion : public FIROpConversion<fir::BoxOffsetOp> {
+struct BoxOffsetOpConversion : public fir::FIROpConversion<fir::BoxOffsetOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
@@ -3782,10 +3436,10 @@ struct BoxOffsetOpConversion : public FIROpConversion<fir::BoxOffsetOp> {
 /// anymore uses.
 /// These operations are normally dead after the pre-codegen pass.
 template <typename FromOp>
-struct MustBeDeadConversion : public FIROpConversion<FromOp> {
+struct MustBeDeadConversion : public fir::FIROpConversion<FromOp> {
   explicit MustBeDeadConversion(const fir::LLVMTypeConverter &lowering,
                                 const fir::FIRToLLVMPassOptions &options)
-      : FIROpConversion<FromOp>(lowering, options) {}
+      : fir::FIROpConversion<FromOp>(lowering, options) {}
   using OpAdaptor = typename FromOp::Adaptor;
 
   mlir::LogicalResult
@@ -3799,7 +3453,7 @@ struct MustBeDeadConversion : public FIROpConversion<FromOp> {
 };
 
 struct UnrealizedConversionCastOpConversion
-    : public FIROpConversion<mlir::UnrealizedConversionCastOp> {
+    : public fir::FIROpConversion<mlir::UnrealizedConversionCastOp> {
   using FIROpConversion::FIROpConversion;
 
   mlir::LogicalResult
diff --git a/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
new file mode 100644
index 000000000000..26871d888815
--- /dev/null
+++ b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
@@ -0,0 +1,315 @@
+//===-- CodeGen.cpp -- bridge to lower to LLVM ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/CodeGen/FIROpPatterns.h"
+#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "llvm/Support/Debug.h"
+
+static inline mlir::Type getLlvmPtrType(mlir::MLIRContext *context,
+                                        unsigned addressSpace = 0) {
+  return mlir::LLVM::LLVMPointerType::get(context, addressSpace);
+}
+
+static unsigned getTypeDescFieldId(mlir::Type ty) {
+  auto isArray = fir::dyn_cast_ptrOrBoxEleTy(ty).isa<fir::SequenceType>();
+  return isArray ? kOptTypePtrPosInBox : kDimsPosInBox;
+}
+
+namespace fir {
+
+ConvertFIRToLLVMPattern::ConvertFIRToLLVMPattern(
+    llvm::StringRef rootOpName, mlir::MLIRContext *context,
+    const fir::LLVMTypeConverter &typeConverter,
+    const fir::FIRToLLVMPassOptions &options, mlir::PatternBenefit benefit)
+    : ConvertToLLVMPattern(rootOpName, context, typeConverter, benefit),
+      options(options) {}
+
+// Convert FIR type to LLVM without turning fir.box<T> into memory
+// reference.
+mlir::Type
+ConvertFIRToLLVMPattern::convertObjectType(mlir::Type firType) const {
+  if (auto boxTy = firType.dyn_cast<fir::BaseBoxType>())
+    return lowerTy().convertBoxTypeAsStruct(boxTy);
+  return lowerTy().convertType(firType);
+}
+
+mlir::LLVM::ConstantOp ConvertFIRToLLVMPattern::genI32Constant(
+    mlir::Location loc, mlir::ConversionPatternRewriter &rewriter,
+    int value) const {
+  mlir::Type i32Ty = rewriter.getI32Type();
+  mlir::IntegerAttr attr = rewriter.getI32IntegerAttr(value);
+  return rewriter.create<mlir::LLVM::ConstantOp>(loc, i32Ty, attr);
+}
+
+mlir::LLVM::ConstantOp ConvertFIRToLLVMPattern::genConstantOffset(
+    mlir::Location loc, mlir::ConversionPatternRewriter &rewriter,
+    int offset) const {
+  mlir::Type ity = lowerTy().offsetType();
+  mlir::IntegerAttr cattr = rewriter.getI32IntegerAttr(offset);
+  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
+}
+
+/// Perform an extension or truncation as needed on an integer value. Lowering
+/// to the specific target may involve some sign-extending or truncation of
+/// values, particularly to fit them from abstract box types to the
+/// appropriate reified structures.
+mlir::Value
+ConvertFIRToLLVMPattern::integerCast(mlir::Location loc,
+                                     mlir::ConversionPatternRewriter &rewriter,
+                                     mlir::Type ty, mlir::Value val) const {
+  auto valTy = val.getType();
+  // If the value was not yet lowered, lower its type so that it can
+  // be used in getPrimitiveTypeSizeInBits.
+  if (!valTy.isa<mlir::IntegerType>())
+    valTy = convertType(valTy);
+  auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
+  auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy);
+  if (toSize < fromSize)
+    return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
+  if (toSize > fromSize)
+    return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
+  return val;
+}
+
+fir::ConvertFIRToLLVMPattern::TypePair
+ConvertFIRToLLVMPattern::getBoxTypePair(mlir::Type firBoxTy) const {
+  mlir::Type llvmBoxTy =
+      lowerTy().convertBoxTypeAsStruct(mlir::cast<fir::BaseBoxType>(firBoxTy));
+  return TypePair{firBoxTy, llvmBoxTy};
+}
+
+/// Construct code sequence to extract the specific value from a `fir.box`.
+mlir::Value ConvertFIRToLLVMPattern::getValueFromBox(
+    mlir::Location loc, TypePair boxTy, mlir::Value box, mlir::Type resultTy,
+    mlir::ConversionPatternRewriter &rewriter, int boxValue) const {
+  if (box.getType().isa<mlir::LLVM::LLVMPointerType>()) {
+    auto pty = getLlvmPtrType(resultTy.getContext());
+    auto p = rewriter.create<mlir::LLVM::GEPOp>(
+        loc, pty, boxTy.llvm, box,
+        llvm::ArrayRef<mlir::LLVM::GEPArg>{0, boxValue});
+    auto loadOp = rewriter.create<mlir::LLVM::LoadOp>(loc, resultTy, p);
+    attachTBAATag(loadOp, boxTy.fir, nullptr, p);
+    return loadOp;
+  }
+  return rewriter.create<mlir::LLVM::ExtractValueOp>(loc, box, boxValue);
+}
+
+/// Method to construct code sequence to get the triple for dimension `dim`
+/// from a box.
+llvm::SmallVector<mlir::Value, 3> ConvertFIRToLLVMPattern::getDimsFromBox(
+    mlir::Location loc, llvm::ArrayRef<mlir::Type> retTys, TypePair boxTy,
+    mlir::Value box, mlir::Value dim,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  mlir::Value l0 =
+      loadDimFieldFromBox(loc, boxTy, box, dim, 0, retTys[0], rewriter);
+  mlir::Value l1 =
+      loadDimFieldFromBox(loc, boxTy, box, dim, 1, retTys[1], rewriter);
+  mlir::Value l2 =
+      loadDimFieldFromBox(loc, boxTy, box, dim, 2, retTys[2], rewriter);
+  return {l0, l1, l2};
+}
+
+llvm::SmallVector<mlir::Value, 3> ConvertFIRToLLVMPattern::getDimsFromBox(
+    mlir::Location loc, llvm::ArrayRef<mlir::Type> retTys, TypePair boxTy,
+    mlir::Value box, int dim, mlir::ConversionPatternRewriter &rewriter) const {
+  mlir::Value l0 =
+      getDimFieldFromBox(loc, boxTy, box, dim, 0, retTys[0], rewriter);
+  mlir::Value l1 =
+      getDimFieldFromBox(loc, boxTy, box, dim, 1, retTys[1], rewriter);
+  mlir::Value l2 =
+      getDimFieldFromBox(loc, boxTy, box, dim, 2, retTys[2], rewriter);
+  return {l0, l1, l2};
+}
+
+mlir::Value ConvertFIRToLLVMPattern::loadDimFieldFromBox(
+    mlir::Location loc, TypePair boxTy, mlir::Value box, mlir::Value dim,
+    int off, mlir::Type ty, mlir::ConversionPatternRewriter &rewriter) const {
+  assert(box.getType().isa<mlir::LLVM::LLVMPointerType>() &&
+         "descriptor inquiry with runtime dim can only be done on descriptor "
+         "in memory");
+  mlir::LLVM::GEPOp p = genGEP(loc, boxTy.llvm, rewriter, box, 0,
+                               static_cast<int>(kDimsPosInBox), dim, off);
+  auto loadOp = rewriter.create<mlir::LLVM::LoadOp>(loc, ty, p);
+  attachTBAATag(loadOp, boxTy.fir, nullptr, p);
+  return loadOp;
+}
+
+mlir::Value ConvertFIRToLLVMPattern::getDimFieldFromBox(
+    mlir::Location loc, TypePair boxTy, mlir::Value box, int dim, int off,
+    mlir::Type ty, mlir::ConversionPatternRewriter &rewriter) const {
+  if (box.getType().isa<mlir::LLVM::LLVMPointerType>()) {
+    mlir::LLVM::GEPOp p = genGEP(loc, boxTy.llvm, rewriter, box, 0,
+                                 static_cast<int>(kDimsPosInBox), dim, off);
+    auto loadOp = rewriter.create<mlir::LLVM::LoadOp>(loc, ty, p);
+    attachTBAATag(loadOp, boxTy.fir, nullptr, p);
+    return loadOp;
+  }
+  return rewriter.create<mlir::LLVM::ExtractValueOp>(
+      loc, box, llvm::ArrayRef<std::int64_t>{kDimsPosInBox, dim, off});
+}
+
+mlir::Value ConvertFIRToLLVMPattern::getStrideFromBox(
+    mlir::Location loc, TypePair boxTy, mlir::Value box, unsigned dim,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  auto idxTy = lowerTy().indexType();
+  return getDimFieldFromBox(loc, boxTy, box, dim, kDimStridePos, idxTy,
+                            rewriter);
+}
+
+/// Read base address from a fir.box. Returned address has type ty.
+mlir::Value ConvertFIRToLLVMPattern::getBaseAddrFromBox(
+    mlir::Location loc, TypePair boxTy, mlir::Value box,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  mlir::Type resultTy = ::getLlvmPtrType(boxTy.llvm.getContext());
+  return getValueFromBox(loc, boxTy, box, resultTy, rewriter, kAddrPosInBox);
+}
+
+mlir::Value ConvertFIRToLLVMPattern::getElementSizeFromBox(
+    mlir::Location loc, mlir::Type resultTy, TypePair boxTy, mlir::Value box,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  return getValueFromBox(loc, boxTy, box, resultTy, rewriter, kElemLenPosInBox);
+}
+
+// Get the element type given an LLVM type that is of the form
+// (array|struct|vector)+ and the provided indexes.
+mlir::Type ConvertFIRToLLVMPattern::getBoxEleTy(
+    mlir::Type type, llvm::ArrayRef<std::int64_t> indexes) const {
+  for (unsigned i : indexes) {
+    if (auto t = type.dyn_cast<mlir::LLVM::LLVMStructType>()) {
+      assert(!t.isOpaque() && i < t.getBody().size());
+      type = t.getBody()[i];
+    } else if (auto t = type.dyn_cast<mlir::LLVM::LLVMArrayType>()) {
+      type = t.getElementType();
+    } else if (auto t = type.dyn_cast<mlir::VectorType>()) {
+      type = t.getElementType();
+    } else {
+      fir::emitFatalError(mlir::UnknownLoc::get(type.getContext()),
+                          "request for invalid box element type");
+    }
+  }
+  return type;
+}
+
+// Return LLVM type of the object described by a fir.box of \p boxType.
+mlir::Type ConvertFIRToLLVMPattern::getLlvmObjectTypeFromBoxType(
+    mlir::Type boxType) const {
+  mlir::Type objectType = fir::dyn_cast_ptrOrBoxEleTy(boxType);
+  assert(objectType && "boxType must be a box type");
+  return this->convertType(objectType);
+}
+
+/// Read the address of the type descriptor from a box.
+mlir::Value ConvertFIRToLLVMPattern::loadTypeDescAddress(
+    mlir::Location loc, TypePair boxTy, mlir::Value box,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  unsigned typeDescFieldId = getTypeDescFieldId(boxTy.fir);
+  mlir::Type tdescType = lowerTy().convertTypeDescType(rewriter.getContext());
+  return getValueFromBox(loc, boxTy, box, tdescType, rewriter, typeDescFieldId);
+}
+
+// Load the attribute from the \p box and perform a check against \p maskValue
+// The final comparison is implemented as `(attribute & maskValue) != 0`.
+mlir::Value ConvertFIRToLLVMPattern::genBoxAttributeCheck(
+    mlir::Location loc, TypePair boxTy, mlir::Value box,
+    mlir::ConversionPatternRewriter &rewriter, unsigned maskValue) const {
+  mlir::Type attrTy = rewriter.getI32Type();
+  mlir::Value attribute =
+      getValueFromBox(loc, boxTy, box, attrTy, rewriter, kAttributePosInBox);
+  mlir::LLVM::ConstantOp attrMask = genConstantOffset(loc, rewriter, maskValue);
+  auto maskRes =
+      rewriter.create<mlir::LLVM::AndOp>(loc, attrTy, attribute, attrMask);
+  mlir::LLVM::ConstantOp c0 = genConstantOffset(loc, rewriter, 0);
+  return rewriter.create<mlir::LLVM::ICmpOp>(loc, mlir::LLVM::ICmpPredicate::ne,
+                                             maskRes, c0);
+}
+
+// Find the Block in which the alloca should be inserted.
+// The order to recursively find the proper block:
+// 1. An OpenMP Op that will be outlined.
+// 2. A LLVMFuncOp
+// 3. The first ancestor that is an OpenMP Op or a LLVMFuncOp
+mlir::Block *
+ConvertFIRToLLVMPattern::getBlockForAllocaInsert(mlir::Operation *op) const {
+  if (auto iface = mlir::dyn_cast<mlir::omp::OutlineableOpenMPOpInterface>(op))
+    return iface.getAllocaBlock();
+  if (auto llvmFuncOp = mlir::dyn_cast<mlir::LLVM::LLVMFuncOp>(op))
+    return &llvmFuncOp.front();
+  return getBlockForAllocaInsert(op->getParentOp());
+}
+
+// Generate an alloca of size 1 for an object of type \p llvmObjectTy in the
+// allocation address space provided for the architecture in the DataLayout
+// specification. If the address space is different from the devices
+// program address space we perform a cast. In the case of most architectures
+// the program and allocation address space will be the default of 0 and no
+// cast will be emitted.
+mlir::Value ConvertFIRToLLVMPattern::genAllocaAndAddrCastWithType(
+    mlir::Location loc, mlir::Type llvmObjectTy, unsigned alignment,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  auto thisPt = rewriter.saveInsertionPoint();
+  mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp();
+  if (mlir::isa<mlir::omp::DeclareReductionOp>(parentOp)) {
+    // DeclareReductionOp has multiple child regions. We want to get the first
+    // block of whichever of those regions we are currently in
+    mlir::Region *parentRegion = rewriter.getInsertionBlock()->getParent();
+    rewriter.setInsertionPointToStart(&parentRegion->front());
+  } else {
+    mlir::Block *insertBlock = getBlockForAllocaInsert(parentOp);
+    rewriter.setInsertionPointToStart(insertBlock);
+  }
+  auto size = genI32Constant(loc, rewriter, 1);
+  unsigned allocaAs = getAllocaAddressSpace(rewriter);
+  unsigned programAs = getProgramAddressSpace(rewriter);
+
+  mlir::Value al = rewriter.create<mlir::LLVM::AllocaOp>(
+      loc, ::getLlvmPtrType(llvmObjectTy.getContext(), allocaAs), llvmObjectTy,
+      size, alignment);
+
+  // if our allocation address space, is not the same as the program address
+  // space, then we must emit a cast to the program address space before use.
+  // An example case would be on AMDGPU, where the allocation address space is
+  // the numeric value 5 (private), and the program address space is 0
+  // (generic).
+  if (allocaAs != programAs) {
+    al = rewriter.create<mlir::LLVM::AddrSpaceCastOp>(
+        loc, ::getLlvmPtrType(llvmObjectTy.getContext(), programAs), al);
+  }
+
+  rewriter.restoreInsertionPoint(thisPt);
+  return al;
+}
+
+unsigned ConvertFIRToLLVMPattern::getAllocaAddressSpace(
+    mlir::ConversionPatternRewriter &rewriter) const {
+  mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp();
+  assert(parentOp != nullptr &&
+         "expected insertion block to have parent operation");
+  if (auto module = parentOp->getParentOfType<mlir::ModuleOp>())
+    if (mlir::Attribute addrSpace =
+            mlir::DataLayout(module).getAllocaMemorySpace())
+      return llvm::cast<mlir::IntegerAttr>(addrSpace).getUInt();
+  return defaultAddressSpace;
+}
+
+unsigned ConvertFIRToLLVMPattern::getProgramAddressSpace(
+    mlir::ConversionPatternRewriter &rewriter) const {
+  mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp();
+  assert(parentOp != nullptr &&
+         "expected insertion block to have parent operation");
+  if (auto module = parentOp->getParentOfType<mlir::ModuleOp>())
+    if (mlir::Attribute addrSpace =
+            mlir::DataLayout(module).getProgramMemorySpace())
+      return llvm::cast<mlir::IntegerAttr>(addrSpace).getUInt();
+  return defaultAddressSpace;
+}
+
+} // namespace fir
-- 
GitLab


From 19268ac55106834701b1c41acded4d413a3502e9 Mon Sep 17 00:00:00 2001
From: Mariusz Borsa <wrotki@msn.com>
Date: Fri, 22 Mar 2024 13:36:17 -0700
Subject: [PATCH 005/404] [Sanitizers][Darwin] Bump up
 DEFAULT_SANITIZER_MIN_OSX_VERSION (#86035)

The greendragon was recently moved and now it runs on somewhat newer
macOS
version - which breaks some sanitizers tests

rdar://125052915

Co-authored-by: Mariusz Borsa <m_borsa@apple.com>
---
 compiler-rt/cmake/config-ix.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake
index 4f47142850a5..46a6fdf8728f 100644
--- a/compiler-rt/cmake/config-ix.cmake
+++ b/compiler-rt/cmake/config-ix.cmake
@@ -461,7 +461,7 @@ if(APPLE)
     set(ORC_SUPPORTED_OS osx)
   endif()
 
-  set(DEFAULT_SANITIZER_MIN_OSX_VERSION 10.10)
+  set(DEFAULT_SANITIZER_MIN_OSX_VERSION 10.13)
   set(DARWIN_osx_MIN_VER_FLAG "-mmacosx-version-min")
   if(NOT SANITIZER_MIN_OSX_VERSION)
     string(REGEX MATCH "${DARWIN_osx_MIN_VER_FLAG}=([.0-9]+)"
-- 
GitLab


From 105feb9ac61243a32c84f5c13015602e5de500ff Mon Sep 17 00:00:00 2001
From: Alexander Yermolovich <43973793+ayermolo@users.noreply.github.com>
Date: Fri, 22 Mar 2024 13:41:27 -0700
Subject: [PATCH 006/404] [BOLT][DWARF] Fix handling of DW_TAG_label (#86182)

For DWARF5 BOLT was not retreiving address and instead was setting an
index.
Changed so that an address is used, and added DWARF4 test because it was
missing.
---
 bolt/lib/Rewrite/DWARFRewriter.cpp  |  43 +++--
 bolt/test/X86/dwarf4-label-low-pc.s | 263 ++++++++++++++++++++++++++++
 bolt/test/X86/dwarf5-label-low-pc.s |  36 ++--
 3 files changed, 316 insertions(+), 26 deletions(-)
 create mode 100644 bolt/test/X86/dwarf4-label-low-pc.s

diff --git a/bolt/lib/Rewrite/DWARFRewriter.cpp b/bolt/lib/Rewrite/DWARFRewriter.cpp
index 8dc7b90a6e30..601a2105fc26 100644
--- a/bolt/lib/Rewrite/DWARFRewriter.cpp
+++ b/bolt/lib/Rewrite/DWARFRewriter.cpp
@@ -375,12 +375,11 @@ static cl::opt<bool> AlwaysConvertToRanges(
 extern cl::opt<std::string> CompDirOverride;
 } // namespace opts
 
-static bool getLowAndHighPC(const DIE &Die, const DWARFUnit &DU,
-                            uint64_t &LowPC, uint64_t &HighPC,
-                            uint64_t &SectionIndex) {
+/// If DW_AT_low_pc exists sets LowPC and returns true.
+static bool getLowPC(const DIE &Die, const DWARFUnit &DU, uint64_t &LowPC,
+                     uint64_t &SectionIndex) {
   DIEValue DvalLowPc = Die.findAttribute(dwarf::DW_AT_low_pc);
-  DIEValue DvalHighPc = Die.findAttribute(dwarf::DW_AT_high_pc);
-  if (!DvalLowPc || !DvalHighPc)
+  if (!DvalLowPc)
     return false;
 
   dwarf::Form Form = DvalLowPc.getForm();
@@ -403,14 +402,39 @@ static bool getLowAndHighPC(const DIE &Die, const DWARFUnit &DU,
     LowPC = LowPcValue;
     SectionIndex = 0;
   }
+  return true;
+}
+
+/// If DW_AT_high_pc exists sets HighPC and returns true.
+static bool getHighPC(const DIE &Die, const uint64_t LowPC, uint64_t &HighPC) {
+  DIEValue DvalHighPc = Die.findAttribute(dwarf::DW_AT_high_pc);
+  if (!DvalHighPc)
+    return false;
   if (DvalHighPc.getForm() == dwarf::DW_FORM_addr)
     HighPC = DvalHighPc.getDIEInteger().getValue();
   else
     HighPC = LowPC + DvalHighPc.getDIEInteger().getValue();
-
   return true;
 }
 
+/// If DW_AT_low_pc and DW_AT_high_pc exist sets LowPC and HighPC and returns
+/// true.
+static bool getLowAndHighPC(const DIE &Die, const DWARFUnit &DU,
+                            uint64_t &LowPC, uint64_t &HighPC,
+                            uint64_t &SectionIndex) {
+  uint64_t TempLowPC = LowPC;
+  uint64_t TempHighPC = HighPC;
+  uint64_t TempSectionIndex = SectionIndex;
+  if (getLowPC(Die, DU, TempLowPC, TempSectionIndex) &&
+      getHighPC(Die, TempLowPC, TempHighPC)) {
+    LowPC = TempLowPC;
+    HighPC = TempHighPC;
+    SectionIndex = TempSectionIndex;
+    return true;
+  }
+  return false;
+}
+
 static Expected<llvm::DWARFAddressRangesVector>
 getDIEAddressRanges(const DIE &Die, DWARFUnit &DU) {
   uint64_t LowPC, HighPC, Index;
@@ -1248,10 +1272,9 @@ void DWARFRewriter::updateUnitDebugInfo(
           }
         }
       } else if (LowPCAttrInfo) {
-        const std::optional<uint64_t> Result =
-            LowPCAttrInfo.getDIEInteger().getValue();
-        if (Result.has_value()) {
-          const uint64_t Address = Result.value();
+        uint64_t Address = 0;
+        uint64_t SectionIndex = 0;
+        if (getLowPC(*Die, Unit, Address, SectionIndex)) {
           uint64_t NewAddress = 0;
           if (const BinaryFunction *Function =
                   BC.getBinaryFunctionContainingAddress(Address)) {
diff --git a/bolt/test/X86/dwarf4-label-low-pc.s b/bolt/test/X86/dwarf4-label-low-pc.s
new file mode 100644
index 000000000000..dfd5af18c09b
--- /dev/null
+++ b/bolt/test/X86/dwarf4-label-low-pc.s
@@ -0,0 +1,263 @@
+
+# REQUIRES: system-linux
+
+# RUN: llvm-mc -dwarf-version=4 -filetype=obj -triple x86_64-unknown-linux %s -o %tmain.o
+# RUN: %clang %cflags -dwarf-4 %tmain.o -o %t.exe -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.bolt --update-debug-sections
+# RUN: llvm-dwarfdump --show-form --verbose --debug-info %t.exe | FileCheck --check-prefix=PRECHECK %s
+# RUN: llvm-dwarfdump --show-form --verbose --debug-info %t.bolt > %t.txt
+# RUN: llvm-objdump -d %t.bolt >> %t.txt
+# RUN: cat %t.txt | FileCheck --check-prefix=POSTCHECK %s
+
+## This test checks that we correctly handle DW_AT_low_pc [DW_FORM_addr] that is part of DW_TAG_label.
+
+# PRECHECK: version = 0x0004
+# PRECHECK: DW_TAG_label
+# PRECHECK-NEXT: DW_AT_name
+# PRECHECK-NEXT: DW_AT_decl_file
+# PRECHECK-NEXT: DW_AT_decl_line
+# PRECHECK-NEXT:DW_AT_low_pc [DW_FORM_addr]
+# PRECHECK: DW_TAG_label
+# PRECHECK-NEXT: DW_AT_name
+# PRECHECK-NEXT: DW_AT_decl_file
+# PRECHECK-NEXT: DW_AT_decl_line
+# PRECHECK-NEXT:DW_AT_low_pc [DW_FORM_addr]
+
+# POSTCHECK: version = 0x0004
+# POSTCHECK: DW_TAG_label
+# POSTCHECK-NEXT: DW_AT_name
+# POSTCHECK-NEXT: DW_AT_decl_file
+# POSTCHECK-NEXT: DW_AT_decl_line
+# POSTCHECK-NEXT:DW_AT_low_pc [DW_FORM_addr] (0x[[ADDR:[1-9a-f]*]]
+# POSTCHECK: DW_TAG_label
+# POSTCHECK-NEXT: DW_AT_name
+# POSTCHECK-NEXT: DW_AT_decl_file
+# POSTCHECK-NEXT: DW_AT_decl_line
+# POSTCHECK-NEXT:DW_AT_low_pc [DW_FORM_addr] (0x[[ADDR2:[1-9a-f]*]]
+
+# POSTCHECK: [[ADDR]]: 8b 45 f8
+# POSTCHECK: [[ADDR2]]: 8b 45 f8
+
+## clang++ main.cpp -g2 -gdwarf-4 -S
+## int main() {
+##   int a = 4;
+##   if (a == 5)
+##     goto LABEL1;
+##   else
+##     goto LABEL2;
+##   LABEL1:a++;
+##   LABEL2:a--;
+##   return 0;
+## }
+
+		.text
+	.file	"main.cpp"
+	.globl	main                            # -- Begin function main
+	.p2align	4, 0x90
+	.type	main,@function
+main:                                   # @main
+.Lfunc_begin0:
+	.file	1 "/home" "main.cpp"
+	.loc	1 1 0                           # main.cpp:1:0
+	.cfi_startproc
+# %bb.0:                                # %entry
+	pushq	%rbp
+	.cfi_def_cfa_offset 16
+	.cfi_offset %rbp, -16
+	movq	%rsp, %rbp
+	.cfi_def_cfa_register %rbp
+	movl	$0, -4(%rbp)
+.Ltmp0:
+	.loc	1 2 7 prologue_end              # main.cpp:2:7
+	movl	$4, -8(%rbp)
+.Ltmp1:
+	.loc	1 3 9                           # main.cpp:3:9
+	cmpl	$5, -8(%rbp)
+.Ltmp2:
+	.loc	1 3 7 is_stmt 0                 # main.cpp:3:7
+	jne	.LBB0_2
+# %bb.1:                                # %if.then
+.Ltmp3:
+	.loc	1 4 5 is_stmt 1                 # main.cpp:4:5
+	jmp	.LBB0_3
+.LBB0_2:                                # %if.else
+	.loc	1 6 5                           # main.cpp:6:5
+	jmp	.LBB0_4
+.Ltmp4:
+.LBB0_3:                                # %LABEL1
+	#DEBUG_LABEL: main:LABEL1
+	.loc	1 7 11                          # main.cpp:7:11
+	movl	-8(%rbp), %eax
+	addl	$1, %eax
+	movl	%eax, -8(%rbp)
+.LBB0_4:                                # %LABEL2
+.Ltmp5:
+	#DEBUG_LABEL: main:LABEL2
+	.loc	1 8 11                          # main.cpp:8:11
+	movl	-8(%rbp), %eax
+	addl	$-1, %eax
+	movl	%eax, -8(%rbp)
+	.loc	1 9 3                           # main.cpp:9:3
+	xorl	%eax, %eax
+	.loc	1 9 3 epilogue_begin is_stmt 0  # main.cpp:9:3
+	popq	%rbp
+	.cfi_def_cfa %rsp, 8
+	retq
+.Ltmp6:
+.Lfunc_end0:
+	.size	main, .Lfunc_end0-main
+	.cfi_endproc
+                                        # -- End function
+	.section	.debug_abbrev,"",@progbits
+	.byte	1                               # Abbreviation Code
+	.byte	17                              # DW_TAG_compile_unit
+	.byte	1                               # DW_CHILDREN_yes
+	.byte	37                              # DW_AT_producer
+	.byte	14                              # DW_FORM_strp
+	.byte	19                              # DW_AT_language
+	.byte	5                               # DW_FORM_data2
+	.byte	3                               # DW_AT_name
+	.byte	14                              # DW_FORM_strp
+	.byte	16                              # DW_AT_stmt_list
+	.byte	23                              # DW_FORM_sec_offset
+	.byte	27                              # DW_AT_comp_dir
+	.byte	14                              # DW_FORM_strp
+	.byte	17                              # DW_AT_low_pc
+	.byte	1                               # DW_FORM_addr
+	.byte	18                              # DW_AT_high_pc
+	.byte	6                               # DW_FORM_data4
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	2                               # Abbreviation Code
+	.byte	46                              # DW_TAG_subprogram
+	.byte	1                               # DW_CHILDREN_yes
+	.byte	17                              # DW_AT_low_pc
+	.byte	1                               # DW_FORM_addr
+	.byte	18                              # DW_AT_high_pc
+	.byte	6                               # DW_FORM_data4
+	.byte	64                              # DW_AT_frame_base
+	.byte	24                              # DW_FORM_exprloc
+	.byte	3                               # DW_AT_name
+	.byte	14                              # DW_FORM_strp
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	73                              # DW_AT_type
+	.byte	19                              # DW_FORM_ref4
+	.byte	63                              # DW_AT_external
+	.byte	25                              # DW_FORM_flag_present
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	3                               # Abbreviation Code
+	.byte	52                              # DW_TAG_variable
+	.byte	0                               # DW_CHILDREN_no
+	.byte	2                               # DW_AT_location
+	.byte	24                              # DW_FORM_exprloc
+	.byte	3                               # DW_AT_name
+	.byte	14                              # DW_FORM_strp
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	73                              # DW_AT_type
+	.byte	19                              # DW_FORM_ref4
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	4                               # Abbreviation Code
+	.byte	10                              # DW_TAG_label
+	.byte	0                               # DW_CHILDREN_no
+	.byte	3                               # DW_AT_name
+	.byte	14                              # DW_FORM_strp
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	17                              # DW_AT_low_pc
+	.byte	1                               # DW_FORM_addr
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	5                               # Abbreviation Code
+	.byte	36                              # DW_TAG_base_type
+	.byte	0                               # DW_CHILDREN_no
+	.byte	3                               # DW_AT_name
+	.byte	14                              # DW_FORM_strp
+	.byte	62                              # DW_AT_encoding
+	.byte	11                              # DW_FORM_data1
+	.byte	11                              # DW_AT_byte_size
+	.byte	11                              # DW_FORM_data1
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	0                               # EOM(3)
+	.section	.debug_info,"",@progbits
+.Lcu_begin0:
+	.long	.Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
+.Ldebug_info_start0:
+	.short	4                               # DWARF version number
+	.long	.debug_abbrev                   # Offset Into Abbrev. Section
+	.byte	8                               # Address Size (in bytes)
+	.byte	1                               # Abbrev [1] 0xb:0x6d DW_TAG_compile_unit
+	.long	.Linfo_string0                  # DW_AT_producer
+	.short	33                              # DW_AT_language
+	.long	.Linfo_string1                  # DW_AT_name
+	.long	.Lline_table_start0             # DW_AT_stmt_list
+	.long	.Linfo_string2                  # DW_AT_comp_dir
+	.quad	.Lfunc_begin0                   # DW_AT_low_pc
+	.long	.Lfunc_end0-.Lfunc_begin0       # DW_AT_high_pc
+	.byte	2                               # Abbrev [2] 0x2a:0x46 DW_TAG_subprogram
+	.quad	.Lfunc_begin0                   # DW_AT_low_pc
+	.long	.Lfunc_end0-.Lfunc_begin0       # DW_AT_high_pc
+	.byte	1                               # DW_AT_frame_base
+	.byte	86
+	.long	.Linfo_string3                  # DW_AT_name
+	.byte	1                               # DW_AT_decl_file
+	.byte	1                               # DW_AT_decl_line
+	.long	112                             # DW_AT_type
+                                        # DW_AT_external
+	.byte	3                               # Abbrev [3] 0x43:0xe DW_TAG_variable
+	.byte	2                               # DW_AT_location
+	.byte	145
+	.byte	120
+	.long	.Linfo_string5                  # DW_AT_name
+	.byte	1                               # DW_AT_decl_file
+	.byte	2                               # DW_AT_decl_line
+	.long	112                             # DW_AT_type
+	.byte	4                               # Abbrev [4] 0x51:0xf DW_TAG_label
+	.long	.Linfo_string6                  # DW_AT_name
+	.byte	1                               # DW_AT_decl_file
+	.byte	7                               # DW_AT_decl_line
+	.quad	.Ltmp4                          # DW_AT_low_pc
+	.byte	4                               # Abbrev [4] 0x60:0xf DW_TAG_label
+	.long	.Linfo_string7                  # DW_AT_name
+	.byte	1                               # DW_AT_decl_file
+	.byte	8                               # DW_AT_decl_line
+	.quad	.Ltmp5                          # DW_AT_low_pc
+	.byte	0                               # End Of Children Mark
+	.byte	5                               # Abbrev [5] 0x70:0x7 DW_TAG_base_type
+	.long	.Linfo_string4                  # DW_AT_name
+	.byte	5                               # DW_AT_encoding
+	.byte	4                               # DW_AT_byte_size
+	.byte	0                               # End Of Children Mark
+.Ldebug_info_end0:
+	.section	.debug_str,"MS",@progbits,1
+.Linfo_string0:
+	.asciz	"clang version 19.0.0git"       # string offset=0
+.Linfo_string1:
+	.asciz	"main.cpp"                      # string offset=24
+.Linfo_string2:
+	.asciz	"/home" # string offset=33
+.Linfo_string3:
+	.asciz	"main"                          # string offset=71
+.Linfo_string4:
+	.asciz	"int"                           # string offset=76
+.Linfo_string5:
+	.asciz	"a"                             # string offset=80
+.Linfo_string6:
+	.asciz	"LABEL1"                        # string offset=82
+.Linfo_string7:
+	.asciz	"LABEL2"                        # string offset=89
+	.ident	"clang version 19.0.0git"
+	.section	".note.GNU-stack","",@progbits
+	.addrsig
+	.section	.debug_line,"",@progbits
+.Lline_table_start0:
diff --git a/bolt/test/X86/dwarf5-label-low-pc.s b/bolt/test/X86/dwarf5-label-low-pc.s
index 890d9e024d1a..1e3fc17ad516 100644
--- a/bolt/test/X86/dwarf5-label-low-pc.s
+++ b/bolt/test/X86/dwarf5-label-low-pc.s
@@ -8,9 +8,10 @@
 
 # RUN: llvm-dwarfdump --show-form --verbose --debug-addr %t.bolt > %t.txt
 # RUN: llvm-dwarfdump --show-form --verbose --debug-info %t.bolt >> %t.txt
+# RUN: llvm-objdump -d %t.bolt >> %t.txt
 # RUN: cat %t.txt | FileCheck --check-prefix=POSTCHECK %s
 
-# This test checks that we correctly handle DW_AT_low_pc [DW_FORM_addrx] that is part of DW_TAG_label.
+## This test checks that we correctly handle DW_AT_low_pc [DW_FORM_addrx] that is part of DW_TAG_label.
 
 # PRECHECK: version = 0x0005
 # PRECHECK: DW_TAG_label
@@ -28,8 +29,8 @@
 # POSTCHECK: Addrs: [
 # POSTCHECK-NEXT: 0x
 # POSTCHECK-NEXT: 0x
-# POSTCHECK-NEXT: 0x[[#%.16x,ADDR:]]
-# POSTCHECK-NEXT: 0x[[#%.16x,ADDR2:]]
+# POSTCHECK-NEXT: 0x[[ADDR:[1-9a-f]*]]
+# POSTCHECK-NEXT: 0x[[ADDR2:[1-9a-f]*]]
 
 # POSTCHECK: version = 0x0005
 # POSTCHECK: DW_TAG_label
@@ -37,25 +38,28 @@
 # POSTCHECK-NEXT: DW_AT_decl_file
 # POSTCHECK-NEXT: DW_AT_decl_line
 # POSTCHECK-NEXT:DW_AT_low_pc [DW_FORM_addrx]  (indexed (00000002)
-# POSTCHECK-SAME: 0x[[#ADDR]]
+# POSTCHECK-SAME: 0x[[ADDR]]
 # POSTCHECK: DW_TAG_label
 # POSTCHECK-NEXT: DW_AT_name
 # POSTCHECK-NEXT: DW_AT_decl_file
 # POSTCHECK-NEXT: DW_AT_decl_line
 # POSTCHECK-NEXT:DW_AT_low_pc [DW_FORM_addrx]  (indexed (00000003)
-# POSTCHECK-SAME: 0x[[#ADDR2]]
+# POSTCHECK-SAME: 0x[[ADDR2]]
 
-# clang++ main.cpp -g -S
-# int main() {
-#   int a = 4;
-#   if (a == 5)
-#     goto LABEL1;
-#   else
-#     goto LABEL2;
-#   LABEL1:a++;
-#   LABEL2:a--;
-#   return 0;
-# }
+# POSTCHECK: [[ADDR]]: 8b 45 f8
+# POSTCHECK: [[ADDR2]]: 8b 45 f8
+
+## clang++ main.cpp -g -S
+## int main() {
+##   int a = 4;
+##   if (a == 5)
+##     goto LABEL1;
+##   else
+##     goto LABEL2;
+##   LABEL1:a++;
+##   LABEL2:a--;
+##   return 0;
+## }
 
 	.text
 	.file	"main.cpp"
-- 
GitLab


From f3cfe016c5d8429c0dccfa6f85442e2ea0d45a58 Mon Sep 17 00:00:00 2001
From: Alexander Yermolovich <43973793+ayermolo@users.noreply.github.com>
Date: Fri, 22 Mar 2024 13:48:49 -0700
Subject: [PATCH 007/404] [BOLT][DWARF] Add support for cross-cu references for
 debug-names (#86015)

The DW_AT_abstract_origin can be a cross-cu reference as a by-product of
LTO. On IR level for absolute references an address is stored, vs a DIE
for relative references. Added a map to keep track of cross-cu
referenced DIEs to use when we add an Entry.
---
 bolt/include/bolt/Core/DebugNames.h         |  11 +
 bolt/lib/Core/DIEBuilder.cpp                |   6 +-
 bolt/lib/Core/DebugNames.cpp                | 109 +--
 bolt/test/X86/dwarf5-debug-names-cross-cu.s | 712 ++++++++++++++++++++
 4 files changed, 796 insertions(+), 42 deletions(-)
 create mode 100644 bolt/test/X86/dwarf5-debug-names-cross-cu.s

diff --git a/bolt/include/bolt/Core/DebugNames.h b/bolt/include/bolt/Core/DebugNames.h
index 1f17f1ae4d13..fbaa7f4e68aa 100644
--- a/bolt/include/bolt/Core/DebugNames.h
+++ b/bolt/include/bolt/Core/DebugNames.h
@@ -68,6 +68,16 @@ public:
   std::unique_ptr<DebugBufferVector> releaseBuffer() {
     return std::move(FullTableBuffer);
   }
+  /// Adds a DIE that is referenced across CUs.
+  void addCrossCUDie(const DIE *Die) {
+    CrossCUDies.insert({Die->getOffset(), Die});
+  }
+  /// Returns true if the DIE can generate an entry for a cross cu reference.
+  /// This only checks TAGs of a DIE because when this is invoked DIE might not
+  /// be fully constructed.
+  bool canGenerateEntryWithCrossCUReference(
+      const DWARFUnit &Unit, const DIE &Die,
+      const DWARFAbbreviationDeclaration::AttributeSpec &AttrSpec);
 
 private:
   BinaryContext &BC;
@@ -128,6 +138,7 @@ private:
   llvm::DenseMap<uint64_t, uint32_t> CUOffsetsToPatch;
   // Contains a map of Entry ID to Entry relative offset.
   llvm::DenseMap<uint64_t, uint32_t> EntryRelativeOffsets;
+  llvm::DenseMap<uint64_t, const DIE *> CrossCUDies;
   /// Adds Unit to either CUList, LocalTUList or ForeignTUList.
   /// Input Unit being processed, and DWO ID if Unit is being processed comes
   /// from a DWO section.
diff --git a/bolt/lib/Core/DIEBuilder.cpp b/bolt/lib/Core/DIEBuilder.cpp
index 0cf8a5e8c2c3..354fe5059443 100644
--- a/bolt/lib/Core/DIEBuilder.cpp
+++ b/bolt/lib/Core/DIEBuilder.cpp
@@ -545,6 +545,10 @@ void DIEBuilder::cloneDieReferenceAttribute(
   NewRefDie = DieInfo.Die;
 
   if (AttrSpec.Form == dwarf::DW_FORM_ref_addr) {
+    // Adding referenced DIE to DebugNames to be used when entries are created
+    // that contain cross cu references.
+    if (DebugNamesTable.canGenerateEntryWithCrossCUReference(U, Die, AttrSpec))
+      DebugNamesTable.addCrossCUDie(DieInfo.Die);
     // no matter forward reference or backward reference, we are supposed
     // to calculate them in `finish` due to the possible modification of
     // the DIE.
@@ -554,7 +558,7 @@ void DIEBuilder::cloneDieReferenceAttribute(
         std::make_pair(CurDieInfo, AddrReferenceInfo(&DieInfo, AttrSpec)));
 
     Die.addValue(getState().DIEAlloc, AttrSpec.Attr, dwarf::DW_FORM_ref_addr,
-                 DIEInteger(0xDEADBEEF));
+                 DIEInteger(DieInfo.Die->getOffset()));
     return;
   }
 
diff --git a/bolt/lib/Core/DebugNames.cpp b/bolt/lib/Core/DebugNames.cpp
index 23a29f52513c..049244c4b515 100644
--- a/bolt/lib/Core/DebugNames.cpp
+++ b/bolt/lib/Core/DebugNames.cpp
@@ -146,6 +146,55 @@ static bool shouldIncludeVariable(const DWARFUnit &Unit, const DIE &Die) {
   return false;
 }
 
+bool static canProcess(const DWARFUnit &Unit, const DIE &Die,
+                       std::string &NameToUse, const bool TagsOnly) {
+  switch (Die.getTag()) {
+  case dwarf::DW_TAG_base_type:
+  case dwarf::DW_TAG_class_type:
+  case dwarf::DW_TAG_enumeration_type:
+  case dwarf::DW_TAG_imported_declaration:
+  case dwarf::DW_TAG_pointer_type:
+  case dwarf::DW_TAG_structure_type:
+  case dwarf::DW_TAG_typedef:
+  case dwarf::DW_TAG_unspecified_type:
+    if (TagsOnly || Die.findAttribute(dwarf::Attribute::DW_AT_name))
+      return true;
+    return false;
+  case dwarf::DW_TAG_namespace:
+    // According to DWARF5 spec namespaces without DW_AT_name needs to have
+    // "(anonymous namespace)"
+    if (!Die.findAttribute(dwarf::Attribute::DW_AT_name))
+      NameToUse = "(anonymous namespace)";
+    return true;
+  case dwarf::DW_TAG_inlined_subroutine:
+  case dwarf::DW_TAG_label:
+  case dwarf::DW_TAG_subprogram:
+    if (TagsOnly || Die.findAttribute(dwarf::Attribute::DW_AT_low_pc) ||
+        Die.findAttribute(dwarf::Attribute::DW_AT_high_pc) ||
+        Die.findAttribute(dwarf::Attribute::DW_AT_ranges) ||
+        Die.findAttribute(dwarf::Attribute::DW_AT_entry_pc))
+      return true;
+    return false;
+  case dwarf::DW_TAG_variable:
+    return TagsOnly || shouldIncludeVariable(Unit, Die);
+  default:
+    break;
+  }
+  return false;
+}
+
+bool DWARF5AcceleratorTable::canGenerateEntryWithCrossCUReference(
+    const DWARFUnit &Unit, const DIE &Die,
+    const DWARFAbbreviationDeclaration::AttributeSpec &AttrSpec) {
+  if (!isCreated())
+    return false;
+  std::string NameToUse = "";
+  if (!canProcess(Unit, Die, NameToUse, true))
+    return false;
+  return (AttrSpec.Attr == dwarf::Attribute::DW_AT_abstract_origin ||
+          AttrSpec.Attr == dwarf::Attribute::DW_AT_specification) &&
+         AttrSpec.Form == dwarf::DW_FORM_ref_addr;
+}
 /// Returns name offset in String Offset section.
 static uint64_t getNameOffset(BinaryContext &BC, DWARFUnit &Unit,
                               const uint64_t Index) {
@@ -175,41 +224,6 @@ DWARF5AcceleratorTable::addAccelTableEntry(
   if (Unit.getVersion() < 5 || !NeedToCreate)
     return std::nullopt;
   std::string NameToUse = "";
-  auto canProcess = [&](const DIE &Die) -> bool {
-    switch (Die.getTag()) {
-    case dwarf::DW_TAG_base_type:
-    case dwarf::DW_TAG_class_type:
-    case dwarf::DW_TAG_enumeration_type:
-    case dwarf::DW_TAG_imported_declaration:
-    case dwarf::DW_TAG_pointer_type:
-    case dwarf::DW_TAG_structure_type:
-    case dwarf::DW_TAG_typedef:
-    case dwarf::DW_TAG_unspecified_type:
-      if (Die.findAttribute(dwarf::Attribute::DW_AT_name))
-        return true;
-      return false;
-    case dwarf::DW_TAG_namespace:
-      // According to DWARF5 spec namespaces without DW_AT_name needs to have
-      // "(anonymous namespace)"
-      if (!Die.findAttribute(dwarf::Attribute::DW_AT_name))
-        NameToUse = "(anonymous namespace)";
-      return true;
-    case dwarf::DW_TAG_inlined_subroutine:
-    case dwarf::DW_TAG_label:
-    case dwarf::DW_TAG_subprogram:
-      if (Die.findAttribute(dwarf::Attribute::DW_AT_low_pc) ||
-          Die.findAttribute(dwarf::Attribute::DW_AT_high_pc) ||
-          Die.findAttribute(dwarf::Attribute::DW_AT_ranges) ||
-          Die.findAttribute(dwarf::Attribute::DW_AT_entry_pc))
-        return true;
-      return false;
-    case dwarf::DW_TAG_variable:
-      return shouldIncludeVariable(Unit, Die);
-    default:
-      break;
-    }
-    return false;
-  };
 
   auto getUnitID = [&](const DWARFUnit &Unit, bool &IsTU,
                        uint32_t &DieTag) -> uint32_t {
@@ -223,7 +237,7 @@ DWARF5AcceleratorTable::addAccelTableEntry(
     return CUList.size() - 1;
   };
 
-  if (!canProcess(Die))
+  if (!canProcess(Unit, Die, NameToUse, false))
     return std::nullopt;
 
   // Addes a Unit to either CU, LocalTU or ForeignTU list the first time we
@@ -318,10 +332,24 @@ DWARF5AcceleratorTable::addAccelTableEntry(
     const DIEValue Value = Die.findAttribute(Attr);
     if (!Value)
       return std::nullopt;
-    const DIEEntry &DIEENtry = Value.getDIEEntry();
-    DIE &EntryDie = DIEENtry.getEntry();
-    addEntry(EntryDie.findAttribute(dwarf::Attribute::DW_AT_linkage_name));
-    return addEntry(EntryDie.findAttribute(dwarf::Attribute::DW_AT_name));
+    const DIE *EntryDie = nullptr;
+    if (Value.getForm() == dwarf::DW_FORM_ref_addr) {
+      auto Iter = CrossCUDies.find(Value.getDIEInteger().getValue());
+      if (Iter == CrossCUDies.end()) {
+        BC.errs() << "BOLT-WARNING: [internal-dwarf-warning]: Could not find "
+                     "referenced DIE in CrossCUDies for "
+                  << Twine::utohexstr(Value.getDIEInteger().getValue())
+                  << ".\n";
+        return std::nullopt;
+      }
+      EntryDie = Iter->second;
+    } else {
+      const DIEEntry &DIEENtry = Value.getDIEEntry();
+      EntryDie = &DIEENtry.getEntry();
+    }
+
+    addEntry(EntryDie->findAttribute(dwarf::Attribute::DW_AT_linkage_name));
+    return addEntry(EntryDie->findAttribute(dwarf::Attribute::DW_AT_name));
   };
 
   if (std::optional<BOLTDWARF5AccelTableData *> Entry =
@@ -332,7 +360,6 @@ DWARF5AcceleratorTable::addAccelTableEntry(
     return *Entry;
 
   return addEntry(Die.findAttribute(dwarf::Attribute::DW_AT_name));
-  ;
 }
 
 /// Algorithm from llvm implementation.
diff --git a/bolt/test/X86/dwarf5-debug-names-cross-cu.s b/bolt/test/X86/dwarf5-debug-names-cross-cu.s
new file mode 100644
index 000000000000..73c50d6d41db
--- /dev/null
+++ b/bolt/test/X86/dwarf5-debug-names-cross-cu.s
@@ -0,0 +1,712 @@
+
+# REQUIRES: system-linux
+
+# RUN: llvm-mc -dwarf-version=5 -filetype=obj -triple x86_64-unknown-linux %s -o %tmain.o
+# RUN: %clang %cflags -dwarf-5 %tmain.o -o %t.exe -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.bolt --update-debug-sections
+# RUN: llvm-dwarfdump --debug-info -r 0 --debug-names %t.bolt > %t.txt
+# RUN: cat %t.txt | FileCheck --check-prefix=CHECK %s
+
+## This test checks that BOLT generates Entries for DW_AT_abstract_origin when it has cross cu reference.
+
+# CHECK: [[OFFSET1:0x[0-9a-f]*]]: Compile Unit
+# CHECK: [[OFFSET2:0x[0-9a-f]*]]: Compile Unit
+# CHECK:        Name Index @ 0x0 {
+# CHECK-NEXT:     Header {
+# CHECK-NEXT:       Length: 0xD2
+# CHECK-NEXT:       Format: DWARF32
+# CHECK-NEXT:       Version: 5
+# CHECK-NEXT:       CU count: 2
+# CHECK-NEXT:       Local TU count: 0
+# CHECK-NEXT:       Foreign TU count: 0
+# CHECK-NEXT:       Bucket count: 5
+# CHECK-NEXT:       Name count: 5
+# CHECK-NEXT:       Abbreviations table size: 0x1F
+# CHECK-NEXT:       Augmentation: 'BOLT'
+# CHECK-NEXT:     }
+# CHECK-NEXT:     Compilation Unit offsets [
+# CHECK-NEXT:       CU[0]: [[OFFSET1]]
+# CHECK-NEXT:       CU[1]: [[OFFSET2]]
+# CHECK-NEXT:     ]
+# CHECK-NEXT:     Abbreviations [
+# CHECK-NEXT:       Abbreviation [[ABBREV1:0x[0-9a-f]*]] {
+# CHECK-NEXT:         Tag: DW_TAG_subprogram
+# CHECK-NEXT:         DW_IDX_compile_unit: DW_FORM_data1
+# CHECK-NEXT:         DW_IDX_die_offset: DW_FORM_ref4
+# CHECK-NEXT:         DW_IDX_parent: DW_FORM_flag_present
+# CHECK-NEXT:       }
+# CHECK-NEXT:       Abbreviation [[ABBREV2:0x[0-9a-f]*]] {
+# CHECK-NEXT:         Tag: DW_TAG_inlined_subroutine
+# CHECK-NEXT:         DW_IDX_compile_unit: DW_FORM_data1
+# CHECK-NEXT:         DW_IDX_die_offset: DW_FORM_ref4
+# CHECK-NEXT:         DW_IDX_parent: DW_FORM_ref4
+# CHECK-NEXT:       }
+# CHECK-NEXT:       Abbreviation [[ABBREV3:0x[0-9a-f]*]] {
+# CHECK-NEXT:         Tag: DW_TAG_base_type
+# CHECK-NEXT:         DW_IDX_compile_unit: DW_FORM_data1
+# CHECK-NEXT:         DW_IDX_die_offset: DW_FORM_ref4
+# CHECK-NEXT:         DW_IDX_parent: DW_FORM_flag_present
+# CHECK-NEXT:       }
+# CHECK-NEXT:     ]
+# CHECK-NEXT:     Bucket 0 [
+# CHECK-NEXT:       EMPTY
+# CHECK-NEXT:     ]
+# CHECK-NEXT:     Bucket 1 [
+# CHECK-NEXT:       Name 1 {
+# CHECK-NEXT:         Hash: 0x7C9A7F6A
+# CHECK-NEXT:         String: {{.+}} "main"
+# CHECK-NEXT:         Entry @ [[ENTRY:0x[0-9a-f]*]] {
+# CHECK-NEXT:           Abbrev: [[ABBREV1]]
+# CHECK-NEXT:           Tag: DW_TAG_subprogram
+# CHECK-NEXT:           DW_IDX_compile_unit: 0x00
+# CHECK-NEXT:           DW_IDX_die_offset: 0x00000024
+# CHECK-NEXT:           DW_IDX_parent: <parent not indexed>
+# CHECK-NEXT:         }
+# CHECK-NEXT:       }
+# CHECK-NEXT:       Name 2 {
+# CHECK-NEXT:         Hash: 0xB5063CFE
+# CHECK-NEXT:         String: {{.+}} "_Z3fooi"
+# CHECK-NEXT:         Entry @ {{.+}} {
+# CHECK-NEXT:           Abbrev: [[ABBREV1]]
+# CHECK-NEXT:           Tag: DW_TAG_subprogram
+# CHECK-NEXT:           DW_IDX_compile_unit: 0x01
+# CHECK-NEXT:           DW_IDX_die_offset: 0x0000003a
+# CHECK-NEXT:           DW_IDX_parent: <parent not indexed>
+# CHECK-NEXT:         }
+# CHECK-NEXT:         Entry @ {{.+}} {
+# CHECK-NEXT:           Abbrev: [[ABBREV2]]
+# CHECK-NEXT:           Tag: DW_TAG_inlined_subroutine
+# CHECK-NEXT:           DW_IDX_compile_unit: 0x00
+# CHECK-NEXT:           DW_IDX_die_offset: 0x00000054
+# CHECK-NEXT:           DW_IDX_parent: Entry @ [[ENTRY]]
+# CHECK-NEXT:         }
+# CHECK-NEXT:       }
+# CHECK-NEXT:     ]
+# CHECK-NEXT:     Bucket 2 [
+# CHECK-NEXT:       EMPTY
+# CHECK-NEXT:     ]
+# CHECK-NEXT:     Bucket 3 [
+# CHECK-NEXT:       Name 3 {
+# CHECK-NEXT:         Hash: 0xB888030
+# CHECK-NEXT:         String: {{.+}} "int"
+# CHECK-NEXT:         Entry @ {{.+}} {
+# CHECK-NEXT:           Abbrev: [[ABBREV3]]
+# CHECK-NEXT:           Tag: DW_TAG_base_type
+# CHECK-NEXT:           DW_IDX_compile_unit: 0x01
+# CHECK-NEXT:           DW_IDX_die_offset: 0x00000036
+# CHECK-NEXT:           DW_IDX_parent: <parent not indexed>
+# CHECK-NEXT:         }
+# CHECK-NEXT:       }
+# CHECK-NEXT:     ]
+# CHECK-NEXT:     Bucket 4 [
+# CHECK-NEXT:       Name 4 {
+# CHECK-NEXT:         Hash: 0xB887389
+# CHECK-NEXT:         String: {{.+}} "foo"
+# CHECK-NEXT:         Entry @ {{.+}} {
+# CHECK-NEXT:           Abbrev: [[ABBREV1]]
+# CHECK-NEXT:           Tag: DW_TAG_subprogram
+# CHECK-NEXT:           DW_IDX_compile_unit: 0x01
+# CHECK-NEXT:           DW_IDX_die_offset: 0x0000003a
+# CHECK-NEXT:           DW_IDX_parent: <parent not indexed>
+# CHECK-NEXT:         }
+# CHECK-NEXT:         Entry @ 0xc4 {
+# CHECK-NEXT:           Abbrev: [[ABBREV2]]
+# CHECK-NEXT:           Tag: DW_TAG_inlined_subroutine
+# CHECK-NEXT:           DW_IDX_compile_unit: 0x00
+# CHECK-NEXT:           DW_IDX_die_offset: 0x00000054
+# CHECK-NEXT:           DW_IDX_parent: Entry @ [[ENTRY]]
+# CHECK-NEXT:         }
+# CHECK-NEXT:       }
+# CHECK-NEXT:       Name 5 {
+# CHECK-NEXT:         Hash: 0x7C952063
+# CHECK-NEXT:         String: {{.+}} "char"
+# CHECK-NEXT:         Entry @ {{.+}} {
+# CHECK-NEXT:           Abbrev: [[ABBREV3]]
+# CHECK-NEXT:           Tag: DW_TAG_base_type
+# CHECK-NEXT:           DW_IDX_compile_unit: 0x00
+# CHECK-NEXT:           DW_IDX_die_offset: 0x00000075
+# CHECK-NEXT:           DW_IDX_parent: <parent not indexed>
+# CHECK-NEXT:         }
+# CHECK-NEXT:       }
+# CHECK-NEXT:     ]
+# CHECK-NEXT:   }
+
+## clang++ -g2 -gpubnames -S -emit-llvm main.cpp -o main.ll
+## clang++ -g2 -gpubnames -S -emit-llvm helper.cpp -o helper.ll
+## llvm-link main.ll helper.ll -o combined.ll
+## clang++ -g2 -gpubnames combined.ll -emit-llvm -S -o combined.opt.ll
+## llc -dwarf-version=5 -filetype=asm -mtriple x86_64-unknown-linux combined.opt.ll -o combined.s
+## main.cpp
+## extern int foo(int);
+## int main(int argc, char* argv[]) {
+##   int i = 0;
+##   [[clang::always_inline]] i = foo(argc);
+##   return i;
+## }
+## helper.cpp
+## int foo(int i) {
+##   return i ++;
+## }
+
+	.text
+	.file	"llvm-link"
+	.globl	main                            # -- Begin function main
+	.p2align	4, 0x90
+	.type	main,@function
+main:                                   # @main
+.Lfunc_begin0:
+	.file	1 "/home" "main.cpp" md5 0x24fb0b4c3900e91fece1ac87ed73ff3b
+	.loc	1 2 0                           # main.cpp:2:0
+	.cfi_startproc
+# %bb.0:                                # %entry
+	pushq	%rbp
+	.cfi_def_cfa_offset 16
+	.cfi_offset %rbp, -16
+	movq	%rsp, %rbp
+	.cfi_def_cfa_register %rbp
+	movl	$0, -16(%rbp)
+	movl	%edi, -12(%rbp)
+	movq	%rsi, -24(%rbp)
+.Ltmp0:
+	.loc	1 3 7 prologue_end              # main.cpp:3:7
+	movl	$0, -4(%rbp)
+	.loc	1 4 36                          # main.cpp:4:36
+	movl	-12(%rbp), %eax
+	movl	%eax, -8(%rbp)
+.Ltmp1:
+	.file	2 "/home" "helper.cpp" md5 0x7d4429e24d8c74d7ee22c1889ad46d6b
+	.loc	2 2 12                          # helper.cpp:2:12
+	movl	-8(%rbp), %eax
+	movl	%eax, %ecx
+	addl	$1, %ecx
+	movl	%ecx, -8(%rbp)
+.Ltmp2:
+	.loc	1 4 30                          # main.cpp:4:30
+	movl	%eax, -4(%rbp)
+	.loc	1 5 10                          # main.cpp:5:10
+	movl	-4(%rbp), %eax
+	.loc	1 5 3 epilogue_begin is_stmt 0  # main.cpp:5:3
+	popq	%rbp
+	.cfi_def_cfa %rsp, 8
+	retq
+.Ltmp3:
+.Lfunc_end0:
+	.size	main, .Lfunc_end0-main
+	.cfi_endproc
+                                        # -- End function
+	.globl	_Z3fooi                         # -- Begin function _Z3fooi
+	.p2align	4, 0x90
+	.type	_Z3fooi,@function
+_Z3fooi:                                # @_Z3fooi
+.Lfunc_begin1:
+	.loc	2 1 0 is_stmt 1                 # helper.cpp:1:0
+	.cfi_startproc
+# %bb.0:                                # %entry
+	pushq	%rbp
+	.cfi_def_cfa_offset 16
+	.cfi_offset %rbp, -16
+	movq	%rsp, %rbp
+	.cfi_def_cfa_register %rbp
+	movl	%edi, -4(%rbp)
+.Ltmp4:
+	.loc	2 2 12 prologue_end             # helper.cpp:2:12
+	movl	-4(%rbp), %eax
+	movl	%eax, %ecx
+	addl	$1, %ecx
+	movl	%ecx, -4(%rbp)
+	.loc	2 2 3 epilogue_begin is_stmt 0  # helper.cpp:2:3
+	popq	%rbp
+	.cfi_def_cfa %rsp, 8
+	retq
+.Ltmp5:
+.Lfunc_end1:
+	.size	_Z3fooi, .Lfunc_end1-_Z3fooi
+	.cfi_endproc
+                                        # -- End function
+	.section	.debug_abbrev,"",@progbits
+	.byte	1                               # Abbreviation Code
+	.byte	17                              # DW_TAG_compile_unit
+	.byte	1                               # DW_CHILDREN_yes
+	.byte	37                              # DW_AT_producer
+	.byte	37                              # DW_FORM_strx1
+	.byte	19                              # DW_AT_language
+	.byte	5                               # DW_FORM_data2
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	114                             # DW_AT_str_offsets_base
+	.byte	23                              # DW_FORM_sec_offset
+	.byte	16                              # DW_AT_stmt_list
+	.byte	23                              # DW_FORM_sec_offset
+	.byte	27                              # DW_AT_comp_dir
+	.byte	37                              # DW_FORM_strx1
+	.byte	17                              # DW_AT_low_pc
+	.byte	27                              # DW_FORM_addrx
+	.byte	18                              # DW_AT_high_pc
+	.byte	6                               # DW_FORM_data4
+	.byte	115                             # DW_AT_addr_base
+	.byte	23                              # DW_FORM_sec_offset
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	2                               # Abbreviation Code
+	.byte	46                              # DW_TAG_subprogram
+	.byte	1                               # DW_CHILDREN_yes
+	.byte	17                              # DW_AT_low_pc
+	.byte	27                              # DW_FORM_addrx
+	.byte	18                              # DW_AT_high_pc
+	.byte	6                               # DW_FORM_data4
+	.byte	64                              # DW_AT_frame_base
+	.byte	24                              # DW_FORM_exprloc
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	73                              # DW_AT_type
+	.byte	16                              # DW_FORM_ref_addr
+	.byte	63                              # DW_AT_external
+	.byte	25                              # DW_FORM_flag_present
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	3                               # Abbreviation Code
+	.byte	5                               # DW_TAG_formal_parameter
+	.byte	0                               # DW_CHILDREN_no
+	.byte	2                               # DW_AT_location
+	.byte	24                              # DW_FORM_exprloc
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	73                              # DW_AT_type
+	.byte	16                              # DW_FORM_ref_addr
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	4                               # Abbreviation Code
+	.byte	5                               # DW_TAG_formal_parameter
+	.byte	0                               # DW_CHILDREN_no
+	.byte	2                               # DW_AT_location
+	.byte	24                              # DW_FORM_exprloc
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	73                              # DW_AT_type
+	.byte	19                              # DW_FORM_ref4
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	5                               # Abbreviation Code
+	.byte	52                              # DW_TAG_variable
+	.byte	0                               # DW_CHILDREN_no
+	.byte	2                               # DW_AT_location
+	.byte	24                              # DW_FORM_exprloc
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	73                              # DW_AT_type
+	.byte	16                              # DW_FORM_ref_addr
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	6                               # Abbreviation Code
+	.byte	29                              # DW_TAG_inlined_subroutine
+	.byte	1                               # DW_CHILDREN_yes
+	.byte	49                              # DW_AT_abstract_origin
+	.byte	16                              # DW_FORM_ref_addr
+	.byte	17                              # DW_AT_low_pc
+	.byte	27                              # DW_FORM_addrx
+	.byte	18                              # DW_AT_high_pc
+	.byte	6                               # DW_FORM_data4
+	.byte	88                              # DW_AT_call_file
+	.byte	11                              # DW_FORM_data1
+	.byte	89                              # DW_AT_call_line
+	.byte	11                              # DW_FORM_data1
+	.byte	87                              # DW_AT_call_column
+	.byte	11                              # DW_FORM_data1
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	7                               # Abbreviation Code
+	.byte	5                               # DW_TAG_formal_parameter
+	.byte	0                               # DW_CHILDREN_no
+	.byte	2                               # DW_AT_location
+	.byte	24                              # DW_FORM_exprloc
+	.byte	49                              # DW_AT_abstract_origin
+	.byte	16                              # DW_FORM_ref_addr
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	8                               # Abbreviation Code
+	.byte	15                              # DW_TAG_pointer_type
+	.byte	0                               # DW_CHILDREN_no
+	.byte	73                              # DW_AT_type
+	.byte	19                              # DW_FORM_ref4
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	9                               # Abbreviation Code
+	.byte	36                              # DW_TAG_base_type
+	.byte	0                               # DW_CHILDREN_no
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	62                              # DW_AT_encoding
+	.byte	11                              # DW_FORM_data1
+	.byte	11                              # DW_AT_byte_size
+	.byte	11                              # DW_FORM_data1
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	10                              # Abbreviation Code
+	.byte	46                              # DW_TAG_subprogram
+	.byte	1                               # DW_CHILDREN_yes
+	.byte	110                             # DW_AT_linkage_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	73                              # DW_AT_type
+	.byte	19                              # DW_FORM_ref4
+	.byte	63                              # DW_AT_external
+	.byte	25                              # DW_FORM_flag_present
+	.byte	32                              # DW_AT_inline
+	.byte	33                              # DW_FORM_implicit_const
+	.byte	1
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	11                              # Abbreviation Code
+	.byte	5                               # DW_TAG_formal_parameter
+	.byte	0                               # DW_CHILDREN_no
+	.byte	3                               # DW_AT_name
+	.byte	37                              # DW_FORM_strx1
+	.byte	58                              # DW_AT_decl_file
+	.byte	11                              # DW_FORM_data1
+	.byte	59                              # DW_AT_decl_line
+	.byte	11                              # DW_FORM_data1
+	.byte	73                              # DW_AT_type
+	.byte	19                              # DW_FORM_ref4
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	12                              # Abbreviation Code
+	.byte	46                              # DW_TAG_subprogram
+	.byte	1                               # DW_CHILDREN_yes
+	.byte	17                              # DW_AT_low_pc
+	.byte	27                              # DW_FORM_addrx
+	.byte	18                              # DW_AT_high_pc
+	.byte	6                               # DW_FORM_data4
+	.byte	64                              # DW_AT_frame_base
+	.byte	24                              # DW_FORM_exprloc
+	.byte	49                              # DW_AT_abstract_origin
+	.byte	19                              # DW_FORM_ref4
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	13                              # Abbreviation Code
+	.byte	5                               # DW_TAG_formal_parameter
+	.byte	0                               # DW_CHILDREN_no
+	.byte	2                               # DW_AT_location
+	.byte	24                              # DW_FORM_exprloc
+	.byte	49                              # DW_AT_abstract_origin
+	.byte	19                              # DW_FORM_ref4
+	.byte	0                               # EOM(1)
+	.byte	0                               # EOM(2)
+	.byte	0                               # EOM(3)
+	.section	.debug_info,"",@progbits
+.Lcu_begin0:
+	.long	.Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
+.Ldebug_info_start0:
+	.short	5                               # DWARF version number
+	.byte	1                               # DWARF Unit Type
+	.byte	8                               # Address Size (in bytes)
+	.long	.debug_abbrev                   # Offset Into Abbrev. Section
+	.byte	1                               # Abbrev [1] 0xc:0x6d DW_TAG_compile_unit
+	.byte	0                               # DW_AT_producer
+	.short	33                              # DW_AT_language
+	.byte	1                               # DW_AT_name
+	.long	.Lstr_offsets_base0             # DW_AT_str_offsets_base
+	.long	.Lline_table_start0             # DW_AT_stmt_list
+	.byte	2                               # DW_AT_comp_dir
+	.byte	0                               # DW_AT_low_pc
+	.long	.Lfunc_end0-.Lfunc_begin0       # DW_AT_high_pc
+	.long	.Laddr_table_base0              # DW_AT_addr_base
+	.byte	2                               # Abbrev [2] 0x23:0x47 DW_TAG_subprogram
+	.byte	0                               # DW_AT_low_pc
+	.long	.Lfunc_end0-.Lfunc_begin0       # DW_AT_high_pc
+	.byte	1                               # DW_AT_frame_base
+	.byte	86
+	.byte	8                               # DW_AT_name
+	.byte	1                               # DW_AT_decl_file
+	.byte	2                               # DW_AT_decl_line
+	.long	.debug_info+174                 # DW_AT_type
+                                        # DW_AT_external
+	.byte	3                               # Abbrev [3] 0x32:0xb DW_TAG_formal_parameter
+	.byte	2                               # DW_AT_location
+	.byte	145
+	.byte	116
+	.byte	9                               # DW_AT_name
+	.byte	1                               # DW_AT_decl_file
+	.byte	2                               # DW_AT_decl_line
+	.long	.debug_info+174                 # DW_AT_type
+	.byte	4                               # Abbrev [4] 0x3d:0xb DW_TAG_formal_parameter
+	.byte	2                               # DW_AT_location
+	.byte	145
+	.byte	104
+	.byte	10                              # DW_AT_name
+	.byte	1                               # DW_AT_decl_file
+	.byte	2                               # DW_AT_decl_line
+	.long	106                             # DW_AT_type
+	.byte	5                               # Abbrev [5] 0x48:0xb DW_TAG_variable
+	.byte	2                               # DW_AT_location
+	.byte	145
+	.byte	124
+	.byte	7                               # DW_AT_name
+	.byte	1                               # DW_AT_decl_file
+	.byte	3                               # DW_AT_decl_line
+	.long	.debug_info+174                 # DW_AT_type
+	.byte	6                               # Abbrev [6] 0x53:0x16 DW_TAG_inlined_subroutine
+	.long	.debug_info+156                 # DW_AT_abstract_origin
+	.byte	1                               # DW_AT_low_pc
+	.long	.Ltmp2-.Ltmp1                   # DW_AT_high_pc
+	.byte	1                               # DW_AT_call_file
+	.byte	4                               # DW_AT_call_line
+	.byte	32                              # DW_AT_call_column
+	.byte	7                               # Abbrev [7] 0x60:0x8 DW_TAG_formal_parameter
+	.byte	2                               # DW_AT_location
+	.byte	145
+	.byte	120
+	.long	.debug_info+165                 # DW_AT_abstract_origin
+	.byte	0                               # End Of Children Mark
+	.byte	0                               # End Of Children Mark
+	.byte	8                               # Abbrev [8] 0x6a:0x5 DW_TAG_pointer_type
+	.long	111                             # DW_AT_type
+	.byte	8                               # Abbrev [8] 0x6f:0x5 DW_TAG_pointer_type
+	.long	116                             # DW_AT_type
+	.byte	9                               # Abbrev [9] 0x74:0x4 DW_TAG_base_type
+	.byte	11                              # DW_AT_name
+	.byte	6                               # DW_AT_encoding
+	.byte	1                               # DW_AT_byte_size
+	.byte	0                               # End Of Children Mark
+.Ldebug_info_end0:
+.Lcu_begin1:
+	.long	.Ldebug_info_end1-.Ldebug_info_start1 # Length of Unit
+.Ldebug_info_start1:
+	.short	5                               # DWARF version number
+	.byte	1                               # DWARF Unit Type
+	.byte	8                               # Address Size (in bytes)
+	.long	.debug_abbrev                   # Offset Into Abbrev. Section
+	.byte	1                               # Abbrev [1] 0xc:0x43 DW_TAG_compile_unit
+	.byte	0                               # DW_AT_producer
+	.short	33                              # DW_AT_language
+	.byte	3                               # DW_AT_name
+	.long	.Lstr_offsets_base0             # DW_AT_str_offsets_base
+	.long	.Lline_table_start0             # DW_AT_stmt_list
+	.byte	2                               # DW_AT_comp_dir
+	.byte	2                               # DW_AT_low_pc
+	.long	.Lfunc_end1-.Lfunc_begin1       # DW_AT_high_pc
+	.long	.Laddr_table_base0              # DW_AT_addr_base
+	.byte	10                              # Abbrev [10] 0x23:0x12 DW_TAG_subprogram
+	.byte	4                               # DW_AT_linkage_name
+	.byte	5                               # DW_AT_name
+	.byte	2                               # DW_AT_decl_file
+	.byte	1                               # DW_AT_decl_line
+	.long	53                              # DW_AT_type
+                                        # DW_AT_external
+                                        # DW_AT_inline
+	.byte	11                              # Abbrev [11] 0x2c:0x8 DW_TAG_formal_parameter
+	.byte	7                               # DW_AT_name
+	.byte	2                               # DW_AT_decl_file
+	.byte	1                               # DW_AT_decl_line
+	.long	53                              # DW_AT_type
+	.byte	0                               # End Of Children Mark
+	.byte	9                               # Abbrev [9] 0x35:0x4 DW_TAG_base_type
+	.byte	6                               # DW_AT_name
+	.byte	5                               # DW_AT_encoding
+	.byte	4                               # DW_AT_byte_size
+	.byte	12                              # Abbrev [12] 0x39:0x15 DW_TAG_subprogram
+	.byte	2                               # DW_AT_low_pc
+	.long	.Lfunc_end1-.Lfunc_begin1       # DW_AT_high_pc
+	.byte	1                               # DW_AT_frame_base
+	.byte	86
+	.long	35                              # DW_AT_abstract_origin
+	.byte	13                              # Abbrev [13] 0x45:0x8 DW_TAG_formal_parameter
+	.byte	2                               # DW_AT_location
+	.byte	145
+	.byte	124
+	.long	44                              # DW_AT_abstract_origin
+	.byte	0                               # End Of Children Mark
+	.byte	0                               # End Of Children Mark
+.Ldebug_info_end1:
+	.section	.debug_str_offsets,"",@progbits
+	.long	52                              # Length of String Offsets Set
+	.short	5
+	.short	0
+.Lstr_offsets_base0:
+	.section	.debug_str,"MS",@progbits,1
+.Linfo_string0:
+	.asciz	"clang version 19.0.0git"       # string offset=0
+.Linfo_string1:
+	.asciz	"main.cpp"                      # string offset=24
+.Linfo_string2:
+	.asciz	"/home/ayermolo/local/tasks/T182867349" # string offset=33
+.Linfo_string3:
+	.asciz	"helper.cpp"                    # string offset=71
+.Linfo_string4:
+	.asciz	"_Z3fooi"                       # string offset=82
+.Linfo_string5:
+	.asciz	"foo"                           # string offset=90
+.Linfo_string6:
+	.asciz	"int"                           # string offset=94
+.Linfo_string7:
+	.asciz	"i"                             # string offset=98
+.Linfo_string8:
+	.asciz	"main"                          # string offset=100
+.Linfo_string9:
+	.asciz	"argc"                          # string offset=105
+.Linfo_string10:
+	.asciz	"argv"                          # string offset=110
+.Linfo_string11:
+	.asciz	"char"                          # string offset=115
+	.section	.debug_str_offsets,"",@progbits
+	.long	.Linfo_string0
+	.long	.Linfo_string1
+	.long	.Linfo_string2
+	.long	.Linfo_string3
+	.long	.Linfo_string4
+	.long	.Linfo_string5
+	.long	.Linfo_string6
+	.long	.Linfo_string7
+	.long	.Linfo_string8
+	.long	.Linfo_string9
+	.long	.Linfo_string10
+	.long	.Linfo_string11
+	.section	.debug_addr,"",@progbits
+	.long	.Ldebug_addr_end0-.Ldebug_addr_start0 # Length of contribution
+.Ldebug_addr_start0:
+	.short	5                               # DWARF version number
+	.byte	8                               # Address size
+	.byte	0                               # Segment selector size
+.Laddr_table_base0:
+	.quad	.Lfunc_begin0
+	.quad	.Ltmp1
+	.quad	.Lfunc_begin1
+.Ldebug_addr_end0:
+	.section	.debug_names,"",@progbits
+	.long	.Lnames_end0-.Lnames_start0     # Header: unit length
+.Lnames_start0:
+	.short	5                               # Header: version
+	.short	0                               # Header: padding
+	.long	2                               # Header: compilation unit count
+	.long	0                               # Header: local type unit count
+	.long	0                               # Header: foreign type unit count
+	.long	5                               # Header: bucket count
+	.long	5                               # Header: name count
+	.long	.Lnames_abbrev_end0-.Lnames_abbrev_start0 # Header: abbreviation table size
+	.long	8                               # Header: augmentation string size
+	.ascii	"LLVM0700"                      # Header: augmentation string
+	.long	.Lcu_begin0                     # Compilation unit 0
+	.long	.Lcu_begin1                     # Compilation unit 1
+	.long	0                               # Bucket 0
+	.long	1                               # Bucket 1
+	.long	0                               # Bucket 2
+	.long	3                               # Bucket 3
+	.long	4                               # Bucket 4
+	.long	2090499946                      # Hash in Bucket 1
+	.long	-1257882370                     # Hash in Bucket 1
+	.long	193495088                       # Hash in Bucket 3
+	.long	193491849                       # Hash in Bucket 4
+	.long	2090147939                      # Hash in Bucket 4
+	.long	.Linfo_string8                  # String in Bucket 1: main
+	.long	.Linfo_string4                  # String in Bucket 1: _Z3fooi
+	.long	.Linfo_string6                  # String in Bucket 3: int
+	.long	.Linfo_string5                  # String in Bucket 4: foo
+	.long	.Linfo_string11                 # String in Bucket 4: char
+	.long	.Lnames1-.Lnames_entries0       # Offset in Bucket 1
+	.long	.Lnames3-.Lnames_entries0       # Offset in Bucket 1
+	.long	.Lnames0-.Lnames_entries0       # Offset in Bucket 3
+	.long	.Lnames2-.Lnames_entries0       # Offset in Bucket 4
+	.long	.Lnames4-.Lnames_entries0       # Offset in Bucket 4
+.Lnames_abbrev_start0:
+	.byte	1                               # Abbrev code
+	.byte	46                              # DW_TAG_subprogram
+	.byte	1                               # DW_IDX_compile_unit
+	.byte	11                              # DW_FORM_data1
+	.byte	3                               # DW_IDX_die_offset
+	.byte	19                              # DW_FORM_ref4
+	.byte	4                               # DW_IDX_parent
+	.byte	25                              # DW_FORM_flag_present
+	.byte	0                               # End of abbrev
+	.byte	0                               # End of abbrev
+	.byte	2                               # Abbrev code
+	.byte	29                              # DW_TAG_inlined_subroutine
+	.byte	1                               # DW_IDX_compile_unit
+	.byte	11                              # DW_FORM_data1
+	.byte	3                               # DW_IDX_die_offset
+	.byte	19                              # DW_FORM_ref4
+	.byte	4                               # DW_IDX_parent
+	.byte	19                              # DW_FORM_ref4
+	.byte	0                               # End of abbrev
+	.byte	0                               # End of abbrev
+	.byte	3                               # Abbrev code
+	.byte	36                              # DW_TAG_base_type
+	.byte	1                               # DW_IDX_compile_unit
+	.byte	11                              # DW_FORM_data1
+	.byte	3                               # DW_IDX_die_offset
+	.byte	19                              # DW_FORM_ref4
+	.byte	4                               # DW_IDX_parent
+	.byte	25                              # DW_FORM_flag_present
+	.byte	0                               # End of abbrev
+	.byte	0                               # End of abbrev
+	.byte	0                               # End of abbrev list
+.Lnames_abbrev_end0:
+.Lnames_entries0:
+.Lnames1:
+.L3:
+	.byte	1                               # Abbreviation code
+	.byte	0                               # DW_IDX_compile_unit
+	.long	35                              # DW_IDX_die_offset
+	.byte	0                               # DW_IDX_parent
+                                        # End of list: main
+.Lnames3:
+.L0:
+	.byte	1                               # Abbreviation code
+	.byte	1                               # DW_IDX_compile_unit
+	.long	57                              # DW_IDX_die_offset
+.L2:                                    # DW_IDX_parent
+	.byte	2                               # Abbreviation code
+	.byte	0                               # DW_IDX_compile_unit
+	.long	83                              # DW_IDX_die_offset
+	.long	.L3-.Lnames_entries0            # DW_IDX_parent
+	.byte	0                               # End of list: _Z3fooi
+.Lnames0:
+.L4:
+	.byte	3                               # Abbreviation code
+	.byte	1                               # DW_IDX_compile_unit
+	.long	53                              # DW_IDX_die_offset
+	.byte	0                               # DW_IDX_parent
+                                        # End of list: int
+.Lnames2:
+	.byte	1                               # Abbreviation code
+	.byte	1                               # DW_IDX_compile_unit
+	.long	57                              # DW_IDX_die_offset
+	.byte	2                               # DW_IDX_parent
+                                        # Abbreviation code
+	.byte	0                               # DW_IDX_compile_unit
+	.long	83                              # DW_IDX_die_offset
+	.long	.L3-.Lnames_entries0            # DW_IDX_parent
+	.byte	0                               # End of list: foo
+.Lnames4:
+.L1:
+	.byte	3                               # Abbreviation code
+	.byte	0                               # DW_IDX_compile_unit
+	.long	116                             # DW_IDX_die_offset
+	.byte	0                               # DW_IDX_parent
+                                        # End of list: char
+	.p2align	2, 0x0
+.Lnames_end0:
+	.ident	"clang version 19.0.0git"
+	.ident	"clang version 19.0.0git"
+	.section	".note.GNU-stack","",@progbits
+	.section	.debug_line,"",@progbits
+.Lline_table_start0:
-- 
GitLab


From dcbddc25250158469c5635ad2ae4095faef53dfd Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Fri, 22 Mar 2024 16:13:58 -0500
Subject: [PATCH 008/404] [Libomptarget] Unify and simplify plugin CMake
 (#86191)

Summary:
This patch reworks the CMake handling for building plugins. All this
does is pull a lot of shared and common logic into a single helper
function.
This also simplifies the OMPT libraries from being built separately
instead of just added.
---
 .../plugins-nextgen/CMakeLists.txt            | 59 ++++++++++++++
 .../plugins-nextgen/amdgpu/CMakeLists.txt     | 80 +++----------------
 .../plugins-nextgen/common/CMakeLists.txt     | 54 ++-----------
 .../common/OMPT/CMakeLists.txt                | 70 ----------------
 .../plugins-nextgen/cuda/CMakeLists.txt       | 40 ++--------
 .../plugins-nextgen/host/CMakeLists.txt       | 29 +------
 6 files changed, 87 insertions(+), 245 deletions(-)
 delete mode 100644 openmp/libomptarget/plugins-nextgen/common/OMPT/CMakeLists.txt

diff --git a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt
index 75540f055844..8e8b040fd562 100644
--- a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt
+++ b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt
@@ -10,7 +10,66 @@
 #
 ##===----------------------------------------------------------------------===##
 
+# Common interface to handle creating a plugin library.
+set(common_dir ${CMAKE_CURRENT_SOURCE_DIR}/common)
 add_subdirectory(common)
+function(add_target_library target_name lib_name)
+  llvm_map_components_to_libnames(llvm_libs
+    ${LLVM_TARGETS_TO_BUILD}
+    AggressiveInstCombine
+    Analysis
+    BinaryFormat
+    BitReader
+    BitWriter
+    CodeGen
+    Core
+    Extensions
+    InstCombine
+    Instrumentation
+    IPO
+    IRReader
+    Linker
+    MC
+    Object
+    Passes
+    Remarks
+    ScalarOpts
+    Support
+    Target
+    TargetParser
+    TransformUtils
+    Vectorize
+  )
+
+  add_llvm_library(${target_name} SHARED
+    NO_INSTALL_RPATH
+    BUILDTREE_ONLY
+  )
+
+  llvm_update_compile_flags(${target_name})
+  target_link_libraries(${target_name} PUBLIC 
+                        PluginCommon ${llvm_libs} ${OPENMP_PTHREAD_LIB})
+
+  target_compile_definitions(${target_name} PRIVATE TARGET_NAME=${lib_name})
+  target_compile_definitions(${target_name} PRIVATE 
+                             DEBUG_PREFIX="TARGET ${lib_name} RTL")
+
+  if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
+    # On FreeBSD, the 'environ' symbol is undefined at link time, but resolved by
+    # the dynamic linker at runtime. Therefore, allow the symbol to be undefined
+    # when creating a shared library.
+    target_link_libraries(${target_name} PRIVATE "-Wl,--allow-shlib-undefined")
+  else()
+    target_link_libraries(${target_name} PRIVATE "-Wl,-z,defs")
+  endif()
+
+  if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
+    target_link_libraries(${target_name} PRIVATE
+    "-Wl,--version-script=${common_dir}/../exports")
+  endif()
+  set_target_properties(${target_name} PROPERTIES CXX_VISIBILITY_PRESET protected)
+endfunction()
+
 add_subdirectory(amdgpu)
 add_subdirectory(cuda)
 add_subdirectory(host)
diff --git a/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt
index 8fbfe4d9b13f..40df77102c78 100644
--- a/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt
+++ b/openmp/libomptarget/plugins-nextgen/amdgpu/CMakeLists.txt
@@ -27,76 +27,23 @@ if(NOT (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(ppc64le)|(aarch64)$" AND CMAKE
   return()
 endif()
 
-################################################################################
-# Define the suffix for the runtime messaging dumps.
-add_definitions(-DTARGET_NAME=AMDGPU)
-
-# Define debug prefix. TODO: This should be automatized in the Debug.h but it
-# requires changing the original plugins.
-add_definitions(-DDEBUG_PREFIX="TARGET AMDGPU RTL")
+# Create the library and add the default arguments.
+add_target_library(omptarget.rtl.amdgpu AMDGPU)
 
-set(LIBOMPTARGET_DLOPEN_LIBHSA OFF)
-option(LIBOMPTARGET_FORCE_DLOPEN_LIBHSA "Build with dlopened libhsa" ${LIBOMPTARGET_DLOPEN_LIBHSA})
-
-if (${hsa-runtime64_FOUND} AND NOT LIBOMPTARGET_FORCE_DLOPEN_LIBHSA)
-  libomptarget_say("Building AMDGPU NextGen plugin linked against libhsa")
-  set(LIBOMPTARGET_EXTRA_SOURCE)
-  set(LIBOMPTARGET_DEP_LIBRARIES hsa-runtime64::hsa-runtime64)
-else()
-  libomptarget_say("Building AMDGPU NextGen plugin for dlopened libhsa")
-  include_directories(dynamic_hsa)
-  set(LIBOMPTARGET_EXTRA_SOURCE dynamic_hsa/hsa.cpp)
-  set(LIBOMPTARGET_DEP_LIBRARIES)
-endif()
+target_sources(omptarget.rtl.amdgpu PRIVATE src/rtl.cpp)
+target_include_directories(omptarget.rtl.amdgpu PRIVATE
+                           ${CMAKE_CURRENT_SOURCE_DIR}/utils)
 
-if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
-  # On FreeBSD, the 'environ' symbol is undefined at link time, but resolved by
-  # the dynamic linker at runtime. Therefore, allow the symbol to be undefined
-  # when creating a shared library.
-  set(LDFLAGS_UNDEFINED "-Wl,--allow-shlib-undefined")
+option(LIBOMPTARGET_FORCE_DLOPEN_LIBHSA "Build with dlopened libhsa" OFF)
+if(hsa-runtime64_FOUND AND NOT LIBOMPTARGET_FORCE_DLOPEN_LIBHSA)
+  libomptarget_say("Building AMDGPU plugin linked against libhsa")
+  target_link_libraries(omptarget.rtl.amdgpu PRIVATE hsa-runtime64::hsa-runtime64)
 else()
-  set(LDFLAGS_UNDEFINED "-Wl,-z,defs")
+  libomptarget_say("Building AMDGPU plugin for dlopened libhsa")
+  target_include_directories(omptarget.rtl.amdgpu PRIVATE dynamic_hsa)
+  target_sources(omptarget.rtl.amdgpu PRIVATE dynamic_hsa/hsa.cpp)
 endif()
 
-add_llvm_library(omptarget.rtl.amdgpu SHARED
-  src/rtl.cpp
-  ${LIBOMPTARGET_EXTRA_SOURCE}
-
-  ADDITIONAL_HEADER_DIRS
-  ${LIBOMPTARGET_INCLUDE_DIR}
-  ${CMAKE_CURRENT_SOURCE_DIR}/utils
-
-  LINK_COMPONENTS
-  Support
-  Object
-
-  LINK_LIBS
-  PRIVATE
-  PluginCommon
-  ${LIBOMPTARGET_DEP_LIBRARIES}
-  ${OPENMP_PTHREAD_LIB}
-  ${LDFLAGS_UNDEFINED}
-
-  NO_INSTALL_RPATH
-  BUILDTREE_ONLY
-)
-
-if ((OMPT_TARGET_DEFAULT) AND (LIBOMPTARGET_OMPT_SUPPORT))
-  target_link_libraries(omptarget.rtl.amdgpu PRIVATE OMPT)
-endif()
-
-if (LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
-  target_link_libraries(omptarget.rtl.amdgpu PRIVATE
-    "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports")
-endif()
-
-target_include_directories(
-  omptarget.rtl.amdgpu
-  PRIVATE
-  ${LIBOMPTARGET_INCLUDE_DIR}
-  ${CMAKE_CURRENT_SOURCE_DIR}/utils
-)
-
 # Configure testing for the AMDGPU plugin. We will build tests if we could a
 # functional AMD GPU on the system, or if manually specifies by the user.
 option(LIBOMPTARGET_FORCE_AMDGPU_TESTS "Build AMDGPU libomptarget tests" OFF)
@@ -114,5 +61,4 @@ endif()
 # Install plugin under the lib destination folder.
 install(TARGETS omptarget.rtl.amdgpu LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}")
 set_target_properties(omptarget.rtl.amdgpu PROPERTIES
-  INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/.."
-  CXX_VISIBILITY_PRESET protected)
+  INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
diff --git a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt
index 085d44307165..0420d0e6f1f8 100644
--- a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt
+++ b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt
@@ -26,45 +26,6 @@ foreach(Target ${TargetsSupported})
 	target_compile_definitions(PluginCommon PRIVATE "LIBOMPTARGET_JIT_${Target}")
 endforeach()
 
-# This is required when using LLVM libraries.
-llvm_update_compile_flags(PluginCommon)
-
-if (LLVM_LINK_LLVM_DYLIB)
-  set(llvm_libs LLVM)
-else()
-  llvm_map_components_to_libnames(llvm_libs
-    ${LLVM_TARGETS_TO_BUILD}
-    AggressiveInstCombine
-    Analysis
-    BinaryFormat
-    BitReader
-    BitWriter
-    CodeGen
-    Core
-    Extensions
-    InstCombine
-    Instrumentation
-    IPO
-    IRReader
-    Linker
-    MC
-    Object
-    Passes
-    Remarks
-    ScalarOpts
-    Support
-    Target
-    TargetParser
-    TransformUtils
-    Vectorize
-  )
-endif()
-
-target_link_libraries(PluginCommon
-  PUBLIC
-    ${llvm_libs}
-)
-
 # Include the RPC server from the `libc` project if availible.
 if(TARGET llvmlibc_rpc_server AND ${LIBOMPTARGET_GPU_LIBC_SUPPORT})
 	target_link_libraries(PluginCommon PRIVATE llvmlibc_rpc_server)
@@ -82,8 +43,10 @@ elseif(${LIBOMPTARGET_GPU_LIBC_SUPPORT})
   endif()
 endif()
 
-if ((OMPT_TARGET_DEFAULT) AND (LIBOMPTARGET_OMPT_SUPPORT))
-	target_link_libraries(PluginCommon PUBLIC OMPT)
+# If we have OMPT enabled include it in the list of sources.
+if (OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT)
+  target_sources(PluginCommon PRIVATE OMPT/OmptCallback.cpp)
+  target_include_directories(PluginCommon PRIVATE OMPT)
 endif()
 
 # Define the TARGET_NAME and DEBUG_PREFIX.
@@ -95,16 +58,11 @@ target_compile_definitions(PluginCommon PRIVATE
 target_compile_options(PluginCommon PUBLIC ${offload_compile_flags})
 target_link_options(PluginCommon PUBLIC ${offload_link_flags})
 
-target_include_directories(PluginCommon
-  PRIVATE
-  ${LIBOMPTARGET_INCLUDE_DIR}
-  PUBLIC
+target_include_directories(PluginCommon PUBLIC 
   ${CMAKE_CURRENT_SOURCE_DIR}/include
+  ${LIBOMPTARGET_INCLUDE_DIR}
 )
 
 set_target_properties(PluginCommon PROPERTIES
   POSITION_INDEPENDENT_CODE ON
   CXX_VISIBILITY_PRESET protected)
-
-add_subdirectory(OMPT)
-
diff --git a/openmp/libomptarget/plugins-nextgen/common/OMPT/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/common/OMPT/CMakeLists.txt
deleted file mode 100644
index be4c743665b3..000000000000
--- a/openmp/libomptarget/plugins-nextgen/common/OMPT/CMakeLists.txt
+++ /dev/null
@@ -1,70 +0,0 @@
-##===----------------------------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-##===----------------------------------------------------------------------===##
-#
-# Aggregation of parts which can be used by OpenMP tools
-#
-##===----------------------------------------------------------------------===##
-
-# NOTE: Don't try to build `OMPT` using `add_llvm_library` because we
-# don't want to export `OMPT` while `add_llvm_library` requires that.
-add_library(OMPT OBJECT
-  OmptCallback.cpp)
-
-# This is required when using LLVM libraries.
-llvm_update_compile_flags(OMPT)
-
-if (LLVM_LINK_LLVM_DYLIB)
-  set(llvm_libs LLVM)
-else()
-  llvm_map_components_to_libnames(llvm_libs
-    ${LLVM_TARGETS_TO_BUILD}
-    AggressiveInstCombine
-    Analysis
-    BinaryFormat
-    BitReader
-    BitWriter
-    CodeGen
-    Core
-    Extensions
-    InstCombine
-    Instrumentation
-    IPO
-    IRReader
-    Linker
-    MC
-    Object
-    Passes
-    Remarks
-    ScalarOpts
-    Support
-    Target
-    TargetParser
-    TransformUtils
-    Vectorize
-  )
-endif()
-
-target_link_libraries(OMPT
-  PUBLIC
-    ${llvm_libs}
-)
-
-# Define the TARGET_NAME and DEBUG_PREFIX.
-target_compile_definitions(OMPT PRIVATE
-  TARGET_NAME="OMPT"
-  DEBUG_PREFIX="OMPT"
-)
-
-target_include_directories(OMPT
-  INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}
-  PRIVATE ${LIBOMPTARGET_INCLUDE_DIR}
-)
-
-set_target_properties(OMPT PROPERTIES
-  POSITION_INDEPENDENT_CODE ON
-  CXX_VISIBILITY_PRESET protected)
diff --git a/openmp/libomptarget/plugins-nextgen/cuda/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/cuda/CMakeLists.txt
index 2bfb47168a7f..b3530462aa19 100644
--- a/openmp/libomptarget/plugins-nextgen/cuda/CMakeLists.txt
+++ b/openmp/libomptarget/plugins-nextgen/cuda/CMakeLists.txt
@@ -23,34 +23,12 @@ endif()
 
 libomptarget_say("Building CUDA NextGen offloading plugin.")
 
-set(LIBOMPTARGET_DLOPEN_LIBCUDA OFF)
-option(LIBOMPTARGET_FORCE_DLOPEN_LIBCUDA "Build with dlopened libcuda" ${LIBOMPTARGET_DLOPEN_LIBCUDA})
-
-add_llvm_library(omptarget.rtl.cuda SHARED
-  src/rtl.cpp
-
-  LINK_COMPONENTS
-  Support
-  Object
-
-  LINK_LIBS PRIVATE
-  PluginCommon
-  ${OPENMP_PTHREAD_LIB}
-
-  NO_INSTALL_RPATH
-  BUILDTREE_ONLY
-)
-
-if ((OMPT_TARGET_DEFAULT) AND (LIBOMPTARGET_OMPT_SUPPORT))
-  target_link_libraries(omptarget.rtl.cuda PRIVATE OMPT)
-endif()
-
-if (LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
-  target_link_libraries(omptarget.rtl.cuda PRIVATE
-  "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports,-z,defs")
-endif()
+# Create the library and add the default arguments.
+add_target_library(omptarget.rtl.cuda CUDA)
 
+target_sources(omptarget.rtl.cuda PRIVATE src/rtl.cpp)
 
+option(LIBOMPTARGET_FORCE_DLOPEN_LIBCUDA "Build with dlopened libcuda" OFF)
 if(LIBOMPTARGET_DEP_CUDA_FOUND AND NOT LIBOMPTARGET_FORCE_DLOPEN_LIBCUDA)
   libomptarget_say("Building CUDA plugin linked against libcuda")
   target_link_libraries(omptarget.rtl.cuda PRIVATE CUDA::cuda_driver)
@@ -60,13 +38,6 @@ else()
   target_sources(omptarget.rtl.cuda PRIVATE dynamic_cuda/cuda.cpp)
 endif()
 
-# Define debug prefix. TODO: This should be automatized in the Debug.h but it
-# requires changing the original plugins.
-target_compile_definitions(omptarget.rtl.cuda PRIVATE TARGET_NAME="CUDA")
-target_compile_definitions(omptarget.rtl.cuda PRIVATE DEBUG_PREFIX="TARGET CUDA RTL")
-
-target_include_directories(omptarget.rtl.cuda PRIVATE ${LIBOMPTARGET_INCLUDE_DIR})
-
 # Configure testing for the CUDA plugin. We will build tests if we could a
 # functional NVIDIA GPU on the system, or if manually specifies by the user.
 option(LIBOMPTARGET_FORCE_NVIDIA_TESTS "Build NVIDIA libomptarget tests" OFF)
@@ -84,5 +55,4 @@ endif()
 # Install plugin under the lib destination folder.
 install(TARGETS omptarget.rtl.cuda LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}")
 set_target_properties(omptarget.rtl.cuda PROPERTIES
-  INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/.."
-  CXX_VISIBILITY_PRESET protected)
+  INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/..")
diff --git a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt
index 58a79898ff80..d30680e10431 100644
--- a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt
+++ b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt
@@ -2,11 +2,6 @@ if(NOT CMAKE_SYSTEM_NAME MATCHES "Linux")
   return()
 endif()
 
-  # build_generic_elf64("s390x" "S390X" "s390x" "systemz" "s390x-ibm-linux-gnu" "22")
-  # build_generic_elf64("aarch64" "aarch64" "aarch64" "aarch64" "aarch64-unknown-linux-gnu" "183")
-  # build_generic_elf64("ppc64" "PPC64" "ppc64" "ppc64" "powerpc64-ibm-linux-gnu" "21")
-  # build_generic_elf64("x86_64" "x86_64" "x86_64" "x86_64" "x86_64-pc-linux-gnu" "62")
-  # build_generic_elf64("ppc64le" "PPC64le" "ppc64" "ppc64le" "powerpc64le-ibm-linux-gnu" "21")
 set(supported_targets x86_64 aarch64 ppc64 ppc64le s390x)
 if(NOT ${CMAKE_SYSTEM_PROCESSOR} IN_LIST supported_targets)
   libomptarget_say("Not building ${machine} NextGen offloading plugin")
@@ -18,16 +13,10 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le$")
   set(machine ppc64)
 endif()
 
-add_llvm_library(omptarget.rtl.${machine} SHARED
-  src/rtl.cpp
-  ADDITIONAL_HEADER_DIRS
-    ${LIBOMPTARGET_INCLUDE_DIR}
-  LINK_LIBS PRIVATE
-    PluginCommon
-    ${OPENMP_PTHREAD_LIB}
-  NO_INSTALL_RPATH
-  BUILDTREE_ONLY
-)
+# Create the library and add the default arguments.
+add_target_library(omptarget.rtl.${machine} ${machine})
+
+target_sources(omptarget.rtl.${machine} PRIVATE src/rtl.cpp)
 
 if(LIBOMPTARGET_DEP_LIBFFI_FOUND)
   libomptarget_say("Building ${machine} plugin linked with libffi")
@@ -42,10 +31,6 @@ else()
   target_include_directories(omptarget.rtl.${machine} PRIVATE dynamic_ffi)
 endif()
 
-if(OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT)
-  target_link_libraries(omptarget.rtl.${machine} PRIVATE OMPT)
-endif()
-
 if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
   target_link_libraries(omptarget.rtl.${machine} PRIVATE
     "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports")
@@ -70,12 +55,6 @@ else()
   libomptarget_say("Not generating ${tmachine_name} tests. LibFFI not found.")
 endif()
 
-# Define macro to be used as prefix of the runtime messages for this target.
-target_compile_definitions(omptarget.rtl.${machine} PRIVATE TARGET_NAME=${machine})
-# TODO: This should be automatized in Debug.h.
-target_compile_definitions(omptarget.rtl.${machine} PRIVATE
-                           DEBUG_PREFIX="TARGET ${machine} RTL")
-
 # Define the target specific triples and ELF machine values.
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le$" OR
    CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64$")
-- 
GitLab


From 215f105ca5d0b42d00bbbc315605b222d63be63a Mon Sep 17 00:00:00 2001
From: Florian Mayer <fmayer@google.com>
Date: Fri, 22 Mar 2024 14:14:43 -0700
Subject: [PATCH 009/404] [MTE] Fix test (#85875)

llc runs the stack tagging instrumentation, so if we run opt before, we
double instrument
---
 llvm/test/CodeGen/AArch64/stack-tagging-stack-coloring.ll | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-stack-coloring.ll b/llvm/test/CodeGen/AArch64/stack-tagging-stack-coloring.ll
index 81349620fb77..5d1c91e434dc 100644
--- a/llvm/test/CodeGen/AArch64/stack-tagging-stack-coloring.ll
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-stack-coloring.ll
@@ -1,17 +1,15 @@
 ; Test that storage for allocas with disjoint lifetimes is reused with stack
 ; tagging.
 
-; RUN: opt -S -aarch64-stack-tagging -stack-tagging-use-stack-safety=0 %s -o - | \
-; RUN:   llc --mattr=+mte -no-stack-coloring=false -o - | \
+; RUN: llc --mattr=+mte -no-stack-coloring=false -stack-tagging-use-stack-safety=0 -o - %s | \
 ; RUN:   FileCheck %s --check-prefix=COLOR
-; RUN: opt -S -aarch64-stack-tagging %s -stack-tagging-use-stack-safety=0 -o - | \
-; RUN:   llc --mattr=+mte -no-stack-coloring=true -o - | \
+; RUN: llc --mattr=+mte -no-stack-coloring=true -stack-tagging-use-stack-safety=0 -o - %s | \
 ; RUN:   FileCheck %s --check-prefix=NOCOLOR
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64"
 
-; COLOR: sub	sp, sp, #208
+; COLOR: sub	sp, sp, #192
 ; NOCOLOR: sub	sp, sp, #336
 
 define i32 @myCall_w2(i32 %in) sanitize_memtag {
-- 
GitLab


From 14be4930c10bcc6f6c0096097350cb3cafff9661 Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Fri, 22 Mar 2024 14:17:51 -0700
Subject: [PATCH 010/404] [bazel] Make compiler-rt analyze on macOS (#86001)

Previously the select above would fail for non-linux platforms if you
did a `bazel build @llvm-project//...`, now this target specifies that
it's only supported on the linux platform through bazel's
`target_compatible_with` feature. This makes all targets in the tree be
ignored when building on incompatible platforms (and fail if built
directly)
---
 utils/bazel/llvm-project-overlay/compiler-rt/BUILD.bazel | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/compiler-rt/BUILD.bazel b/utils/bazel/llvm-project-overlay/compiler-rt/BUILD.bazel
index 9bdd454e1e36..577e6c033b4e 100644
--- a/utils/bazel/llvm-project-overlay/compiler-rt/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/compiler-rt/BUILD.bazel
@@ -19,6 +19,10 @@ cc_library(
         ],
         # Will raise error unless supported platforms.
     }),
+    target_compatible_with = select({
+        "@platforms//os:linux": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
 )
 
 WIN32_ONLY_FILES = [
-- 
GitLab


From 85af772f3b4067fce703b33cee0e2cdafc74a6d6 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Fri, 22 Mar 2024 16:38:10 -0500
Subject: [PATCH 011/404] [Libomptarget][FIX] Fix unintentinally used PUBLIC
 interface

Summary:
This was supposed to be private and caused some issues with certain
configs.
---
 openmp/libomptarget/plugins-nextgen/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt
index 8e8b040fd562..c19fd80592d6 100644
--- a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt
+++ b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt
@@ -47,7 +47,7 @@ function(add_target_library target_name lib_name)
   )
 
   llvm_update_compile_flags(${target_name})
-  target_link_libraries(${target_name} PUBLIC 
+  target_link_libraries(${target_name} PRIVATE
                         PluginCommon ${llvm_libs} ${OPENMP_PTHREAD_LIB})
 
   target_compile_definitions(${target_name} PRIVATE TARGET_NAME=${lib_name})
-- 
GitLab


From 5d0d9eb52dbb3bcf6f500c7b18d58c8bdf6659ce Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellis.sparky.hoag@gmail.com>
Date: Fri, 22 Mar 2024 14:47:31 -0700
Subject: [PATCH 012/404] [NFC][BP] Remove unused parameter from function
 (#86333)

Remove the unused parameter `RecDepth` from `runIterations()`.
---
 llvm/include/llvm/Support/BalancedPartitioning.h | 5 ++---
 llvm/lib/Support/BalancedPartitioning.cpp        | 4 ++--
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/llvm/include/llvm/Support/BalancedPartitioning.h b/llvm/include/llvm/Support/BalancedPartitioning.h
index 9738e742f7f1..539d157343fb 100644
--- a/llvm/include/llvm/Support/BalancedPartitioning.h
+++ b/llvm/include/llvm/Support/BalancedPartitioning.h
@@ -142,9 +142,8 @@ private:
               std::optional<BPThreadPool> &TP) const;
 
   /// Run bisection iterations
-  void runIterations(const FunctionNodeRange Nodes, unsigned RecDepth,
-                     unsigned LeftBucket, unsigned RightBucket,
-                     std::mt19937 &RNG) const;
+  void runIterations(const FunctionNodeRange Nodes, unsigned LeftBucket,
+                     unsigned RightBucket, std::mt19937 &RNG) const;
 
   /// Run a bisection iteration to improve the optimization goal
   /// \returns the total number of moved FunctionNodes
diff --git a/llvm/lib/Support/BalancedPartitioning.cpp b/llvm/lib/Support/BalancedPartitioning.cpp
index f4254b50d26c..141f0034a23f 100644
--- a/llvm/lib/Support/BalancedPartitioning.cpp
+++ b/llvm/lib/Support/BalancedPartitioning.cpp
@@ -136,7 +136,7 @@ void BalancedPartitioning::bisect(const FunctionNodeRange Nodes,
   // Split into two and assign to the left and right buckets
   split(Nodes, LeftBucket);
 
-  runIterations(Nodes, RecDepth, LeftBucket, RightBucket, RNG);
+  runIterations(Nodes, LeftBucket, RightBucket, RNG);
 
   // Split nodes wrt the resulting buckets
   auto NodesMid =
@@ -163,7 +163,7 @@ void BalancedPartitioning::bisect(const FunctionNodeRange Nodes,
 }
 
 void BalancedPartitioning::runIterations(const FunctionNodeRange Nodes,
-                                         unsigned RecDepth, unsigned LeftBucket,
+                                         unsigned LeftBucket,
                                          unsigned RightBucket,
                                          std::mt19937 &RNG) const {
   unsigned NumNodes = std::distance(Nodes.begin(), Nodes.end());
-- 
GitLab


From 4652ec0e291ca4ba4ddef3fd59b202646e9a6694 Mon Sep 17 00:00:00 2001
From: Patrick O'Neill <102189596+patrick-rivos@users.noreply.github.com>
Date: Fri, 22 Mar 2024 14:52:27 -0700
Subject: [PATCH 013/404] [SLP] Delete vectorized users when tree contains an
 invalid cost (#86344)

---
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  2 +-
 .../RISCV/partial-vec-invalid-cost.ll         | 57 +++++++++++++++++++
 2 files changed, 58 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 0f7afa2fc25c..f98d15c285a6 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -15959,7 +15959,7 @@ public:
         LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost
                           << " for reduction\n");
         if (!Cost.isValid())
-          return nullptr;
+          break;
         if (Cost >= -SLPCostThreshold) {
           V.getORE()->emit([&]() {
             return OptimizationRemarkMissed(
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll
new file mode 100644
index 000000000000..31f16801b7a6
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=slp-vectorizer -S | FileCheck %s
+
+target triple = "riscv64-unknown-linux-gnu"
+
+define void @partial_vec_invalid_cost() #0 {
+; CHECK-LABEL: define void @partial_vec_invalid_cost(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LSHR_1:%.*]] = lshr i96 0, 0
+; CHECK-NEXT:    [[LSHR_2:%.*]] = lshr i96 0, 0
+; CHECK-NEXT:    [[TRUNC_I96_1:%.*]] = trunc i96 [[LSHR_1]] to i32
+; CHECK-NEXT:    [[TRUNC_I96_2:%.*]] = trunc i96 [[LSHR_2]] to i32
+; CHECK-NEXT:    [[TRUNC_I96_3:%.*]] = trunc i96 0 to i32
+; CHECK-NEXT:    [[TRUNC_I96_4:%.*]] = trunc i96 0 to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[OP_RDX:%.*]] = or i32 [[TMP0]], [[TRUNC_I96_2]]
+; CHECK-NEXT:    [[OP_RDX1:%.*]] = or i32 [[TRUNC_I96_1]], [[TRUNC_I96_3]]
+; CHECK-NEXT:    [[OP_RDX2:%.*]] = or i32 [[OP_RDX]], [[OP_RDX1]]
+; CHECK-NEXT:    [[OP_RDX3:%.*]] = or i32 [[OP_RDX2]], [[TRUNC_I96_4]]
+; CHECK-NEXT:    [[STORE_THIS:%.*]] = zext i32 [[OP_RDX3]] to i96
+; CHECK-NEXT:    store i96 [[STORE_THIS]], ptr null, align 16
+; CHECK-NEXT:    ret void
+;
+entry:
+
+  %lshr.1 = lshr i96 0, 0 ; These ops
+  %lshr.2 = lshr i96 0, 0 ; return an
+  %add.0 = add i96 0, 0   ; invalid
+  %add.1 = add i96 0, 0   ; vector cost.
+
+  %trunc.i96.1 = trunc i96 %lshr.1 to i32 ; These ops
+  %trunc.i96.2 = trunc i96 %lshr.2 to i32 ; return an
+  %trunc.i96.3 = trunc i96 %add.0 to i32  ; invalid
+  %trunc.i96.4 = trunc i96 %add.1 to i32  ; vector cost.
+
+  %or.0 = or i32 %trunc.i96.1, %trunc.i96.2
+  %or.1 = or i32 %or.0, %trunc.i96.3
+  %or.2 = or i32 %or.1, %trunc.i96.4
+
+  %zext.0 = zext i1 0 to i32 ; These
+  %zext.1 = zext i1 0 to i32 ; ops
+  %zext.2 = zext i1 0 to i32 ; are
+  %zext.3 = zext i1 0 to i32 ; vectorized
+
+  %or.3 = or i32 %or.2, %zext.0 ; users
+  %or.4 = or i32 %or.3, %zext.1 ; of
+  %or.5 = or i32 %or.4, %zext.2 ; vectorized
+  %or.6 = or i32 %or.5, %zext.3 ; ops
+
+  %store.this = zext i32 %or.6 to i96
+
+  store i96 %store.this, ptr null, align 16
+  ret void
+}
+
+attributes #0 = { "target-features"="+v" }
-- 
GitLab


From 913e29966bac5fec08998a1acc3e793f9b7bcc12 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Fri, 22 Mar 2024 15:11:26 -0700
Subject: [PATCH 014/404] [NFC][tsan] Use the result of placement new (#86341)

---
 compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp
index 77488f843285..06c34a2e4383 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp
@@ -200,9 +200,8 @@ void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
 }
 
 void ThreadContext::OnStarted(void *arg) {
-  thr = static_cast<ThreadState *>(arg);
   DPrintf("#%d: ThreadStart\n", tid);
-  new (thr) ThreadState(tid);
+  thr = new (arg) ThreadState(tid);
   if (common_flags()->detect_deadlocks)
     thr->dd_lt = ctx->dd->CreateLogicalThread(tid);
   thr->tctx = this;
-- 
GitLab


From 0ba678a53d3ef7d125f38720a59875035739dc9b Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Fri, 22 Mar 2024 15:12:09 -0700
Subject: [PATCH 015/404] [tsan] Set `thr->is_inited` after SlotAttachAndLock
 (#86342)

Almost NFC.

This is symmetrical to `ThreadFinish`, which
resets the slot after `is_inited`.
---
 compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp
index 06c34a2e4383..5316a7862e44 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp
@@ -160,6 +160,10 @@ void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
   }
   Free(thr->tctx->sync);
 
+#if !SANITIZER_GO
+  thr->is_inited = true;
+#endif
+
   uptr stk_addr = 0;
   uptr stk_size = 0;
   uptr tls_addr = 0;
@@ -205,9 +209,6 @@ void ThreadContext::OnStarted(void *arg) {
   if (common_flags()->detect_deadlocks)
     thr->dd_lt = ctx->dd->CreateLogicalThread(tid);
   thr->tctx = this;
-#if !SANITIZER_GO
-  thr->is_inited = true;
-#endif
 }
 
 void ThreadFinish(ThreadState *thr) {
-- 
GitLab


From 362d26366d0175f01ffb6085eb747a6e40f01147 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Fri, 22 Mar 2024 15:24:44 -0700
Subject: [PATCH 016/404] [tsan] Process SIGPROF as sync signal only if thread
 is alive (#86343)

Otherwise it may crash too early.

This is followup to #85188
---
 compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
index 2bebe651b994..810ce69663d0 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
@@ -2169,8 +2169,7 @@ static bool is_sync_signal(ThreadSignalContext *sctx, int sig,
     return false;
 #endif
   return sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || sig == SIGTRAP ||
-         sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS ||
-         sig == SIGPROF;
+         sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS;
 }
 
 void sighandler(int sig, __sanitizer_siginfo *info, void *ctx) {
@@ -2181,7 +2180,8 @@ void sighandler(int sig, __sanitizer_siginfo *info, void *ctx) {
     return;
   }
   // Don't mess with synchronous signals.
-  const bool sync = is_sync_signal(sctx, sig, info);
+  const bool sync = is_sync_signal(sctx, sig, info) ||
+                    (sig == SIGPROF && thr->is_inited && !thr->is_dead);
   if (sync ||
       // If we are in blocking function, we can safely process it now
       // (but check if we are in a recursive interceptor,
-- 
GitLab


From b1e97d60bd5b1d3f994345caa4012ea11c2a0f62 Mon Sep 17 00:00:00 2001
From: Yeoul Na <yeoul_na@apple.com>
Date: Sat, 23 Mar 2024 07:26:35 +0900
Subject: [PATCH 017/404] Unwrap CountAttributed for debug info (#86017)

Fix crash caused by 3eb9ff30959a670559bcba03d149d4c51bf7c9c9
---
 clang/lib/CodeGen/CGDebugInfo.cpp              |  3 +++
 .../test/CodeGen/attr-counted-by-debug-info.c  | 18 ++++++++++++++++++
 2 files changed, 21 insertions(+)
 create mode 100644 clang/test/CodeGen/attr-counted-by-debug-info.c

diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 07ecaa81c47d..7453ed14aef4 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -3463,6 +3463,9 @@ static QualType UnwrapTypeForDebugInfo(QualType T, const ASTContext &C) {
     case Type::BTFTagAttributed:
       T = cast<BTFTagAttributedType>(T)->getWrappedType();
       break;
+    case Type::CountAttributed:
+      T = cast<CountAttributedType>(T)->desugar();
+      break;
     case Type::Elaborated:
       T = cast<ElaboratedType>(T)->getNamedType();
       break;
diff --git a/clang/test/CodeGen/attr-counted-by-debug-info.c b/clang/test/CodeGen/attr-counted-by-debug-info.c
new file mode 100644
index 000000000000..a6c2b1382b79
--- /dev/null
+++ b/clang/test/CodeGen/attr-counted-by-debug-info.c
@@ -0,0 +1,18 @@
+// RUN: %clang -emit-llvm -DCOUNTED_BY -S -g %s -o - | FileCheck %s
+// RUN: %clang -emit-llvm -S -g %s -o - | FileCheck %s
+
+#ifdef COUNTED_BY
+#define __counted_by(member)    __attribute__((__counted_by__(member)))
+#else
+#define __counted_by(member)
+#endif
+
+struct {
+  int num_counters;
+  long value[] __counted_by(num_counters);
+} agent_send_response_port_num;
+
+// CHECK: !DICompositeType(tag: DW_TAG_array_type, baseType: ![[BT:.*]], elements: ![[ELEMENTS:.*]])
+// CHECK: ![[BT]] = !DIBasicType(name: "long", size: {{.*}}, encoding: DW_ATE_signed)
+// CHECK: ![[ELEMENTS]] = !{![[COUNT:.*]]}
+// CHECK: ![[COUNT]] = !DISubrange(count: -1)
\ No newline at end of file
-- 
GitLab


From 56197d732e5d5d158fce2f2dfddf3d0bf0d12525 Mon Sep 17 00:00:00 2001
From: Maksim Panchenko <maks@fb.com>
Date: Fri, 22 Mar 2024 15:28:54 -0700
Subject: [PATCH 018/404] [BOLT] Skip functions with unsupported Linux kernel
 features (#86345)

Do not overwrite functions with alternative and paravirtual instructions
until a proper update support is implemented.
---
 bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 47 +++++++++++++++++++++++-
 1 file changed, 46 insertions(+), 1 deletion(-)

diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
index 303e8b18fd32..42df96817275 100644
--- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
+++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
@@ -252,11 +252,17 @@ class LinuxKernelRewriter final : public MetadataRewriter {
 
   /// Paravirtual instruction patch sites.
   Error readParaInstructions();
+  Error rewriteParaInstructions();
 
   Error readBugTable();
 
-  /// Read alternative instruction info from .altinstructions.
+  /// Do no process functions containing instruction annotated with
+  /// \p Annotation.
+  void skipFunctionsWithAnnotation(StringRef Annotation) const;
+
+  /// Handle alternative instruction info from .altinstructions.
   Error readAltInstructions();
+  Error rewriteAltInstructions();
 
   /// Read .pci_fixup
   Error readPCIFixupTable();
@@ -318,6 +324,12 @@ public:
     if (Error E = rewriteExceptionTable())
       return E;
 
+    if (Error E = rewriteAltInstructions())
+      return E;
+
+    if (Error E = rewriteParaInstructions())
+      return E;
+
     if (Error E = rewriteORCTables())
       return E;
 
@@ -1126,6 +1138,31 @@ Error LinuxKernelRewriter::readParaInstructions() {
   return Error::success();
 }
 
+void LinuxKernelRewriter::skipFunctionsWithAnnotation(
+    StringRef Annotation) const {
+  for (BinaryFunction &BF : llvm::make_second_range(BC.getBinaryFunctions())) {
+    if (!BC.shouldEmit(BF))
+      continue;
+    for (const BinaryBasicBlock &BB : BF) {
+      const bool HasAnnotation = llvm::any_of(BB, [&](const MCInst &Inst) {
+        return BC.MIB->hasAnnotation(Inst, Annotation);
+      });
+      if (HasAnnotation) {
+        BF.setSimple(false);
+        break;
+      }
+    }
+  }
+}
+
+Error LinuxKernelRewriter::rewriteParaInstructions() {
+  // Disable output of functions with paravirtual instructions before the
+  // rewrite support is complete.
+  skipFunctionsWithAnnotation("ParaSite");
+
+  return Error::success();
+}
+
 /// Process __bug_table section.
 /// This section contains information useful for kernel debugging.
 /// Each entry in the section is a struct bug_entry that contains a pointer to
@@ -1305,6 +1342,14 @@ Error LinuxKernelRewriter::readAltInstructions() {
   return Error::success();
 }
 
+Error LinuxKernelRewriter::rewriteAltInstructions() {
+  // Disable output of functions with alt instructions before the rewrite
+  // support is complete.
+  skipFunctionsWithAnnotation("AltInst");
+
+  return Error::success();
+}
+
 /// When the Linux kernel needs to handle an error associated with a given PCI
 /// device, it uses a table stored in .pci_fixup section to locate a fixup code
 /// specific to the vendor and the problematic device. The section contains a
-- 
GitLab


From 51268a57fd4d7f67fe9fdb337f63ec390fa2379a Mon Sep 17 00:00:00 2001
From: Maksim Panchenko <maks@fb.com>
Date: Fri, 22 Mar 2024 15:29:26 -0700
Subject: [PATCH 019/404] [BOLT] Enable --keep-nops option for Linux kernel by
 default (#86349)

Preserve nop instructions in the Linux kernel since they could be used
for runtime patching.
---
 bolt/lib/Rewrite/BinaryPassManager.cpp |  2 +-
 bolt/lib/Rewrite/RewriteInstance.cpp   |  4 ++++
 bolt/test/X86/linux-alt-instruction.s  | 15 +++++++--------
 bolt/test/X86/linux-orc.s              |  4 ++--
 bolt/test/X86/linux-parainstructions.s |  2 +-
 5 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/bolt/lib/Rewrite/BinaryPassManager.cpp b/bolt/lib/Rewrite/BinaryPassManager.cpp
index 489b33fe1c7c..6c26bb795726 100644
--- a/bolt/lib/Rewrite/BinaryPassManager.cpp
+++ b/bolt/lib/Rewrite/BinaryPassManager.cpp
@@ -72,7 +72,7 @@ static cl::opt<bool> JTFootprintReductionFlag(
              "instructions at jump sites"),
     cl::cat(BoltOptCategory));
 
-static cl::opt<bool>
+cl::opt<bool>
     KeepNops("keep-nops",
              cl::desc("keep no-op instructions. By default they are removed."),
              cl::Hidden, cl::cat(BoltOptCategory));
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 03f4298e817d..2ead51ff6a12 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -81,6 +81,7 @@ extern cl::list<std::string> HotTextMoveSections;
 extern cl::opt<bool> Hugify;
 extern cl::opt<bool> Instrument;
 extern cl::opt<JumpTableSupportLevel> JumpTables;
+extern cl::opt<bool> KeepNops;
 extern cl::list<std::string> ReorderData;
 extern cl::opt<bolt::ReorderFunctions::ReorderType> ReorderFunctions;
 extern cl::opt<bool> TimeBuild;
@@ -2031,6 +2032,9 @@ void RewriteInstance::adjustCommandLineOptions() {
 
   if (opts::Lite)
     BC->outs() << "BOLT-INFO: enabling lite mode\n";
+
+  if (BC->IsLinuxKernel && !opts::KeepNops.getNumOccurrences())
+    opts::KeepNops = true;
 }
 
 namespace {
diff --git a/bolt/test/X86/linux-alt-instruction.s b/bolt/test/X86/linux-alt-instruction.s
index 5dcc6fe3ab0c..2cdf31519682 100644
--- a/bolt/test/X86/linux-alt-instruction.s
+++ b/bolt/test/X86/linux-alt-instruction.s
@@ -6,8 +6,8 @@
 # RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o
 # RUN: %clang %cflags -nostdlib %t.o -o %t.exe \
 # RUN:   -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie
-# RUN: llvm-bolt %t.exe --print-normalized --keep-nops -o %t.out \
-# RUN:   --alt-inst-feature-size=2 | FileCheck %s
+# RUN: llvm-bolt %t.exe --print-normalized --alt-inst-feature-size=2 -o %t.out \
+# RUN:   | FileCheck %s
 
 ## Older kernels used to have padlen field in alt_instr. Check compatibility.
 
@@ -15,8 +15,8 @@
 # RUN:   %s -o %t.o
 # RUN: %clang %cflags -nostdlib %t.o -o %t.exe \
 # RUN:   -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie
-# RUN: llvm-bolt %t.exe --print-normalized --keep-nops --alt-inst-has-padlen \
-# RUN:   -o %t.out | FileCheck %s
+# RUN: llvm-bolt %t.exe --print-normalized --alt-inst-has-padlen -o %t.out \
+# RUN:   | FileCheck %s
 
 ## Check with a larger size of "feature" field in alt_instr.
 
@@ -24,13 +24,12 @@
 # RUN:   --defsym FEATURE_SIZE_4=1 %s -o %t.o
 # RUN: %clang %cflags -nostdlib %t.o -o %t.exe \
 # RUN:   -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr,--no-pie
-# RUN: llvm-bolt %t.exe --print-normalized --keep-nops \
-# RUN:   --alt-inst-feature-size=4 -o %t.out | FileCheck %s
+# RUN: llvm-bolt %t.exe --print-normalized --alt-inst-feature-size=4 -o %t.out \
+# RUN:   | FileCheck %s
 
 ## Check that out-of-bounds read is handled properly.
 
-# RUN: not llvm-bolt %t.exe --print-normalized --keep-nops \
-# RUN:   --alt-inst-feature-size=2 -o %t.out
+# RUN: not llvm-bolt %t.exe --print-normalized --alt-inst-feature-size=2 -o %t.out
 
 # CHECK:      BOLT-INFO: Linux kernel binary detected
 # CHECK:      BOLT-INFO: parsed 2 alternative instruction entries
diff --git a/bolt/test/X86/linux-orc.s b/bolt/test/X86/linux-orc.s
index 4da19989408e..5f2096278e92 100644
--- a/bolt/test/X86/linux-orc.s
+++ b/bolt/test/X86/linux-orc.s
@@ -27,7 +27,7 @@
 ## Verify ORC bindings to instructions.
 
 # RUN: llvm-bolt %t.exe --print-normalized --dump-orc --print-orc -o %t.out \
-# RUN:   --bolt-info=0 |& FileCheck %s
+# RUN:   --keep-nops=0 --bolt-info=0 |& FileCheck %s
 
 
 ## Verify ORC bindings after rewrite.
@@ -37,7 +37,7 @@
 
 ## Verify ORC binding after rewrite when some of the functions are skipped.
 
-# RUN: llvm-bolt %t.exe -o %t.out --skip-funcs=bar --bolt-info=0
+# RUN: llvm-bolt %t.exe -o %t.out --skip-funcs=bar --bolt-info=0 --keep-nops=0
 # RUN: llvm-bolt %t.out -o %t.out.1 --print-normalized --print-orc \
 # RUN:   |& FileCheck %s
 
diff --git a/bolt/test/X86/linux-parainstructions.s b/bolt/test/X86/linux-parainstructions.s
index 4bdfde5fb7f2..07fca6bbedaf 100644
--- a/bolt/test/X86/linux-parainstructions.s
+++ b/bolt/test/X86/linux-parainstructions.s
@@ -8,7 +8,7 @@
 
 ## Verify paravirtual bindings to instructions.
 
-# RUN: llvm-bolt %t.exe --print-normalized -o %t.out | FileCheck %s
+# RUN: llvm-bolt %t.exe --print-normalized -o %t.out --keep-nops=0 | FileCheck %s
 
 # CHECK:      BOLT-INFO: Linux kernel binary detected
 # CHECK:      BOLT-INFO: parsed 2 paravirtual patch sites
-- 
GitLab


From 3bc71c2abfa00413fd15cf0e5c08af6ec0d4768b Mon Sep 17 00:00:00 2001
From: Usama Hameed <u_hameed@apple.com>
Date: Fri, 22 Mar 2024 15:29:36 -0700
Subject: [PATCH 020/404] Get the linker version and pass the it to compiler-rt
 tests on Darwin. (#86220)

The HOST_LINK_VERSION is a hardcoded string in Darwin clang that detects
the linker version at configure time. The driver uses this information
to build the correct set of arguments for the linker. This patch detects
the linker version again during compiler-rt configuration and passes it
to the tests. This allows a clang built on a machine with a new linker
to run compiler-rt tests on a machine with an old linker.

rdar://125198603
---
 clang/CMakeLists.txt                       | 16 ++--------------
 cmake/Modules/GetDarwinLinkerVersion.cmake | 19 +++++++++++++++++++
 compiler-rt/CMakeLists.txt                 | 10 ++++++++++
 compiler-rt/test/lit.common.cfg.py         |  4 ++++
 compiler-rt/test/lit.common.configured.in  |  1 +
 5 files changed, 36 insertions(+), 14 deletions(-)
 create mode 100644 cmake/Modules/GetDarwinLinkerVersion.cmake

diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index 761dab8c28c1..ee783d52e4a4 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -15,6 +15,7 @@ endif()
 
 # Must go below project(..)
 include(GNUInstallDirs)
+include(GetDarwinLinkerVersion)
 
 if(CLANG_BUILT_STANDALONE)
   set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to conform to")
@@ -346,20 +347,7 @@ endif ()
 # Determine HOST_LINK_VERSION on Darwin.
 set(HOST_LINK_VERSION)
 if (APPLE AND NOT CMAKE_LINKER MATCHES ".*lld.*")
-  set(LD_V_OUTPUT)
-  execute_process(
-    COMMAND sh -c "${CMAKE_LINKER} -v 2>&1 | head -1"
-    RESULT_VARIABLE HAD_ERROR
-    OUTPUT_VARIABLE LD_V_OUTPUT
-  )
-  if (HAD_ERROR)
-    message(FATAL_ERROR "${CMAKE_LINKER} failed with status ${HAD_ERROR}")
-  endif()
-  if ("${LD_V_OUTPUT}" MATCHES ".*ld64-([0-9.]+).*")
-    string(REGEX REPLACE ".*ld64-([0-9.]+).*" "\\1" HOST_LINK_VERSION ${LD_V_OUTPUT})
-  elseif ("${LD_V_OUTPUT}" MATCHES "[^0-9]*([0-9.]+).*")
-    string(REGEX REPLACE "[^0-9]*([0-9.]+).*" "\\1" HOST_LINK_VERSION ${LD_V_OUTPUT})
-  endif()
+  get_darwin_linker_version(HOST_LINK_VERSION)
   message(STATUS "Host linker version: ${HOST_LINK_VERSION}")
 endif()
 
diff --git a/cmake/Modules/GetDarwinLinkerVersion.cmake b/cmake/Modules/GetDarwinLinkerVersion.cmake
new file mode 100644
index 000000000000..c27e50128586
--- /dev/null
+++ b/cmake/Modules/GetDarwinLinkerVersion.cmake
@@ -0,0 +1,19 @@
+# Get the linker version on Darwin
+function(get_darwin_linker_version variable)
+  set(LINK_VERSION)
+  set(LD_V_OUTPUT)
+  execute_process(
+    COMMAND sh -c "${CMAKE_LINKER} -v 2>&1 | head -1"
+    RESULT_VARIABLE HAD_ERROR
+    OUTPUT_VARIABLE LD_V_OUTPUT
+    )
+  if (HAD_ERROR)
+    message(FATAL_ERROR "${CMAKE_LINKER} failed with status ${HAD_ERROR}")
+  endif()
+  if ("${LD_V_OUTPUT}" MATCHES ".*ld64-([0-9.]+).*")
+    string(REGEX REPLACE ".*ld64-([0-9.]+).*" "\\1" LINK_VERSION ${LD_V_OUTPUT})
+  elseif ("${LD_V_OUTPUT}" MATCHES "[^0-9]*([0-9.]+).*")
+    string(REGEX REPLACE "[^0-9]*([0-9.]+).*" "\\1" LINK_VERSION ${LD_V_OUTPUT})
+  endif()
+  set(${variable} ${LINK_VERSION} PARENT_SCOPE)
+endfunction()
diff --git a/compiler-rt/CMakeLists.txt b/compiler-rt/CMakeLists.txt
index d562a6206c00..f4e92f14db85 100644
--- a/compiler-rt/CMakeLists.txt
+++ b/compiler-rt/CMakeLists.txt
@@ -36,6 +36,7 @@ include(SetPlatformToolchainTools)
 include(base-config-ix)
 include(CompilerRTUtils)
 include(CMakeDependentOption)
+include(GetDarwinLinkerVersion)
 
 option(COMPILER_RT_BUILD_BUILTINS "Build builtins" ON)
 mark_as_advanced(COMPILER_RT_BUILD_BUILTINS)
@@ -444,6 +445,15 @@ else()
   set(SANITIZER_USE_SYMBOLS FALSE)
 endif()
 
+# Get the linker version while configuring compiler-rt and explicitly pass it
+# in cflags during testing. This fixes the compiler/linker version mismatch
+# issue when running a clang built with a newer Xcode in an older Xcode
+set(COMPILER_RT_DARWIN_LINKER_VERSION)
+if (APPLE AND NOT CMAKE_LINKER MATCHES ".*lld.*")
+  get_darwin_linker_version(COMPILER_RT_DARWIN_LINKER_VERSION)
+  message(STATUS "Host linker version: ${COMPILER_RT_DARWIN_LINKER_VERSION}")
+endif()
+
 # Build sanitizer runtimes with debug info.
 if(MSVC)
   # Use /Z7 instead of /Zi for the asan runtime. This avoids the LNK4099
diff --git a/compiler-rt/test/lit.common.cfg.py b/compiler-rt/test/lit.common.cfg.py
index bd9b926c1505..0ac20a9831d9 100644
--- a/compiler-rt/test/lit.common.cfg.py
+++ b/compiler-rt/test/lit.common.cfg.py
@@ -882,6 +882,10 @@ if config.use_lld and config.has_lld and not config.use_lto:
 elif config.use_lld and (not config.has_lld):
     config.unsupported = True
 
+if config.host_os == "Darwin":
+    if getattr(config, "darwin_linker_version", None):
+        extra_cflags += ["-mlinker-version=" + config.darwin_linker_version]
+
 # Append any extra flags passed in lit_config
 append_target_cflags = lit_config.params.get("append_target_cflags", None)
 if append_target_cflags:
diff --git a/compiler-rt/test/lit.common.configured.in b/compiler-rt/test/lit.common.configured.in
index db5d7c598b73..fff5dc6cc750 100644
--- a/compiler-rt/test/lit.common.configured.in
+++ b/compiler-rt/test/lit.common.configured.in
@@ -51,6 +51,7 @@ set_default("expensive_checks", @LLVM_ENABLE_EXPENSIVE_CHECKS_PYBOOL@)
 set_default("test_standalone_build_libs", @COMPILER_RT_TEST_STANDALONE_BUILD_LIBS_PYBOOL@)
 set_default("has_compiler_rt_libatomic", @COMPILER_RT_BUILD_STANDALONE_LIBATOMIC_PYBOOL@)
 set_default("aarch64_sme", @COMPILER_RT_HAS_AARCH64_SME_PYBOOL@)
+set_default("darwin_linker_version", "@COMPILER_RT_DARWIN_LINKER_VERSION@")
 # True iff the test suite supports ignoring the test compiler's runtime library path
 # and using `config.compiler_rt_libdir` instead. This only matters when the runtime
 # library paths differ.
-- 
GitLab


From 4406e4a8bd5acadd980d84356b36030cadf9a61d Mon Sep 17 00:00:00 2001
From: Muhammad Omair Javaid <omair.javaid@linaro.org>
Date: Sat, 23 Mar 2024 03:22:24 +0500
Subject: [PATCH 021/404] Revert "Missed a null-ptr check in previous PR for
 Debuginfod testing (#86292)"

This reverts commit b1575f9082071702bd6aaa2600ce9fe011a091e9.
---
 lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp
index a9956aa9075f..91b8b4a979e0 100644
--- a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp
+++ b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp
@@ -53,7 +53,7 @@ static bool IsDwpSymbolFile(const lldb::ModuleSP &module_sp,
   ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin(
       module_sp, &file_spec, 0, FileSystem::Instance().GetByteSize(file_spec),
       dwp_file_data_sp, dwp_file_data_offset);
-  if (!dwp_obj_file || !ObjectFileELF::classof(dwp_obj_file.get()))
+  if (!ObjectFileELF::classof(dwp_obj_file.get()))
     return false;
   // The presence of a debug_cu_index section is the key identifying feature of
   // a DWP file. Make sure we don't fill in the section list on dwp_obj_file
-- 
GitLab


From 7fc2fbb3f1961e0ad0722c2d749ddd6264195a1c Mon Sep 17 00:00:00 2001
From: Muhammad Omair Javaid <omair.javaid@linaro.org>
Date: Sat, 23 Mar 2024 03:22:33 +0500
Subject: [PATCH 022/404] Revert "DebugInfoD tests + fixing issues exposed by
 tests (#85693)"

This reverts commit 6d939a6ec69adf284cdbef2034b49fd02ba503fc.

This broke following LLDB bots:
https://lab.llvm.org/buildbot/#/builders/96/builds/54867
https://lab.llvm.org/buildbot/#/builders/17/builds/50824
---
 .../Python/lldbsuite/test/make/Makefile.rules |  33 +--
 .../SymbolFile/DWARF/SymbolFileDWARF.cpp      |  38 ++--
 .../Plugins/SymbolLocator/CMakeLists.txt      |   7 +-
 .../SymbolVendor/ELF/SymbolVendorELF.cpp      |  30 +--
 lldb/test/API/debuginfod/Normal/Makefile      |  25 ---
 .../API/debuginfod/Normal/TestDebuginfod.py   | 185 -----------------
 lldb/test/API/debuginfod/Normal/main.c        |   7 -
 lldb/test/API/debuginfod/SplitDWARF/Makefile  |  28 ---
 .../SplitDWARF/TestDebuginfodDWP.py           | 194 ------------------
 lldb/test/API/debuginfod/SplitDWARF/main.c    |   7 -
 10 files changed, 17 insertions(+), 537 deletions(-)
 delete mode 100644 lldb/test/API/debuginfod/Normal/Makefile
 delete mode 100644 lldb/test/API/debuginfod/Normal/TestDebuginfod.py
 delete mode 100644 lldb/test/API/debuginfod/Normal/main.c
 delete mode 100644 lldb/test/API/debuginfod/SplitDWARF/Makefile
 delete mode 100644 lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py
 delete mode 100644 lldb/test/API/debuginfod/SplitDWARF/main.c

diff --git a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
index 75efcde1f040..bfd249ccd43f 100644
--- a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
+++ b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
@@ -51,7 +51,7 @@ LLDB_BASE_DIR := $(THIS_FILE_DIR)/../../../../../
 #
 # GNUWin32 uname gives "windows32" or "server version windows32" while
 # some versions of MSYS uname return "MSYS_NT*", but most environments
-# standardize on "Windows_NT", so we'll make it consistent here.
+# standardize on "Windows_NT", so we'll make it consistent here. 
 # When running tests from Visual Studio, the environment variable isn't
 # inherited all the way down to the process spawned for make.
 #----------------------------------------------------------------------
@@ -210,12 +210,6 @@ else
 	ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES"
 		DSYM = $(EXE).debug
 	endif
-
-	ifeq "$(MAKE_DWP)" "YES"
-		MAKE_DWO := YES
-		DWP_NAME = $(EXE).dwp
-		DYLIB_DWP_NAME = $(DYLIB_NAME).dwp
-	endif
 endif
 
 LIMIT_DEBUG_INFO_FLAGS =
@@ -363,7 +357,6 @@ ifneq "$(OS)" "Darwin"
 
 	OBJCOPY ?= $(call replace_cc_with,objcopy)
 	ARCHIVER ?= $(call replace_cc_with,ar)
-	DWP ?= $(call replace_cc_with,dwp)
 	override AR = $(ARCHIVER)
 endif
 
@@ -534,10 +527,6 @@ ifneq "$(CXX)" ""
 	endif
 endif
 
-ifeq "$(GEN_GNU_BUILD_ID)" "YES"
-	LDFLAGS += -Wl,--build-id
-endif
-
 #----------------------------------------------------------------------
 # DYLIB_ONLY variable can be used to skip the building of a.out.
 # See the sections below regarding dSYM file as well as the building of
@@ -576,25 +565,11 @@ else
 endif
 else
 ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES"
-ifeq "$(SAVE_FULL_DEBUG_BINARY)" "YES"
-	cp "$(EXE)" "$(EXE).full"
-endif
 	$(OBJCOPY) --only-keep-debug "$(EXE)" "$(DSYM)"
 	$(OBJCOPY) --strip-debug --add-gnu-debuglink="$(DSYM)" "$(EXE)" "$(EXE)"
 endif
-ifeq "$(MAKE_DWP)" "YES"
-	$(DWP) -o "$(DWP_NAME)" $(DWOS)
-endif
 endif
 
-
-#----------------------------------------------------------------------
-# Support emitting the content of the GNU build-id into a file
-# This needs to be used in conjunction with GEN_GNU_BUILD_ID := YES
-#----------------------------------------------------------------------
-$(EXE).uuid : $(EXE)
-	$(OBJCOPY) --dump-section=.note.gnu.build-id=$@ $<
-
 #----------------------------------------------------------------------
 # Make the dylib
 #----------------------------------------------------------------------
@@ -635,15 +610,9 @@ endif
 else
 	$(LD) $(DYLIB_OBJECTS) $(LDFLAGS) -shared -o "$(DYLIB_FILENAME)"
 ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES"
-	ifeq "$(SAVE_FULL_DEBUG_BINARY)" "YES"
-	cp "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME).full"
-	endif
 	$(OBJCOPY) --only-keep-debug "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME).debug"
 	$(OBJCOPY) --strip-debug --add-gnu-debuglink="$(DYLIB_FILENAME).debug" "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME)"
 endif
-ifeq "$(MAKE_DWP)" "YES"
-	$(DWP) -o $(DYLIB_DWP_FILE) $(DYLIB_DWOS)
-endif
 endif
 
 #----------------------------------------------------------------------
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index 08ce7b82b0c1..5f67658f86ea 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -4377,38 +4377,26 @@ const std::shared_ptr<SymbolFileDWARFDwo> &SymbolFileDWARF::GetDwpSymbolFile() {
     FileSpecList search_paths = Target::GetDefaultDebugFileSearchPaths();
     ModuleSpec module_spec;
     module_spec.GetFileSpec() = m_objfile_sp->GetFileSpec();
-    FileSpec dwp_filespec;
     for (const auto &symfile : symfiles.files()) {
       module_spec.GetSymbolFileSpec() =
           FileSpec(symfile.GetPath() + ".dwp", symfile.GetPathStyle());
       LLDB_LOG(log, "Searching for DWP using: \"{0}\"",
                module_spec.GetSymbolFileSpec());
-      dwp_filespec =
+      FileSpec dwp_filespec =
           PluginManager::LocateExecutableSymbolFile(module_spec, search_paths);
       if (FileSystem::Instance().Exists(dwp_filespec)) {
-        break;
-      }
-    }
-    if (!FileSystem::Instance().Exists(dwp_filespec)) {
-      LLDB_LOG(log, "No DWP file found locally");
-      // Fill in the UUID for the module we're trying to match for, so we can
-      // find the correct DWP file, as the Debuginfod plugin uses *only* this
-      // data to correctly match the DWP file with the binary.
-      module_spec.GetUUID() = m_objfile_sp->GetUUID();
-      dwp_filespec =
-          PluginManager::LocateExecutableSymbolFile(module_spec, search_paths);
-    }
-    if (FileSystem::Instance().Exists(dwp_filespec)) {
-      LLDB_LOG(log, "Found DWP file: \"{0}\"", dwp_filespec);
-      DataBufferSP dwp_file_data_sp;
-      lldb::offset_t dwp_file_data_offset = 0;
-      ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin(
-          GetObjectFile()->GetModule(), &dwp_filespec, 0,
-          FileSystem::Instance().GetByteSize(dwp_filespec), dwp_file_data_sp,
-          dwp_file_data_offset);
-      if (dwp_obj_file) {
-        m_dwp_symfile = std::make_shared<SymbolFileDWARFDwo>(
-            *this, dwp_obj_file, DIERef::k_file_index_mask);
+        LLDB_LOG(log, "Found DWP file: \"{0}\"", dwp_filespec);
+        DataBufferSP dwp_file_data_sp;
+        lldb::offset_t dwp_file_data_offset = 0;
+        ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin(
+            GetObjectFile()->GetModule(), &dwp_filespec, 0,
+            FileSystem::Instance().GetByteSize(dwp_filespec), dwp_file_data_sp,
+            dwp_file_data_offset);
+        if (dwp_obj_file) {
+          m_dwp_symfile = std::make_shared<SymbolFileDWARFDwo>(
+              *this, dwp_obj_file, DIERef::k_file_index_mask);
+          break;
+        }
       }
     }
     if (!m_dwp_symfile) {
diff --git a/lldb/source/Plugins/SymbolLocator/CMakeLists.txt b/lldb/source/Plugins/SymbolLocator/CMakeLists.txt
index 3367022639ab..ca969626f4ff 100644
--- a/lldb/source/Plugins/SymbolLocator/CMakeLists.txt
+++ b/lldb/source/Plugins/SymbolLocator/CMakeLists.txt
@@ -1,10 +1,5 @@
-# Order matters here: the first symbol locator prevents further searching.
-# For DWARF binaries that are both stripped and split, the Default plugin
-# will return the stripped binary when asked for the ObjectFile, which then
-# prevents an unstripped binary from being requested from the Debuginfod
-# provider.
-add_subdirectory(Debuginfod)
 add_subdirectory(Default)
 if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
   add_subdirectory(DebugSymbols)
 endif()
+add_subdirectory(Debuginfod)
diff --git a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp
index 91b8b4a979e0..b5fe35d71032 100644
--- a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp
+++ b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp
@@ -44,25 +44,6 @@ llvm::StringRef SymbolVendorELF::GetPluginDescriptionStatic() {
          "executables.";
 }
 
-// If this is needed elsewhere, it can be exported/moved.
-static bool IsDwpSymbolFile(const lldb::ModuleSP &module_sp,
-                            const FileSpec &file_spec) {
-  DataBufferSP dwp_file_data_sp;
-  lldb::offset_t dwp_file_data_offset = 0;
-  // Try to create an ObjectFile from the file_spec.
-  ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin(
-      module_sp, &file_spec, 0, FileSystem::Instance().GetByteSize(file_spec),
-      dwp_file_data_sp, dwp_file_data_offset);
-  if (!ObjectFileELF::classof(dwp_obj_file.get()))
-    return false;
-  // The presence of a debug_cu_index section is the key identifying feature of
-  // a DWP file. Make sure we don't fill in the section list on dwp_obj_file
-  // (by calling GetSectionList(false)) as this is invoked before we may have
-  // all the symbol files collected and available.
-  return dwp_obj_file && dwp_obj_file->GetSectionList(false)->FindSectionByType(
-                             eSectionTypeDWARFDebugCuIndex, false);
-}
-
 // CreateInstance
 //
 // Platforms can register a callback to use when creating symbol vendors to
@@ -106,15 +87,8 @@ SymbolVendorELF::CreateInstance(const lldb::ModuleSP &module_sp,
   FileSpecList search_paths = Target::GetDefaultDebugFileSearchPaths();
   FileSpec dsym_fspec =
       PluginManager::LocateExecutableSymbolFile(module_spec, search_paths);
-  if (!dsym_fspec || IsDwpSymbolFile(module_sp, dsym_fspec)) {
-    // If we have a stripped binary or if we got a DWP file, we should prefer
-    // symbols in the executable acquired through a plugin.
-    ModuleSpec unstripped_spec =
-        PluginManager::LocateExecutableObjectFile(module_spec);
-    if (!unstripped_spec)
-      return nullptr;
-    dsym_fspec = unstripped_spec.GetFileSpec();
-  }
+  if (!dsym_fspec)
+    return nullptr;
 
   DataBufferSP dsym_file_data_sp;
   lldb::offset_t dsym_file_data_offset = 0;
diff --git a/lldb/test/API/debuginfod/Normal/Makefile b/lldb/test/API/debuginfod/Normal/Makefile
deleted file mode 100644
index bd2fa623df47..000000000000
--- a/lldb/test/API/debuginfod/Normal/Makefile
+++ /dev/null
@@ -1,25 +0,0 @@
-C_SOURCES := main.c
-
-# For normal (non DWP) Debuginfod tests, we need:
-
-# * The "full" binary: a.out.debug
-#   Produced by Makefile.rules with KEEP_FULL_DEBUG_BINARY set to YES and
-#   SPLIT_DEBUG_SYMBOLS set to YES
-
-# * The stripped binary (a.out)
-#   Produced by Makefile.rules with SPLIT_DEBUG_SYMBOLS set to YES
-
-# * The 'only-keep-debug' binary (a.out.dbg)
-#   Produced below
-
-# * The .uuid file (for a little easier testing code)
-#   Produced below
-
-# Don't strip the debug info from a.out:
-SPLIT_DEBUG_SYMBOLS := YES
-SAVE_FULL_DEBUG_BINARY := YES
-GEN_GNU_BUILD_ID := YES
-
-all: a.out.uuid a.out
-
-include Makefile.rules
diff --git a/lldb/test/API/debuginfod/Normal/TestDebuginfod.py b/lldb/test/API/debuginfod/Normal/TestDebuginfod.py
deleted file mode 100644
index eb5efe83c17a..000000000000
--- a/lldb/test/API/debuginfod/Normal/TestDebuginfod.py
+++ /dev/null
@@ -1,185 +0,0 @@
-import os
-import shutil
-import tempfile
-import struct
-
-import lldb
-from lldbsuite.test.decorators import *
-import lldbsuite.test.lldbutil as lldbutil
-from lldbsuite.test.lldbtest import *
-
-
-def getUUID(aoutuuid):
-    """
-    Pull the 20 byte UUID out of the .note.gnu.build-id section that was dumped
-    to a file already, as part of the build.
-    """
-    with open(aoutuuid, "rb") as f:
-        data = f.read(36)
-        if len(data) != 36:
-            return None
-        header = struct.unpack_from("<4I", data)
-        if len(header) != 4:
-            return None
-        # 4 element 'prefix', 20 bytes of uuid, 3 byte long string: 'GNU':
-        if header[0] != 4 or header[1] != 20 or header[2] != 3 or header[3] != 0x554E47:
-            return None
-        return data[16:].hex()
-
-
-"""
-Test support for the DebugInfoD network symbol acquisition protocol.
-This one is for simple / no split-dwarf scenarios.
-
-For no-split-dwarf scenarios, there are 2 variations:
-1 - A stripped binary with it's corresponding unstripped binary:
-2 - A stripped binary with a corresponding --only-keep-debug symbols file
-"""
-
-
-@skipUnlessPlatform(["linux", "freebsd"])
-class DebugInfodTests(TestBase):
-    # No need to try every flavor of debug inf.
-    NO_DEBUG_INFO_TESTCASE = True
-
-    def test_normal_no_symbols(self):
-        """
-        Validate behavior with no symbols or symbol locator.
-        ('baseline negative' behavior)
-        """
-        test_root = self.config_test(["a.out"])
-        self.try_breakpoint(False)
-
-    def test_normal_default(self):
-        """
-        Validate behavior with symbols, but no symbol locator.
-        ('baseline positive' behavior)
-        """
-        test_root = self.config_test(["a.out", "a.out.debug"])
-        self.try_breakpoint(True)
-
-    def test_debuginfod_symbols(self):
-        """
-        Test behavior with the full binary available from Debuginfod as
-        'debuginfo' from the plug-in.
-        """
-        test_root = self.config_test(["a.out"], "a.out.full")
-        self.try_breakpoint(True)
-
-    def test_debuginfod_executable(self):
-        """
-        Test behavior with the full binary available from Debuginfod as
-        'executable' from the plug-in.
-        """
-        test_root = self.config_test(["a.out"], None, "a.out.full")
-        self.try_breakpoint(True)
-
-    def test_debuginfod_okd_symbols(self):
-        """
-        Test behavior with the 'only-keep-debug' symbols available from Debuginfod.
-        """
-        test_root = self.config_test(["a.out"], "a.out.debug")
-        self.try_breakpoint(True)
-
-    def try_breakpoint(self, should_have_loc):
-        """
-        This function creates a target from self.aout, sets a function-name
-        breakpoint, and checks to see if we have a file/line location,
-        as a way to validate that the symbols have been loaded.
-        should_have_loc specifies if we're testing that symbols have or
-        haven't been loaded.
-        """
-        target = self.dbg.CreateTarget(self.aout)
-        self.assertTrue(target and target.IsValid(), "Target is valid")
-
-        bp = target.BreakpointCreateByName("func")
-        self.assertTrue(bp and bp.IsValid(), "Breakpoint is valid")
-        self.assertEqual(bp.GetNumLocations(), 1)
-
-        loc = bp.GetLocationAtIndex(0)
-        self.assertTrue(loc and loc.IsValid(), "Location is valid")
-        addr = loc.GetAddress()
-        self.assertTrue(addr and addr.IsValid(), "Loc address is valid")
-        line_entry = addr.GetLineEntry()
-        self.assertEqual(
-            should_have_loc,
-            line_entry != None and line_entry.IsValid(),
-            "Loc line entry is valid",
-        )
-        if should_have_loc:
-            self.assertEqual(line_entry.GetLine(), 4)
-            self.assertEqual(
-                line_entry.GetFileSpec().GetFilename(),
-                self.main_source_file.GetFilename(),
-            )
-        self.dbg.DeleteTarget(target)
-        shutil.rmtree(self.tmp_dir)
-
-    def config_test(self, local_files, debuginfo=None, executable=None):
-        """
-        Set up a test with local_files[] copied to a different location
-        so that we control which files are, or are not, found in the file system.
-        Also, create a stand-alone file-system 'hosted' debuginfod server with the
-        provided debuginfo and executable files (if they exist)
-
-        Make the filesystem look like:
-
-        /tmp/<tmpdir>/test/[local_files]
-
-        /tmp/<tmpdir>/cache (for lldb to use as a temp cache)
-
-        /tmp/<tmpdir>/buildid/<uuid>/executable -> <executable>
-        /tmp/<tmpdir>/buildid/<uuid>/debuginfo -> <debuginfo>
-        Returns the /tmp/<tmpdir> path
-        """
-
-        self.build()
-
-        uuid = getUUID(self.getBuildArtifact("a.out.uuid"))
-
-        self.main_source_file = lldb.SBFileSpec("main.c")
-        self.tmp_dir = tempfile.mkdtemp()
-        test_dir = os.path.join(self.tmp_dir, "test")
-        os.makedirs(test_dir)
-
-        self.aout = ""
-        # Copy the files used by the test:
-        for f in local_files:
-            shutil.copy(self.getBuildArtifact(f), test_dir)
-            # The first item is the binary to be used for the test
-            if self.aout == "":
-                self.aout = os.path.join(test_dir, f)
-
-        use_debuginfod = debuginfo != None or executable != None
-
-        # Populated the 'file://... mocked' Debuginfod server:
-        if use_debuginfod:
-            os.makedirs(os.path.join(self.tmp_dir, "cache"))
-            uuid_dir = os.path.join(self.tmp_dir, "buildid", uuid)
-            os.makedirs(uuid_dir)
-            if debuginfo:
-                shutil.copy(
-                    self.getBuildArtifact(debuginfo),
-                    os.path.join(uuid_dir, "debuginfo"),
-                )
-            if executable:
-                shutil.copy(
-                    self.getBuildArtifact(executable),
-                    os.path.join(uuid_dir, "executable"),
-                )
-
-        # Configure LLDB for the test:
-        self.runCmd(
-            "settings set symbols.enable-external-lookup %s"
-            % str(use_debuginfod).lower()
-        )
-        self.runCmd("settings clear plugin.symbol-locator.debuginfod.server-urls")
-        if use_debuginfod:
-            self.runCmd(
-                "settings set plugin.symbol-locator.debuginfod.cache-path %s/cache"
-                % self.tmp_dir
-            )
-            self.runCmd(
-                "settings insert-before plugin.symbol-locator.debuginfod.server-urls 0 file://%s"
-                % self.tmp_dir
-            )
diff --git a/lldb/test/API/debuginfod/Normal/main.c b/lldb/test/API/debuginfod/Normal/main.c
deleted file mode 100644
index 4c7184609b45..000000000000
--- a/lldb/test/API/debuginfod/Normal/main.c
+++ /dev/null
@@ -1,7 +0,0 @@
-// This is a dump little pair of test files
-
-int func(int argc, const char *argv[]) {
-  return (argc + 1) * (argv[argc][0] + 2);
-}
-
-int main(int argc, const char *argv[]) { return func(0, argv); }
diff --git a/lldb/test/API/debuginfod/SplitDWARF/Makefile b/lldb/test/API/debuginfod/SplitDWARF/Makefile
deleted file mode 100644
index 266d74cf9062..000000000000
--- a/lldb/test/API/debuginfod/SplitDWARF/Makefile
+++ /dev/null
@@ -1,28 +0,0 @@
-C_SOURCES := main.c
-
-# For split-dwarf Debuginfod tests, we need:
-
-# * A .DWP file (a.out.dwp)
-#   Produced by Makefile.rules with MAKE_DWO and MERGE_DWOS both set to YES
-
-# * The "full" binary: it's missing things that live in .dwo's (a.out.debug)
-#   Produced by Makefile.rules with KEEP_FULL_DEBUG_BINARY set to YES and
-#   SPLIT_DEBUG_SYMBOLS set to YES
-
-# * The stripped binary (a.out)
-#   Produced by Makefile.rules
-
-# * The 'only-keep-debug' binary (a.out.dbg)
-#   Produced below
-
-# * The .uuid file (for a little easier testing code)
-#   Produced here in the rule below
-
-MAKE_DWP := YES
-SPLIT_DEBUG_SYMBOLS := YES
-SAVE_FULL_DEBUG_BINARY := YES
-GEN_GNU_BUILD_ID := YES
-
-all: a.out.uuid a.out
-
-include Makefile.rules
diff --git a/lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py b/lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py
deleted file mode 100644
index 09f91b6f1c6c..000000000000
--- a/lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py
+++ /dev/null
@@ -1,194 +0,0 @@
-"""
-Test support for the DebugInfoD network symbol acquisition protocol.
-"""
-import os
-import shutil
-import tempfile
-import struct
-
-import lldb
-from lldbsuite.test.decorators import *
-import lldbsuite.test.lldbutil as lldbutil
-from lldbsuite.test.lldbtest import *
-
-
-def getUUID(aoutuuid):
-    """
-    Pull the 20 byte UUID out of the .note.gnu.build-id section that was dumped
-    to a file already, as part of the build.
-    """
-    with open(aoutuuid, "rb") as f:
-        data = f.read(36)
-        if len(data) != 36:
-            return None
-        header = struct.unpack_from("<4I", data)
-        if len(header) != 4:
-            return None
-        # 4 element 'prefix', 20 bytes of uuid, 3 byte long string: 'GNU':
-        if header[0] != 4 or header[1] != 20 or header[2] != 3 or header[3] != 0x554E47:
-            return None
-        return data[16:].hex()
-
-
-"""
-Test support for the DebugInfoD network symbol acquisition protocol.
-This file is for split-dwarf (dwp) scenarios.
-
-1 - A split binary target with it's corresponding DWP file
-2 - A stripped, split binary target with an unstripped binary and a DWP file
-3 - A stripped, split binary target with an --only-keep-debug symbols file and a DWP file
-"""
-
-
-@skipUnlessPlatform(["linux", "freebsd"])
-class DebugInfodDWPTests(TestBase):
-    # No need to try every flavor of debug inf.
-    NO_DEBUG_INFO_TESTCASE = True
-
-    def test_normal_stripped(self):
-        """
-        Validate behavior with a stripped binary, no symbols or symbol locator.
-        """
-        self.config_test(["a.out"])
-        self.try_breakpoint(False)
-
-    def test_normal_stripped_split_with_dwp(self):
-        """
-        Validate behavior with symbols, but no symbol locator.
-        """
-        self.config_test(["a.out", "a.out.debug", "a.out.dwp"])
-        self.try_breakpoint(True)
-
-    def test_normal_stripped_only_dwp(self):
-        """
-        Validate behavior *with* dwp symbols only, but missing other symbols,
-        but no symbol locator. This shouldn't work: without the other symbols
-        DWO's appear mostly useless.
-        """
-        self.config_test(["a.out", "a.out.dwp"])
-        self.try_breakpoint(False)
-
-    def test_debuginfod_dwp_from_service(self):
-        """
-        Test behavior with the unstripped binary, and DWP from the service.
-        """
-        self.config_test(["a.out.debug"], "a.out.dwp")
-        self.try_breakpoint(True)
-
-    def test_debuginfod_both_symfiles_from_service(self):
-        """
-        Test behavior with a stripped binary, with the unstripped binary and
-        dwp symbols from Debuginfod.
-        """
-        self.config_test(["a.out"], "a.out.dwp", "a.out.full")
-        self.try_breakpoint(True)
-
-    def test_debuginfod_both_okd_symfiles_from_service(self):
-        """
-        Test behavior with both the only-keep-debug symbols and the dwp symbols
-        from Debuginfod.
-        """
-        self.config_test(["a.out"], "a.out.dwp", "a.out.debug")
-        self.try_breakpoint(True)
-
-    def try_breakpoint(self, should_have_loc):
-        """
-        This function creates a target from self.aout, sets a function-name
-        breakpoint, and checks to see if we have a file/line location,
-        as a way to validate that the symbols have been loaded.
-        should_have_loc specifies if we're testing that symbols have or
-        haven't been loaded.
-        """
-        target = self.dbg.CreateTarget(self.aout)
-        self.assertTrue(target and target.IsValid(), "Target is valid")
-
-        bp = target.BreakpointCreateByName("func")
-        self.assertTrue(bp and bp.IsValid(), "Breakpoint is valid")
-        self.assertEqual(bp.GetNumLocations(), 1)
-
-        loc = bp.GetLocationAtIndex(0)
-        self.assertTrue(loc and loc.IsValid(), "Location is valid")
-        addr = loc.GetAddress()
-        self.assertTrue(addr and addr.IsValid(), "Loc address is valid")
-        line_entry = addr.GetLineEntry()
-        self.assertEqual(
-            should_have_loc,
-            line_entry != None and line_entry.IsValid(),
-            "Loc line entry is valid",
-        )
-        if should_have_loc:
-            self.assertEqual(line_entry.GetLine(), 4)
-            self.assertEqual(
-                line_entry.GetFileSpec().GetFilename(),
-                self.main_source_file.GetFilename(),
-            )
-        self.dbg.DeleteTarget(target)
-        shutil.rmtree(self.tmp_dir)
-
-    def config_test(self, local_files, debuginfo=None, executable=None):
-        """
-        Set up a test with local_files[] copied to a different location
-        so that we control which files are, or are not, found in the file system.
-        Also, create a stand-alone file-system 'hosted' debuginfod server with the
-        provided debuginfo and executable files (if they exist)
-
-        Make the filesystem look like:
-
-        /tmp/<tmpdir>/test/[local_files]
-
-        /tmp/<tmpdir>/cache (for lldb to use as a temp cache)
-
-        /tmp/<tmpdir>/buildid/<uuid>/executable -> <executable>
-        /tmp/<tmpdir>/buildid/<uuid>/debuginfo -> <debuginfo>
-        Returns the /tmp/<tmpdir> path
-        """
-
-        self.build()
-
-        uuid = getUUID(self.getBuildArtifact("a.out.uuid"))
-
-        self.main_source_file = lldb.SBFileSpec("main.c")
-        self.tmp_dir = tempfile.mkdtemp()
-        self.test_dir = os.path.join(self.tmp_dir, "test")
-        os.makedirs(self.test_dir)
-
-        self.aout = ""
-        # Copy the files used by the test:
-        for f in local_files:
-            shutil.copy(self.getBuildArtifact(f), self.test_dir)
-            if self.aout == "":
-                self.aout = os.path.join(self.test_dir, f)
-
-        use_debuginfod = debuginfo != None or executable != None
-
-        # Populated the 'file://... mocked' Debuginfod server:
-        if use_debuginfod:
-            os.makedirs(os.path.join(self.tmp_dir, "cache"))
-            uuid_dir = os.path.join(self.tmp_dir, "buildid", uuid)
-            os.makedirs(uuid_dir)
-            if debuginfo:
-                shutil.copy(
-                    self.getBuildArtifact(debuginfo),
-                    os.path.join(uuid_dir, "debuginfo"),
-                )
-            if executable:
-                shutil.copy(
-                    self.getBuildArtifact(executable),
-                    os.path.join(uuid_dir, "executable"),
-                )
-        os.remove(self.getBuildArtifact("main.dwo"))
-        # Configure LLDB for the test:
-        self.runCmd(
-            "settings set symbols.enable-external-lookup %s"
-            % str(use_debuginfod).lower()
-        )
-        self.runCmd("settings clear plugin.symbol-locator.debuginfod.server-urls")
-        if use_debuginfod:
-            self.runCmd(
-                "settings set plugin.symbol-locator.debuginfod.cache-path %s/cache"
-                % self.tmp_dir
-            )
-            self.runCmd(
-                "settings insert-before plugin.symbol-locator.debuginfod.server-urls 0 file://%s"
-                % self.tmp_dir
-            )
diff --git a/lldb/test/API/debuginfod/SplitDWARF/main.c b/lldb/test/API/debuginfod/SplitDWARF/main.c
deleted file mode 100644
index 4c7184609b45..000000000000
--- a/lldb/test/API/debuginfod/SplitDWARF/main.c
+++ /dev/null
@@ -1,7 +0,0 @@
-// This is a dump little pair of test files
-
-int func(int argc, const char *argv[]) {
-  return (argc + 1) * (argv[argc][0] + 2);
-}
-
-int main(int argc, const char *argv[]) { return func(0, argv); }
-- 
GitLab


From 00248754176d74aed2e0785d9982a5ea8e91a71a Mon Sep 17 00:00:00 2001
From: "Daniel M. Katz" <katzdm@gmail.com>
Date: Fri, 22 Mar 2024 19:08:02 -0400
Subject: [PATCH 023/404] [Clang] Raise an error on namespace aliases with
 qualified names. (#86122)

---
 clang/docs/ReleaseNotes.rst                       | 2 ++
 clang/include/clang/Basic/DiagnosticParseKinds.td | 2 ++
 clang/lib/Parse/ParseDeclCXX.cpp                  | 8 ++++++++
 clang/test/SemaCXX/namespace-alias.cpp            | 2 ++
 4 files changed, 14 insertions(+)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index d6e179ca9d69..8054d90fc70f 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -437,6 +437,8 @@ Bug Fixes to C++ Support
 - Clang's __builtin_bit_cast will now produce a constant value for records with empty bases. See:
   (#GH82383)
 - Fix a crash when instantiating a lambda that captures ``this`` outside of its context. Fixes (#GH85343).
+- Fix an issue where a namespace alias could be defined using a qualified name (all name components
+  following the first `::` were ignored).
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 48de5e2ef5f4..46a44418a315 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -268,6 +268,8 @@ def err_expected_semi_after_namespace_name : Error<
   "expected ';' after namespace name">;
 def err_unexpected_namespace_attributes_alias : Error<
   "attributes cannot be specified on namespace alias">;
+def err_unexpected_qualified_namespace_alias : Error<
+  "namespace alias must be a single identifier">;
 def err_unexpected_nested_namespace_attribute : Error<
   "attributes cannot be specified on a nested namespace definition">;
 def err_inline_namespace_alias : Error<"namespace alias cannot be inline">;
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index 77d2382ea6d9..63fe678cbb29 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -140,6 +140,14 @@ Parser::DeclGroupPtrTy Parser::ParseNamespace(DeclaratorContext Context,
       SkipUntil(tok::semi);
       return nullptr;
     }
+    if (!ExtraNSs.empty()) {
+      Diag(ExtraNSs.front().NamespaceLoc,
+           diag::err_unexpected_qualified_namespace_alias)
+          << SourceRange(ExtraNSs.front().NamespaceLoc,
+                         ExtraNSs.back().IdentLoc);
+      SkipUntil(tok::semi);
+      return nullptr;
+    }
     if (attrLoc.isValid())
       Diag(attrLoc, diag::err_unexpected_namespace_attributes_alias);
     if (InlineLoc.isValid())
diff --git a/clang/test/SemaCXX/namespace-alias.cpp b/clang/test/SemaCXX/namespace-alias.cpp
index 281ee9962e8b..591957a657c0 100644
--- a/clang/test/SemaCXX/namespace-alias.cpp
+++ b/clang/test/SemaCXX/namespace-alias.cpp
@@ -47,6 +47,8 @@ namespace I {
   namespace A1 { int i; }
   
   namespace A2 = A1;
+
+  namespace A3::extra::specifiers = A2;  // expected-error {{alias must be a single identifier}}
 }
 
 int f() {
-- 
GitLab


From 5d187898f625cc54310f51b278b36ad6a97104ee Mon Sep 17 00:00:00 2001
From: Fabian Tschopp <fabian.tschopp@modular.com>
Date: Sat, 23 Mar 2024 00:09:11 +0100
Subject: [PATCH 024/404] [mlir][inliner] Return early if the inliningThreshold
 is 0U or -1U. (#86287)

Computing the inlinling profitability can be costly due to walking the
graph when counting the number of operations.

This PR addresses that by returning early if the threshold is set to
never or always inline.
---
 mlir/lib/Transforms/InlinerPass.cpp | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/mlir/lib/Transforms/InlinerPass.cpp b/mlir/lib/Transforms/InlinerPass.cpp
index 08d8dbf73a6a..9a7d5403a95d 100644
--- a/mlir/lib/Transforms/InlinerPass.cpp
+++ b/mlir/lib/Transforms/InlinerPass.cpp
@@ -93,12 +93,19 @@ InlinerPass::InlinerPass(std::function<void(OpPassManager &)> defaultPipeline,
 // Return true if the inlining ratio does not exceed the threshold.
 static bool isProfitableToInline(const Inliner::ResolvedCall &resolvedCall,
                                  unsigned inliningThreshold) {
+  // Return early, ratio <= 0U will always be false.
+  if (inliningThreshold == 0U)
+    return false;
+  // Return early, ratio <= -1U will always be true.
+  if (inliningThreshold == -1U)
+    return true;
+
   Region *callerRegion = resolvedCall.sourceNode->getCallableRegion();
   Region *calleeRegion = resolvedCall.targetNode->getCallableRegion();
 
   // We should not get external nodes here, but just return true
   // for now to preserve the original behavior of the inliner pass.
-  if (!calleeRegion || !calleeRegion)
+  if (!callerRegion || !calleeRegion)
     return true;
 
   auto countOps = [](Region *region) {
-- 
GitLab


From b768a8c1db85b9e84fd8b356570a3a8fbe37acf6 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda@apple.com>
Date: Fri, 22 Mar 2024 16:09:34 -0700
Subject: [PATCH 025/404] [clang][deps] Lazy dependency directives (#86347)

Since b4c83a13f664582015ea22924b9a0c6290d41f5b, `Preprocessor` and
`Lexer` are aware of the concept of scanning dependency directives. This
makes it possible to scan for them on-demand rather than eagerly on the
first filesystem operation (open, or even just stat).

This might improve performance, but is also necessary for the "PCH as
module" mode. Some precompiled header sources use the ".pch" file
extension, which means they were not getting scanned for dependency
directives. This was okay when the PCH was the main input file in a
separate scan step, because there we just lex the file in a
scanning-specific frontend action. But when such source gets treated as
a module implicitly loaded from a TU, it will get compiled as any other
module - with Sema - which will result in compilation errors. (See
attached test case.)

rdar://107663951
---
 .../DependencyScanningFilesystem.h            | 20 +++---
 .../DependencyScanningFilesystem.cpp          | 63 +++++--------------
 .../DependencyScanningWorker.cpp              |  3 +-
 clang/test/ClangScanDeps/modules-extension.c  | 33 ++++++++++
 4 files changed, 60 insertions(+), 59 deletions(-)
 create mode 100644 clang/test/ClangScanDeps/modules-extension.c

diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
index 846fdc725397..9a522a3e2fe2 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -242,6 +242,8 @@ class EntryRef {
   /// The underlying cached entry.
   const CachedFileSystemEntry &Entry;
 
+  friend class DependencyScanningWorkerFilesystem;
+
 public:
   EntryRef(StringRef Name, const CachedFileSystemEntry &Entry)
       : Filename(Name), Entry(Entry) {}
@@ -300,14 +302,15 @@ public:
   ///
   /// Attempts to use the local and shared caches first, then falls back to
   /// using the underlying filesystem.
-  llvm::ErrorOr<EntryRef>
-  getOrCreateFileSystemEntry(StringRef Filename,
-                             bool DisableDirectivesScanning = false);
+  llvm::ErrorOr<EntryRef> getOrCreateFileSystemEntry(StringRef Filename);
 
-private:
-  /// Check whether the file should be scanned for preprocessor directives.
-  bool shouldScanForDirectives(StringRef Filename);
+  /// Ensure the directive tokens are populated for this file entry.
+  ///
+  /// Returns true if the directive tokens are populated for this file entry,
+  /// false if not (i.e. this entry is not a file or its scan fails).
+  bool ensureDirectiveTokensArePopulated(EntryRef Entry);
 
+private:
   /// For a filename that's not yet associated with any entry in the caches,
   /// uses the underlying filesystem to either look up the entry based in the
   /// shared cache indexed by unique ID, or creates new entry from scratch.
@@ -317,11 +320,6 @@ private:
   computeAndStoreResult(StringRef OriginalFilename,
                         StringRef FilenameForLookup);
 
-  /// Scan for preprocessor directives for the given entry if necessary and
-  /// returns a wrapper object with reference semantics.
-  EntryRef scanForDirectivesIfNecessary(const CachedFileSystemEntry &Entry,
-                                        StringRef Filename, bool Disable);
-
   /// Represents a filesystem entry that has been stat-ed (and potentially read)
   /// and that's about to be inserted into the cache as `CachedFileSystemEntry`.
   struct TentativeEntry {
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index 1b750cec41e1..9b7812a1adb9 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -41,24 +41,25 @@ DependencyScanningWorkerFilesystem::readFile(StringRef Filename) {
   return TentativeEntry(Stat, std::move(Buffer));
 }
 
-EntryRef DependencyScanningWorkerFilesystem::scanForDirectivesIfNecessary(
-    const CachedFileSystemEntry &Entry, StringRef Filename, bool Disable) {
-  if (Entry.isError() || Entry.isDirectory() || Disable ||
-      !shouldScanForDirectives(Filename))
-    return EntryRef(Filename, Entry);
+bool DependencyScanningWorkerFilesystem::ensureDirectiveTokensArePopulated(
+    EntryRef Ref) {
+  auto &Entry = Ref.Entry;
+
+  if (Entry.isError() || Entry.isDirectory())
+    return false;
 
   CachedFileContents *Contents = Entry.getCachedContents();
   assert(Contents && "contents not initialized");
 
   // Double-checked locking.
   if (Contents->DepDirectives.load())
-    return EntryRef(Filename, Entry);
+    return true;
 
   std::lock_guard<std::mutex> GuardLock(Contents->ValueLock);
 
   // Double-checked locking.
   if (Contents->DepDirectives.load())
-    return EntryRef(Filename, Entry);
+    return true;
 
   SmallVector<dependency_directives_scan::Directive, 64> Directives;
   // Scan the file for preprocessor directives that might affect the
@@ -69,16 +70,16 @@ EntryRef DependencyScanningWorkerFilesystem::scanForDirectivesIfNecessary(
     Contents->DepDirectiveTokens.clear();
     // FIXME: Propagate the diagnostic if desired by the client.
     Contents->DepDirectives.store(new std::optional<DependencyDirectivesTy>());
-    return EntryRef(Filename, Entry);
+    return false;
   }
 
   // This function performed double-checked locking using `DepDirectives`.
   // Assigning it must be the last thing this function does, otherwise other
-  // threads may skip the
-  // critical section (`DepDirectives != nullptr`), leading to a data race.
+  // threads may skip the critical section (`DepDirectives != nullptr`), leading
+  // to a data race.
   Contents->DepDirectives.store(
       new std::optional<DependencyDirectivesTy>(std::move(Directives)));
-  return EntryRef(Filename, Entry);
+  return true;
 }
 
 DependencyScanningFilesystemSharedCache::
@@ -161,34 +162,11 @@ DependencyScanningFilesystemSharedCache::CacheShard::
   return *EntriesByFilename.insert({Filename, &Entry}).first->getValue();
 }
 
-/// Whitelist file extensions that should be minimized, treating no extension as
-/// a source file that should be minimized.
-///
-/// This is kinda hacky, it would be better if we knew what kind of file Clang
-/// was expecting instead.
-static bool shouldScanForDirectivesBasedOnExtension(StringRef Filename) {
-  StringRef Ext = llvm::sys::path::extension(Filename);
-  if (Ext.empty())
-    return true; // C++ standard library
-  return llvm::StringSwitch<bool>(Ext)
-      .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true)
-      .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true)
-      .CasesLower(".m", ".mm", true)
-      .CasesLower(".i", ".ii", ".mi", ".mmi", true)
-      .CasesLower(".def", ".inc", true)
-      .Default(false);
-}
-
 static bool shouldCacheStatFailures(StringRef Filename) {
   StringRef Ext = llvm::sys::path::extension(Filename);
   if (Ext.empty())
     return false; // This may be the module cache directory.
-  // Only cache stat failures on files that are not expected to change during
-  // the build.
-  StringRef FName = llvm::sys::path::filename(Filename);
-  if (FName == "module.modulemap" || FName == "module.map")
-    return true;
-  return shouldScanForDirectivesBasedOnExtension(Filename);
+  return true;
 }
 
 DependencyScanningWorkerFilesystem::DependencyScanningWorkerFilesystem(
@@ -201,11 +179,6 @@ DependencyScanningWorkerFilesystem::DependencyScanningWorkerFilesystem(
   updateWorkingDirForCacheLookup();
 }
 
-bool DependencyScanningWorkerFilesystem::shouldScanForDirectives(
-    StringRef Filename) {
-  return shouldScanForDirectivesBasedOnExtension(Filename);
-}
-
 const CachedFileSystemEntry &
 DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForUID(
     TentativeEntry TEntry) {
@@ -259,7 +232,7 @@ DependencyScanningWorkerFilesystem::computeAndStoreResult(
 
 llvm::ErrorOr<EntryRef>
 DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
-    StringRef OriginalFilename, bool DisableDirectivesScanning) {
+    StringRef OriginalFilename) {
   StringRef FilenameForLookup;
   SmallString<256> PathBuf;
   if (llvm::sys::path::is_absolute_gnu(OriginalFilename)) {
@@ -276,15 +249,11 @@ DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
   assert(llvm::sys::path::is_absolute_gnu(FilenameForLookup));
   if (const auto *Entry =
           findEntryByFilenameWithWriteThrough(FilenameForLookup))
-    return scanForDirectivesIfNecessary(*Entry, OriginalFilename,
-                                        DisableDirectivesScanning)
-        .unwrapError();
+    return EntryRef(OriginalFilename, *Entry).unwrapError();
   auto MaybeEntry = computeAndStoreResult(OriginalFilename, FilenameForLookup);
   if (!MaybeEntry)
     return MaybeEntry.getError();
-  return scanForDirectivesIfNecessary(*MaybeEntry, OriginalFilename,
-                                      DisableDirectivesScanning)
-      .unwrapError();
+  return EntryRef(OriginalFilename, *MaybeEntry).unwrapError();
 }
 
 llvm::ErrorOr<llvm::vfs::Status>
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index 76f3d950a13b..33b43417a661 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -372,7 +372,8 @@ public:
           -> std::optional<ArrayRef<dependency_directives_scan::Directive>> {
         if (llvm::ErrorOr<EntryRef> Entry =
                 LocalDepFS->getOrCreateFileSystemEntry(File.getName()))
-          return Entry->getDirectiveTokens();
+          if (LocalDepFS->ensureDirectiveTokensArePopulated(*Entry))
+            return Entry->getDirectiveTokens();
         return std::nullopt;
       };
     }
diff --git a/clang/test/ClangScanDeps/modules-extension.c b/clang/test/ClangScanDeps/modules-extension.c
new file mode 100644
index 000000000000..0f27f608440f
--- /dev/null
+++ b/clang/test/ClangScanDeps/modules-extension.c
@@ -0,0 +1,33 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+
+// This test checks that source files with uncommon extensions still undergo
+// dependency directives scan. If header.pch would not and b.h would, the scan
+// would fail when parsing `void function(B)` and not knowing the symbol B.
+
+//--- module.modulemap
+module __PCH { header "header.pch" }
+module B { header "b.h" }
+
+//--- header.pch
+#include "b.h"
+void function(B);
+
+//--- b.h
+typedef int B;
+
+//--- tu.c
+int main() {
+  function(0);
+  return 0;
+}
+
+//--- cdb.json.in
+[{
+  "directory": "DIR",
+  "file": "DIR/tu.c",
+  "command": "clang -c DIR/tu.c -fmodules -fmodules-cache-path=DIR/cache -fimplicit-module-maps -include DIR/header.pch"
+}]
+
+// RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.in > %t/cdb.json
+// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full > %t/deps.json
-- 
GitLab


From af63c6e5d08fcaeacaeee68aa0a1cda71d9a7549 Mon Sep 17 00:00:00 2001
From: Philipp Tomsich <philipp.tomsich@vrull.eu>
Date: Sat, 23 Mar 2024 00:29:58 +0100
Subject: [PATCH 026/404] [AArch64] Adjust ROBsize for Ampere1/Ampere1A (NFC)
 (#86330)

To align more closely with common usage, we now use the size of the
reorder-buffer for MicroOpBufferSize instead of the entries of the
global micro-op scheduler.
---
 llvm/lib/Target/AArch64/AArch64SchedAmpere1.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/AArch64/AArch64SchedAmpere1.td b/llvm/lib/Target/AArch64/AArch64SchedAmpere1.td
index cf9f50c2784b..269f4ec5e5fb 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedAmpere1.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedAmpere1.td
@@ -18,7 +18,7 @@
 
 def Ampere1Model : SchedMachineModel {
   let IssueWidth            =   4;  // 4-way decode and dispatch
-  let MicroOpBufferSize     = 174;  // micro-op re-order buffer size
+  let MicroOpBufferSize     = 192;  // re-order buffer size
   let LoadLatency           =   4;  // Optimistic load latency
   let MispredictPenalty     =  10;  // Branch mispredict penalty
   let LoopMicroOpBufferSize =  32;  // Instruction queue size
-- 
GitLab


From b621269d4a4c08269b1b2d46f277d1918d3dab62 Mon Sep 17 00:00:00 2001
From: Philipp Tomsich <philipp.tomsich@vrull.eu>
Date: Sat, 23 Mar 2024 00:30:12 +0100
Subject: [PATCH 027/404] [AArch64] Adjust ROBsize for Ampere1B (NFC) (#86331)

To align more closely with common usage, we now use the size of the
reorder-buffer for MicroOpBufferSize instead of the entries of the
global micro-op scheduler.
---
 llvm/lib/Target/AArch64/AArch64SchedAmpere1B.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/AArch64/AArch64SchedAmpere1B.td b/llvm/lib/Target/AArch64/AArch64SchedAmpere1B.td
index 9c4f000cf351..67f8593f1577 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedAmpere1B.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedAmpere1B.td
@@ -18,7 +18,7 @@
 
 def Ampere1BModel : SchedMachineModel {
   let IssueWidth            =  12;  // Maximum micro-ops dispatch rate.
-  let MicroOpBufferSize     = 192;  // micro-op re-order buffer size
+  let MicroOpBufferSize     = 208;  // micro-op re-order buffer size
   let LoadLatency           =   3;  // Optimistic load latency
   let MispredictPenalty     =  10;  // Branch mispredict penalty
   let LoopMicroOpBufferSize =  32;  // Instruction queue size
-- 
GitLab


From c3747883a0bf34d271bc89dbfc60590adf75d999 Mon Sep 17 00:00:00 2001
From: Xiaoyang Liu <siujoeng.lau@gmail.com>
Date: Fri, 22 Mar 2024 16:32:02 -0700
Subject: [PATCH 028/404] [libc++][ranges] use `static operator()` for C++23
 ranges (#86052)

## Abstract

This pull request converts the `operator()` of all CPOs and niebloids
related to C++23 ranges to `static`.

## Motivation

In `libc++`, CPOs and niebloids are implemented as function objects.
Currently, the `operator()` for such a function object is a
`const`-qualified member function. This means that even if the function
object is has no data members, an extra register is used to pass in the
`this` pointer when calling `operator()`, unless the compiler can inline
the function call. Declaraing `operator()` as `static` would optimize
away the unnecessary `this` pointer passing for stateless function
objects, since there is no object instance state that needs to be
accessed.

## Reference

- [P1169R4: static `operator()`](https://wg21.link/P1169R4)
---
 libcxx/include/__algorithm/ranges_ends_with.h   |  6 +++---
 libcxx/include/__algorithm/ranges_starts_with.h |  8 ++++----
 libcxx/include/__ranges/as_rvalue_view.h        | 16 ++++++++--------
 libcxx/include/__ranges/repeat_view.h           |  5 ++---
 libcxx/include/__ranges/to.h                    |  4 ++--
 libcxx/include/__ranges/zip_view.h              |  8 ++++----
 6 files changed, 23 insertions(+), 24 deletions(-)

diff --git a/libcxx/include/__algorithm/ranges_ends_with.h b/libcxx/include/__algorithm/ranges_ends_with.h
index c2a3cae9f3b1..bb01918326b8 100644
--- a/libcxx/include/__algorithm/ranges_ends_with.h
+++ b/libcxx/include/__algorithm/ranges_ends_with.h
@@ -39,7 +39,7 @@ namespace ranges {
 namespace __ends_with {
 struct __fn {
   template <class _Iter1, class _Sent1, class _Iter2, class _Sent2, class _Pred, class _Proj1, class _Proj2>
-  static _LIBCPP_HIDE_FROM_ABI constexpr bool __ends_with_fn_impl_bidirectional(
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool __ends_with_fn_impl_bidirectional(
       _Iter1 __first1,
       _Sent1 __last1,
       _Iter2 __first2,
@@ -56,7 +56,7 @@ struct __fn {
   }
 
   template <class _Iter1, class _Sent1, class _Iter2, class _Sent2, class _Pred, class _Proj1, class _Proj2>
-  static _LIBCPP_HIDE_FROM_ABI constexpr bool __ends_with_fn_impl(
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool __ends_with_fn_impl(
       _Iter1 __first1,
       _Sent1 __last1,
       _Iter2 __first2,
@@ -65,7 +65,7 @@ struct __fn {
       _Proj1& __proj1,
       _Proj2& __proj2) {
     if constexpr (std::bidirectional_iterator<_Sent1> && std::bidirectional_iterator<_Sent2> &&
-                  (!std::random_access_iterator<_Sent1>)&&(!std::random_access_iterator<_Sent2>)) {
+                  (!std::random_access_iterator<_Sent1>) && (!std::random_access_iterator<_Sent2>)) {
       return __ends_with_fn_impl_bidirectional(__first1, __last1, __first2, __last2, __pred, __proj1, __proj2);
 
     } else {
diff --git a/libcxx/include/__algorithm/ranges_starts_with.h b/libcxx/include/__algorithm/ranges_starts_with.h
index 90e184aa9bcc..7ba8af13a8d1 100644
--- a/libcxx/include/__algorithm/ranges_starts_with.h
+++ b/libcxx/include/__algorithm/ranges_starts_with.h
@@ -42,14 +42,14 @@ struct __fn {
             class _Proj1 = identity,
             class _Proj2 = identity>
     requires indirectly_comparable<_Iter1, _Iter2, _Pred, _Proj1, _Proj2>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI static constexpr bool operator()(
       _Iter1 __first1,
       _Sent1 __last1,
       _Iter2 __first2,
       _Sent2 __last2,
       _Pred __pred   = {},
       _Proj1 __proj1 = {},
-      _Proj2 __proj2 = {}) const {
+      _Proj2 __proj2 = {}) {
     return __mismatch::__fn::__go(
                std::move(__first1),
                std::move(__last1),
@@ -67,8 +67,8 @@ struct __fn {
             class _Proj1 = identity,
             class _Proj2 = identity>
     requires indirectly_comparable<iterator_t<_Range1>, iterator_t<_Range2>, _Pred, _Proj1, _Proj2>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
-      _Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI static constexpr bool
+  operator()(_Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) {
     return __mismatch::__fn::__go(
                ranges::begin(__range1),
                ranges::end(__range1),
diff --git a/libcxx/include/__ranges/as_rvalue_view.h b/libcxx/include/__ranges/as_rvalue_view.h
index 295aa94ed9fe..2fc272e798d6 100644
--- a/libcxx/include/__ranges/as_rvalue_view.h
+++ b/libcxx/include/__ranges/as_rvalue_view.h
@@ -111,18 +111,18 @@ namespace views {
 namespace __as_rvalue {
 struct __fn : __range_adaptor_closure<__fn> {
   template <class _Range>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range) const
-      noexcept(noexcept(/**/ as_rvalue_view(std::forward<_Range>(__range))))
-          -> decltype(/*--*/ as_rvalue_view(std::forward<_Range>(__range))) {
-    return /*-------------*/ as_rvalue_view(std::forward<_Range>(__range));
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI static constexpr auto
+  operator()(_Range&& __range) noexcept(noexcept(as_rvalue_view(std::forward<_Range>(__range))))
+      -> decltype(/*--------------------------*/ as_rvalue_view(std::forward<_Range>(__range))) {
+    return /*---------------------------------*/ as_rvalue_view(std::forward<_Range>(__range));
   }
 
   template <class _Range>
     requires same_as<range_rvalue_reference_t<_Range>, range_reference_t<_Range>>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range) const
-      noexcept(noexcept(/**/ views::all(std::forward<_Range>(__range))))
-          -> decltype(/*--*/ views::all(std::forward<_Range>(__range))) {
-    return /*-------------*/ views::all(std::forward<_Range>(__range));
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI static constexpr auto
+  operator()(_Range&& __range) noexcept(noexcept(views::all(std::forward<_Range>(__range))))
+      -> decltype(/*--------------------------*/ views::all(std::forward<_Range>(__range))) {
+    return /*---------------------------------*/ views::all(std::forward<_Range>(__range));
   }
 };
 } // namespace __as_rvalue
diff --git a/libcxx/include/__ranges/repeat_view.h b/libcxx/include/__ranges/repeat_view.h
index 620a26454972..5caea757a393 100644
--- a/libcxx/include/__ranges/repeat_view.h
+++ b/libcxx/include/__ranges/repeat_view.h
@@ -229,14 +229,13 @@ namespace views {
 namespace __repeat {
 struct __fn {
   template <class _Tp>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __value) const
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI static constexpr auto operator()(_Tp&& __value)
     noexcept(noexcept(ranges::repeat_view(std::forward<_Tp>(__value))))
     -> decltype(      ranges::repeat_view(std::forward<_Tp>(__value)))
     { return          ranges::repeat_view(std::forward<_Tp>(__value)); }
 
-
   template <class _Tp, class _Bound>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Tp&& __value, _Bound&& __bound_sentinel) const
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI static constexpr auto operator()(_Tp&& __value, _Bound&& __bound_sentinel)
     noexcept(noexcept(ranges::repeat_view(std::forward<_Tp>(__value), std::forward<_Bound>(__bound_sentinel))))
     -> decltype(      ranges::repeat_view(std::forward<_Tp>(__value), std::forward<_Bound>(__bound_sentinel)))
     { return          ranges::repeat_view(std::forward<_Tp>(__value), std::forward<_Bound>(__bound_sentinel)); }
diff --git a/libcxx/include/__ranges/to.h b/libcxx/include/__ranges/to.h
index cf162100ee46..67818c521b15 100644
--- a/libcxx/include/__ranges/to.h
+++ b/libcxx/include/__ranges/to.h
@@ -207,7 +207,7 @@ _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr auto to(_Args&&... __args)
   static_assert(
       !is_volatile_v<_Container>, "The target container cannot be volatile-qualified, please remove the volatile");
 
-  auto __to_func = []<input_range _Range, class... _Tail>(_Range&& __range, _Tail&&... __tail)
+  auto __to_func = []<input_range _Range, class... _Tail>(_Range&& __range, _Tail&&... __tail) static
     requires requires { //
       /**/ ranges::to<_Container>(std::forward<_Range>(__range), std::forward<_Tail>(__tail)...);
     }
@@ -223,7 +223,7 @@ _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr auto to(_Args&&... __args)
   // clang-format off
   auto __to_func = []<input_range _Range, class... _Tail,
                       class _DeducedExpr = typename _Deducer<_Container, _Range, _Tail...>::type>
-    (_Range&& __range, _Tail&& ... __tail)
+    (_Range&& __range, _Tail&& ... __tail) static
       requires requires { //
       /**/ ranges::to<_DeducedExpr>(std::forward<_Range>(__range), std::forward<_Tail>(__tail)...);
     }
diff --git a/libcxx/include/__ranges/zip_view.h b/libcxx/include/__ranges/zip_view.h
index ce00a4e53a48..d3665a149a7c 100644
--- a/libcxx/include/__ranges/zip_view.h
+++ b/libcxx/include/__ranges/zip_view.h
@@ -489,12 +489,12 @@ namespace views {
 namespace __zip {
 
 struct __fn {
-  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()() const noexcept { return empty_view<tuple<>>{}; }
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto operator()() noexcept { return empty_view<tuple<>>{}; }
 
   template <class... _Ranges>
-  _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Ranges&&... __rs) const
-      noexcept(noexcept(zip_view<all_t<_Ranges&&>...>(std::forward<_Ranges>(__rs)...)))
-          -> decltype(zip_view<all_t<_Ranges&&>...>(std::forward<_Ranges>(__rs)...)) {
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto
+  operator()(_Ranges&&... __rs) noexcept(noexcept(zip_view<all_t<_Ranges&&>...>(std::forward<_Ranges>(__rs)...)))
+      -> decltype(zip_view<all_t<_Ranges&&>...>(std::forward<_Ranges>(__rs)...)) {
     return zip_view<all_t<_Ranges>...>(std::forward<_Ranges>(__rs)...);
   }
 };
-- 
GitLab


From 20e0bacd0560382a31ad0d4ecc7472bd4a99c659 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Fri, 22 Mar 2024 19:08:47 -0500
Subject: [PATCH 029/404] [Libomptarget][Fix] Remove duplicate version script
 for host builds

Summary:
This causes an error on some linkers and was mistakenly kept in.
---
 openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt
index d30680e10431..ccbf7d033fd6 100644
--- a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt
+++ b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt
@@ -31,11 +31,6 @@ else()
   target_include_directories(omptarget.rtl.${machine} PRIVATE dynamic_ffi)
 endif()
 
-if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG)
-  target_link_libraries(omptarget.rtl.${machine} PRIVATE
-    "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports")
-endif()
-
 # Install plugin under the lib destination folder.
 install(TARGETS omptarget.rtl.${machine}
         LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}")
-- 
GitLab


From dc43aa439ecee90a9f51c6c1e46df8be61e0457c Mon Sep 17 00:00:00 2001
From: Charlie Barto <chbarto@microsoft.com>
Date: Fri, 22 Mar 2024 17:38:34 -0700
Subject: [PATCH 030/404] [asan][windows] When compiling with clang-cl or MSVC
 pass /Zl (that's a little L) (#85874)

/Zl is the equivalent of -nodefaultlibs. The idea here is to make sure
that the asan runtime doesn't have any defaultlibs directives, which
makes it easier to mix an asan runtime built with the dynamic CRT with
an application built with the static CRT (or vise-versa).

This is part of the overall effort to remove the static asan runtime on
windows entirely: https://github.com/llvm/llvm-project/pull/81677

Co-authored-by: Amy Wishnousky <amyw@microsoft.com>
---
 compiler-rt/lib/asan/CMakeLists.txt             | 3 +++
 compiler-rt/lib/sanitizer_common/CMakeLists.txt | 2 ++
 compiler-rt/lib/ubsan/CMakeLists.txt            | 1 +
 3 files changed, 6 insertions(+)

diff --git a/compiler-rt/lib/asan/CMakeLists.txt b/compiler-rt/lib/asan/CMakeLists.txt
index f83ae82d4293..601750f72175 100644
--- a/compiler-rt/lib/asan/CMakeLists.txt
+++ b/compiler-rt/lib/asan/CMakeLists.txt
@@ -85,6 +85,9 @@ SET(ASAN_HEADERS
 include_directories(..)
 
 set(ASAN_CFLAGS ${SANITIZER_COMMON_CFLAGS})
+
+append_list_if(MSVC /Zl ASAN_CFLAGS)
+
 set(ASAN_COMMON_DEFINITIONS ${COMPILER_RT_ASAN_SHADOW_SCALE_DEFINITION})
 
 append_rtti_flag(OFF ASAN_CFLAGS)
diff --git a/compiler-rt/lib/sanitizer_common/CMakeLists.txt b/compiler-rt/lib/sanitizer_common/CMakeLists.txt
index f762524c333a..f2b4ac72ae15 100644
--- a/compiler-rt/lib/sanitizer_common/CMakeLists.txt
+++ b/compiler-rt/lib/sanitizer_common/CMakeLists.txt
@@ -218,6 +218,8 @@ include_directories(..)
 set(SANITIZER_COMMON_DEFINITIONS
   HAVE_RPC_XDR_H=${HAVE_RPC_XDR_H})
 
+# note: L not I, this is nodefaultlibs for msvc
+append_list_if(MSVC /Zl SANITIZER_COMMON_CFLAGS)
 set(SANITIZER_CFLAGS ${SANITIZER_COMMON_CFLAGS})
 
 # Too many existing bugs, needs cleanup.
diff --git a/compiler-rt/lib/ubsan/CMakeLists.txt b/compiler-rt/lib/ubsan/CMakeLists.txt
index 3f1e12ed9ac6..db0b33f1276e 100644
--- a/compiler-rt/lib/ubsan/CMakeLists.txt
+++ b/compiler-rt/lib/ubsan/CMakeLists.txt
@@ -41,6 +41,7 @@ set(UBSAN_HEADERS
 include_directories(..)
 
 set(UBSAN_CFLAGS ${SANITIZER_COMMON_CFLAGS})
+append_list_if(MSVC /Zl UBSAN_CFLAGS)
 append_rtti_flag(OFF UBSAN_CFLAGS)
 append_list_if(SANITIZER_CAN_USE_CXXABI -DUBSAN_CAN_USE_CXXABI UBSAN_CFLAGS)
 
-- 
GitLab


From b723c57f8fdb12ca8eaa6c5ee2afb820aaeb56c5 Mon Sep 17 00:00:00 2001
From: Corentin Jabot <corentinjabot@gmail.com>
Date: Sat, 23 Mar 2024 09:28:33 +0900
Subject: [PATCH 031/404] [Clang] Update the C++ page with papers approved in
 Tokyo

---
 clang/www/cxx_status.html | 42 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 41 insertions(+), 1 deletion(-)

diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index 1e36b90356c3..c1d95dadbb27 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -163,7 +163,47 @@ C++23, informally referred to as C++26.</p>
   <td><a href="https://wg21.link/P2864R2">P2864R2</a></td>
   <td class="full" align="center">Clang 18</td>
  </tr>
-
+ <!-- Winter 2024 papers (Tokyo) -->
+ <tr>
+  <td>Disallow Binding a Returned Glvalue to a Temporary</td>
+  <td><a href="https://wg21.link/P2748R5">P2748R5</a></td>
+  <td class="none" align="center">No</td>
+ </tr>
+ <tr>
+  <td>Clarifying rules for brace elision in aggregate initialization</td>
+  <td><a href="https://wg21.link/P3106R1">P3106R1</a> (<a href="#dr">DR</a>)</td>
+  <td class="none" align="center">No</td>
+ </tr>
+ <tr>
+  <td>Attributes for Structured Bindings</td>
+  <td><a href="https://wg21.link/P0609R3">P0609R3</a></td>
+  <td class="none" align="center">No</td>
+ </tr>
+ <tr>
+  <td>Module Declarations Shouldn’t be Macros</td>
+  <td><a href="https://wg21.link/P3034R1">P3034R1</a> (<a href="#dr">DR</a>)</td>
+  <td class="none" align="center">No</td>
+ </tr>
+ <tr>
+  <td>Trivial infinite loops are not Undefined Behavior</td>
+  <td><a href="https://wg21.link/P2809R3">P2809R3</a> (<a href="#dr">DR</a>)</td>
+  <td class="none" align="center">No</td>
+ </tr>
+ <tr>
+  <td>Erroneous behaviour for uninitialized reads</td>
+  <td><a href="https://wg21.link/P2795R5">P2795R5</a></td>
+  <td class="none" align="center">No</td>
+ </tr>
+ <tr>
+  <td><tt>= delete("should have a reason");</tt></td>
+  <td><a href="https://wg21.link/P2573R2">P2573R2</a></td>
+  <td class="none" align="center">No</td>
+ </tr>
+ <tr>
+  <td>Variadic friends</td>
+  <td><a href="https://wg21.link/P2893R3">P2893R3</a></td>
+  <td class="none" align="center">No</td>
+ </tr>
 </table>
 </details>
 
-- 
GitLab


From 2f6b1b4b30e3a719b1744baa4cd1ece504998c6e Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Fri, 22 Mar 2024 18:10:21 -0700
Subject: [PATCH 032/404] [ORC] Add default visibility to required JIT
 functions (#86322)

If you build LLVM with `-DCMAKE_CXX_VISIBILITY_PRESET=hidden` to help
reduce binary size, these symbols end up becoming local, and getting
stripped. This forces default visibility to override the global setting
in that case.

Relevant:
https://github.com/llvm/llvm-project/issues/62815#issuecomment-1560078260
---
 llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp
index 8a4145a6b02a..7529d9cef67e 100644
--- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp
@@ -26,11 +26,13 @@ extern "C" {
 // We put information about the JITed function in this global, which the
 // debugger reads.  Make sure to specify the version statically, because the
 // debugger checks the version before we can set it during runtime.
+LLVM_ATTRIBUTE_VISIBILITY_DEFAULT
 struct jit_descriptor __jit_debug_descriptor = {JitDescriptorVersion, 0,
                                                 nullptr, nullptr};
 
 // Debuggers that implement the GDB JIT interface put a special breakpoint in
 // this function.
+LLVM_ATTRIBUTE_VISIBILITY_DEFAULT
 LLVM_ATTRIBUTE_NOINLINE void __jit_debug_register_code() {
   // The noinline and the asm prevent calls to this function from being
   // optimized out.
-- 
GitLab


From 84e20cb0cbc64bb2d8299b30da3ce8e6421c203e Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 22 Mar 2024 16:56:48 -0700
Subject: [PATCH 033/404] [TableGen] Minor cleanup in
 CodeGenRegBank::getConcatSubRegIndex. NFC

Mark a variable const. Capitalize a variable name.

I'm going to add HwMode support to this code and wanted to clean it
up a bit beforehand.
---
 llvm/utils/TableGen/CodeGenRegisters.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/utils/TableGen/CodeGenRegisters.cpp b/llvm/utils/TableGen/CodeGenRegisters.cpp
index 40af0d3077b2..e851d4c16bf7 100644
--- a/llvm/utils/TableGen/CodeGenRegisters.cpp
+++ b/llvm/utils/TableGen/CodeGenRegisters.cpp
@@ -1400,11 +1400,11 @@ CodeGenSubRegIndex *CodeGenRegBank::getConcatSubRegIndex(
   // None exists, synthesize one.
   std::string Name = Parts.front()->getName();
   // Determine whether all parts are contiguous.
-  bool isContinuous = true;
+  bool IsContinuous = true;
   unsigned Size = Parts.front()->Size;
   unsigned LastOffset = Parts.front()->Offset;
   unsigned LastSize = Parts.front()->Size;
-  unsigned UnknownSize = (uint16_t)-1;
+  const unsigned UnknownSize = (uint16_t)-1;
   for (unsigned i = 1, e = Parts.size(); i != e; ++i) {
     Name += '_';
     Name += Parts[i]->getName();
@@ -1413,13 +1413,13 @@ CodeGenSubRegIndex *CodeGenRegBank::getConcatSubRegIndex(
     else
       Size += Parts[i]->Size;
     if (LastSize == UnknownSize || Parts[i]->Offset != (LastOffset + LastSize))
-      isContinuous = false;
+      IsContinuous = false;
     LastOffset = Parts[i]->Offset;
     LastSize = Parts[i]->Size;
   }
   Idx = createSubRegIndex(Name, Parts.front()->getNamespace());
   Idx->Size = Size;
-  Idx->Offset = isContinuous ? Parts.front()->Offset : -1;
+  Idx->Offset = IsContinuous ? Parts.front()->Offset : -1;
   Idx->ConcatenationOf.assign(Parts.begin(), Parts.end());
   return Idx;
 }
-- 
GitLab


From 76fdb5902fbadbc08c6742156071431d8ad801ea Mon Sep 17 00:00:00 2001
From: paperchalice <liujunchang97@outlook.com>
Date: Sat, 23 Mar 2024 10:37:53 +0800
Subject: [PATCH 034/404] [NewPM][DirectX] Add DirectXPassRegistry.def NFCI
 (#86242)

Prepare migration for dag-isel
---
 .../Target/DirectX/DirectXPassRegistry.def    | 29 +++++++++++++++++++
 .../Target/DirectX/DirectXTargetMachine.cpp   | 20 ++-----------
 2 files changed, 31 insertions(+), 18 deletions(-)
 create mode 100644 llvm/lib/Target/DirectX/DirectXPassRegistry.def

diff --git a/llvm/lib/Target/DirectX/DirectXPassRegistry.def b/llvm/lib/Target/DirectX/DirectXPassRegistry.def
new file mode 100644
index 000000000000..bbf0c254bb69
--- /dev/null
+++ b/llvm/lib/Target/DirectX/DirectXPassRegistry.def
@@ -0,0 +1,29 @@
+//===- DirectXPassRegistry.def - Registry of DirectX passes -----*- C++--*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is used as the registry of passes that are part of the
+// DirectX backend.
+//
+//===----------------------------------------------------------------------===//
+
+// NOTE: NO INCLUDE GUARD DESIRED!
+
+#ifndef MODULE_ANALYSIS
+#define MODULE_ANALYSIS(NAME, CREATE_PASS)
+#endif
+MODULE_ANALYSIS("dx-shader-flags", dxil::ShaderFlagsAnalysis())
+MODULE_ANALYSIS("dxil-resource", DXILResourceAnalysis())
+#undef MODULE_ANALYSIS
+
+#ifndef MODULE_PASS
+#define MODULE_PASS(NAME, CREATE_PASS)
+#endif
+// TODO: rename to print<foo> after NPM switch
+MODULE_PASS("print-dx-shader-flags", DXILResourcePrinterPass(dbgs()))
+MODULE_PASS("print-dxil-resource", DXILResourcePrinterPass(dbgs()))
+#undef MODULE_PASS
diff --git a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
index 03c825b3977d..bebca0675522 100644
--- a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
+++ b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
@@ -104,24 +104,8 @@ DirectXTargetMachine::~DirectXTargetMachine() {}
 
 void DirectXTargetMachine::registerPassBuilderCallbacks(
     PassBuilder &PB, bool PopulateClassToPassNames) {
-  PB.registerPipelineParsingCallback(
-      [](StringRef PassName, ModulePassManager &PM,
-         ArrayRef<PassBuilder::PipelineElement>) {
-        if (PassName == "print-dxil-resource") {
-          PM.addPass(DXILResourcePrinterPass(dbgs()));
-          return true;
-        }
-        if (PassName == "print-dx-shader-flags") {
-          PM.addPass(dxil::ShaderFlagsAnalysisPrinter(dbgs()));
-          return true;
-        }
-        return false;
-      });
-
-  PB.registerAnalysisRegistrationCallback([](ModuleAnalysisManager &MAM) {
-    MAM.registerPass([&] { return DXILResourceAnalysis(); });
-    MAM.registerPass([&] { return dxil::ShaderFlagsAnalysis(); });
-  });
+#define GET_PASS_REGISTRY "DirectXPassRegistry.def"
+#include "llvm/Passes/TargetPassRegistry.inc"
 }
 
 bool DirectXTargetMachine::addPassesToEmitFile(
-- 
GitLab


From 7ac7d418ac2b16fd44789dcf48e2b5d73de3e715 Mon Sep 17 00:00:00 2001
From: paperchalice <liujunchang97@outlook.com>
Date: Sat, 23 Mar 2024 11:20:18 +0800
Subject: [PATCH 035/404] [NewPM][NVPTX] Add NVPTXPassRegistry.def NFCI
 (#86246)

Prepare for dag-isel migration.
---
 llvm/lib/Target/NVPTX/NVPTXPassRegistry.def  | 40 +++++++++++++++++++
 llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp | 41 +-------------------
 2 files changed, 42 insertions(+), 39 deletions(-)
 create mode 100644 llvm/lib/Target/NVPTX/NVPTXPassRegistry.def

diff --git a/llvm/lib/Target/NVPTX/NVPTXPassRegistry.def b/llvm/lib/Target/NVPTX/NVPTXPassRegistry.def
new file mode 100644
index 000000000000..6ff15ab6f13c
--- /dev/null
+++ b/llvm/lib/Target/NVPTX/NVPTXPassRegistry.def
@@ -0,0 +1,40 @@
+//===- NVPTXPassRegistry.def - Registry of NVPTX passes ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is used as the registry of passes that are part of the
+// NVPTX backend.
+//
+//===----------------------------------------------------------------------===//
+
+// NOTE: NO INCLUDE GUARD DESIRED!
+
+#ifndef MODULE_PASS
+#define MODULE_PASS(NAME, CREATE_PASS)
+#endif
+MODULE_PASS("generic-to-nvvm", GenericToNVVMPass())
+MODULE_PASS("nvptx-lower-ctor-dtor", NVPTXCtorDtorLoweringPass())
+#undef MODULE_PASS
+
+#ifndef FUNCTION_ANALYSIS
+#define FUNCTION_ANALYSIS(NAME, CREATE_PASS)
+#endif
+
+#ifndef FUNCTION_ALIAS_ANALYSIS
+#define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS)                             \
+  FUNCTION_ANALYSIS(NAME, CREATE_PASS)
+#endif
+FUNCTION_ALIAS_ANALYSIS("nvptx-aa", NVPTXAA())
+#undef FUNCTION_ALIAS_ANALYSIS
+#undef FUNCTION_ANALYSIS
+
+#ifndef FUNCTION_PASS
+#define FUNCTION_PASS(NAME, CREATE_PASS)
+#endif
+FUNCTION_PASS("nvvm-intr-range", NVVMIntrRangePass())
+FUNCTION_PASS("nvvm-reflect", NVVMReflectPass())
+#undef FUNCTION_PASS
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
index 69d4596f7843..78f48652c992 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
@@ -227,45 +227,8 @@ void NVPTXTargetMachine::registerDefaultAliasAnalyses(AAManager &AAM) {
 
 void NVPTXTargetMachine::registerPassBuilderCallbacks(
     PassBuilder &PB, bool PopulateClassToPassNames) {
-  PB.registerPipelineParsingCallback(
-      [](StringRef PassName, FunctionPassManager &PM,
-         ArrayRef<PassBuilder::PipelineElement>) {
-        if (PassName == "nvvm-reflect") {
-          PM.addPass(NVVMReflectPass());
-          return true;
-        }
-        if (PassName == "nvvm-intr-range") {
-          PM.addPass(NVVMIntrRangePass());
-          return true;
-        }
-        return false;
-      });
-
-  PB.registerAnalysisRegistrationCallback([](FunctionAnalysisManager &FAM) {
-    FAM.registerPass([&] { return NVPTXAA(); });
-  });
-
-  PB.registerParseAACallback([](StringRef AAName, AAManager &AAM) {
-    if (AAName == "nvptx-aa") {
-      AAM.registerFunctionAnalysis<NVPTXAA>();
-      return true;
-    }
-    return false;
-  });
-
-  PB.registerPipelineParsingCallback(
-      [](StringRef PassName, ModulePassManager &PM,
-         ArrayRef<PassBuilder::PipelineElement>) {
-        if (PassName == "nvptx-lower-ctor-dtor") {
-          PM.addPass(NVPTXCtorDtorLoweringPass());
-          return true;
-        }
-        if (PassName == "generic-to-nvvm") {
-          PM.addPass(GenericToNVVMPass());
-          return true;
-        }
-        return false;
-      });
+#define GET_PASS_REGISTRY "NVPTXPassRegistry.def"
+#include "llvm/Passes/TargetPassRegistry.inc"
 
   PB.registerPipelineStartEPCallback(
       [this](ModulePassManager &PM, OptimizationLevel Level) {
-- 
GitLab


From 2aa5bae0c03f1f857d0ae2a881b223c4a521853f Mon Sep 17 00:00:00 2001
From: paperchalice <liujunchang97@outlook.com>
Date: Sat, 23 Mar 2024 12:53:26 +0800
Subject: [PATCH 036/404] [NewPM][BPF] Add BPFPassRegistry.def NFCI (#86241)

Prepare migration for dag-isel.
---
 llvm/lib/Target/BPF/BPFPassRegistry.def  | 32 ++++++++++++++++++++++++
 llvm/lib/Target/BPF/BPFTargetMachine.cpp | 25 ++++++------------
 2 files changed, 40 insertions(+), 17 deletions(-)
 create mode 100644 llvm/lib/Target/BPF/BPFPassRegistry.def

diff --git a/llvm/lib/Target/BPF/BPFPassRegistry.def b/llvm/lib/Target/BPF/BPFPassRegistry.def
new file mode 100644
index 000000000000..73a8ef2f95bf
--- /dev/null
+++ b/llvm/lib/Target/BPF/BPFPassRegistry.def
@@ -0,0 +1,32 @@
+//===- BPFPassRegistry.def - Registry of BPF passes -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is used as the registry of passes that are part of the
+// BPF backend.
+//
+//===----------------------------------------------------------------------===//
+
+// NOTE: NO INCLUDE GUARD DESIRED!
+
+#ifndef FUNCTION_PASS
+#define FUNCTION_PASS(NAME, CREATE_PASS)
+#endif
+FUNCTION_PASS("bpf-aspace-simplify", BPFASpaceCastSimplifyPass())
+FUNCTION_PASS("bpf-ir-peephole", BPFIRPeepholePass())
+#undef FUNCTION_PASS
+
+#ifndef FUNCTION_PASS_WITH_PARAMS
+#define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS)
+#endif
+FUNCTION_PASS_WITH_PARAMS(
+    "bpf-preserve-static-offset", "BPFPreserveStaticOffsetPass",
+    [=](bool AllowPartial) {
+      return BPFPreserveStaticOffsetPass(AllowPartial);
+    },
+    parseBPFPreserveStaticOffsetOptions, "allow-partial")
+#undef FUNCTION_PASS_WITH_PARAMS
diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp
index 5f26bec2e390..a7bed69b0f2a 100644
--- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp
+++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp
@@ -108,25 +108,16 @@ TargetPassConfig *BPFTargetMachine::createPassConfig(PassManagerBase &PM) {
   return new BPFPassConfig(*this, PM);
 }
 
+static Expected<bool> parseBPFPreserveStaticOffsetOptions(StringRef Params) {
+  return PassBuilder::parseSinglePassOption(Params, "allow-partial",
+                                            "BPFPreserveStaticOffsetPass");
+}
+
 void BPFTargetMachine::registerPassBuilderCallbacks(
     PassBuilder &PB, bool PopulateClassToPassNames) {
-  PB.registerPipelineParsingCallback(
-      [](StringRef PassName, FunctionPassManager &FPM,
-         ArrayRef<PassBuilder::PipelineElement>) {
-        if (PassName == "bpf-ir-peephole") {
-          FPM.addPass(BPFIRPeepholePass());
-          return true;
-        }
-        if (PassName == "bpf-preserve-static-offset") {
-          FPM.addPass(BPFPreserveStaticOffsetPass(false));
-          return true;
-        }
-        if (PassName == "bpf-aspace-simplify") {
-          FPM.addPass(BPFASpaceCastSimplifyPass());
-          return true;
-        }
-        return false;
-      });
+#define GET_PASS_REGISTRY "BPFPassRegistry.def"
+#include "llvm/Passes/TargetPassRegistry.inc"
+
   PB.registerPipelineStartEPCallback(
       [=](ModulePassManager &MPM, OptimizationLevel) {
         FunctionPassManager FPM;
-- 
GitLab


From 635ea257eca7c8e95c6ea30ca3816a0b5584ab37 Mon Sep 17 00:00:00 2001
From: paperchalice <liujunchang97@outlook.com>
Date: Sat, 23 Mar 2024 13:06:58 +0800
Subject: [PATCH 037/404] [NewPM] Fix BPF build (#86379)

Add Passes in dependency list
---
 llvm/lib/Target/BPF/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt
index cb21ed03a86c..eade4cacb710 100644
--- a/llvm/lib/Target/BPF/CMakeLists.txt
+++ b/llvm/lib/Target/BPF/CMakeLists.txt
@@ -54,6 +54,7 @@ add_llvm_target(BPFCodeGen
   GlobalISel
   IPO
   MC
+  Passes
   Scalar
   SelectionDAG
   Support
-- 
GitLab


From 6c1932ffd82e733325180fe13ef46b24ff606eab Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Sat, 23 Mar 2024 14:57:35 +0800
Subject: [PATCH 038/404] [LLVM] Pass APInt by const reference. NFC. (#86278)

This patch adjusts argument passing for `APInt` to improve the
compile-time.
Compile-time improvement:
https://llvm-compile-time-tracker.com/compare.php?from=d1f182c895728d89c5c3d198b133e212a5d9d4a3&to=32d6611af69bf4e76373f9bc7d9649650f760e48&stat=instructions:u
---
 llvm/include/llvm/Analysis/MemoryBuiltins.h     | 2 +-
 llvm/include/llvm/CodeGen/SelectionDAG.h        | 2 +-
 llvm/include/llvm/MC/MCStreamer.h               | 2 +-
 llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp | 2 +-
 llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp  | 3 ++-
 llvm/lib/MC/MCStreamer.cpp                      | 2 +-
 6 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/Analysis/MemoryBuiltins.h b/llvm/include/llvm/Analysis/MemoryBuiltins.h
index 37ce1518f00c..da092866de65 100644
--- a/llvm/include/llvm/Analysis/MemoryBuiltins.h
+++ b/llvm/include/llvm/Analysis/MemoryBuiltins.h
@@ -217,7 +217,7 @@ struct SizeOffsetAPInt : public SizeOffsetType<APInt, SizeOffsetAPInt> {
   SizeOffsetAPInt() = default;
   SizeOffsetAPInt(APInt Size, APInt Offset) : SizeOffsetType(Size, Offset) {}
 
-  static bool known(APInt V) { return V.getBitWidth() > 1; }
+  static bool known(const APInt &V) { return V.getBitWidth() > 1; }
 };
 
 /// Evaluate the size and offset of an object pointed to by a Value*
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 4785e93d72d1..574c63552ce0 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -883,7 +883,7 @@ public:
 
   /// Returns a vector of type ResVT whose elements contain the linear sequence
   ///   <0, Step, Step * 2, Step * 3, ...>
-  SDValue getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal);
+  SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal);
 
   /// Returns a vector of type ResVT whose elements contain the linear sequence
   ///   <0, 1, 2, 3, ...>
diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h
index 671511ab4b88..69867620e1bf 100644
--- a/llvm/include/llvm/MC/MCStreamer.h
+++ b/llvm/include/llvm/MC/MCStreamer.h
@@ -740,7 +740,7 @@ public:
   /// Special case of EmitValue that avoids the client having
   /// to pass in a MCExpr for constant integers.
   virtual void emitIntValue(uint64_t Value, unsigned Size);
-  virtual void emitIntValue(APInt Value);
+  virtual void emitIntValue(const APInt &Value);
 
   /// Special case of EmitValue that avoids the client having to pass
   /// in a MCExpr for constant integers & prints in Hex format for certain
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 8d608f6ac5e4..1b25da8833e4 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -7855,7 +7855,7 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerBswap(MachineInstr &MI) {
 
 //{ (Src & Mask) >> N } | { (Src << N) & Mask }
 static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B,
-                                 MachineInstrBuilder Src, APInt Mask) {
+                                 MachineInstrBuilder Src, const APInt &Mask) {
   const LLT Ty = Dst.getLLTTy(*B.getMRI());
   MachineInstrBuilder C_N = B.buildConstant(Ty, N);
   MachineInstrBuilder MaskLoNTo0 = B.buildConstant(Ty, Mask);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index cd6f083243d0..e2c07e7cb997 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -2031,7 +2031,8 @@ SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
   return getStepVector(DL, ResVT, One);
 }
 
-SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal) {
+SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT,
+                                    const APInt &StepVal) {
   assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
   if (ResVT.isScalableVector())
     return getNode(
diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp
index d0395770ae8b..176d55aa890b 100644
--- a/llvm/lib/MC/MCStreamer.cpp
+++ b/llvm/lib/MC/MCStreamer.cpp
@@ -141,7 +141,7 @@ void MCStreamer::emitIntValue(uint64_t Value, unsigned Size) {
   unsigned Index = IsLittleEndian ? 0 : 8 - Size;
   emitBytes(StringRef(reinterpret_cast<char *>(&Swapped) + Index, Size));
 }
-void MCStreamer::emitIntValue(APInt Value) {
+void MCStreamer::emitIntValue(const APInt &Value) {
   if (Value.getNumWords() == 1) {
     emitIntValue(Value.getLimitedValue(), Value.getBitWidth() / 8);
     return;
-- 
GitLab


From 2f1f6b704d83f87be7ea885480caf1c86d8cfaee Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Sat, 23 Mar 2024 14:58:25 +0800
Subject: [PATCH 039/404] [LLVM] Use `std::move` for APInt. NFC. (#86257)

This patch adjusts argument passing for `APInt` to improve the
compile-time.
Compile-time improvement:
https://llvm-compile-time-tracker.com/compare.php?from=d1f182c895728d89c5c3d198b133e212a5d9d4a3&to=ba3e326def3a6e5cd6d72ff5a49c74fba18de1df&stat=instructions:u
---
 llvm/include/llvm/Analysis/InlineCost.h     | 3 ++-
 llvm/include/llvm/Analysis/MemoryBuiltins.h | 6 ++++--
 llvm/lib/Analysis/ConstantFolding.cpp       | 2 +-
 llvm/lib/Analysis/InstructionSimplify.cpp   | 6 ++++--
 llvm/lib/IR/LLVMContextImpl.h               | 2 +-
 llvm/lib/Transforms/Scalar/MergeICmps.cpp   | 2 +-
 6 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h
index 3a760e0a85ce..c5978ce54fc1 100644
--- a/llvm/include/llvm/Analysis/InlineCost.h
+++ b/llvm/include/llvm/Analysis/InlineCost.h
@@ -65,7 +65,8 @@ const char MaxInlineStackSizeAttributeName[] = "inline-max-stacksize";
 // The cost-benefit pair computed by cost-benefit analysis.
 class CostBenefitPair {
 public:
-  CostBenefitPair(APInt Cost, APInt Benefit) : Cost(Cost), Benefit(Benefit) {}
+  CostBenefitPair(APInt Cost, APInt Benefit)
+      : Cost(std::move(Cost)), Benefit(std::move(Benefit)) {}
 
   const APInt &getCost() const { return Cost; }
 
diff --git a/llvm/include/llvm/Analysis/MemoryBuiltins.h b/llvm/include/llvm/Analysis/MemoryBuiltins.h
index da092866de65..bb282a1b73d3 100644
--- a/llvm/include/llvm/Analysis/MemoryBuiltins.h
+++ b/llvm/include/llvm/Analysis/MemoryBuiltins.h
@@ -196,7 +196,8 @@ public:
   T Offset;
 
   SizeOffsetType() = default;
-  SizeOffsetType(T Size, T Offset) : Size(Size), Offset(Offset) {}
+  SizeOffsetType(T Size, T Offset)
+      : Size(std::move(Size)), Offset(std::move(Offset)) {}
 
   bool knownSize() const { return C::known(Size); }
   bool knownOffset() const { return C::known(Offset); }
@@ -215,7 +216,8 @@ public:
 /// \p APInts.
 struct SizeOffsetAPInt : public SizeOffsetType<APInt, SizeOffsetAPInt> {
   SizeOffsetAPInt() = default;
-  SizeOffsetAPInt(APInt Size, APInt Offset) : SizeOffsetType(Size, Offset) {}
+  SizeOffsetAPInt(APInt Size, APInt Offset)
+      : SizeOffsetType(std::move(Size), std::move(Offset)) {}
 
   static bool known(const APInt &V) { return V.getBitWidth() > 1; }
 };
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 6139b5be85be..749374a3aa48 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -751,7 +751,7 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty,
 Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty,
                                              const DataLayout &DL) {
   APInt Offset(DL.getIndexTypeSizeInBits(C->getType()), 0);
-  return ConstantFoldLoadFromConstPtr(C, Ty, Offset, DL);
+  return ConstantFoldLoadFromConstPtr(C, Ty, std::move(Offset), DL);
 }
 
 Constant *llvm::ConstantFoldLoadFromUniformValue(Constant *C, Type *Ty,
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 7a37ae86c7f3..9ff3faff7990 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -6115,7 +6115,8 @@ static Value *simplifyRelativeLoad(Constant *Ptr, Constant *Offset,
   if (OffsetInt.srem(4) != 0)
     return nullptr;
 
-  Constant *Loaded = ConstantFoldLoadFromConstPtr(Ptr, Int32Ty, OffsetInt, DL);
+  Constant *Loaded =
+      ConstantFoldLoadFromConstPtr(Ptr, Int32Ty, std::move(OffsetInt), DL);
   if (!Loaded)
     return nullptr;
 
@@ -6983,7 +6984,8 @@ Value *llvm::simplifyLoadInst(LoadInst *LI, Value *PtrOp,
   if (PtrOp == GV) {
     // Index size may have changed due to address space casts.
     Offset = Offset.sextOrTrunc(Q.DL.getIndexTypeSizeInBits(PtrOp->getType()));
-    return ConstantFoldLoadFromConstPtr(GV, LI->getType(), Offset, Q.DL);
+    return ConstantFoldLoadFromConstPtr(GV, LI->getType(), std::move(Offset),
+                                        Q.DL);
   }
 
   return nullptr;
diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h
index 58e0f21244f7..7c67e191348e 100644
--- a/llvm/lib/IR/LLVMContextImpl.h
+++ b/llvm/lib/IR/LLVMContextImpl.h
@@ -441,7 +441,7 @@ template <> struct MDNodeKeyImpl<DIEnumerator> {
   bool IsUnsigned;
 
   MDNodeKeyImpl(APInt Value, bool IsUnsigned, MDString *Name)
-      : Value(Value), Name(Name), IsUnsigned(IsUnsigned) {}
+      : Value(std::move(Value)), Name(Name), IsUnsigned(IsUnsigned) {}
   MDNodeKeyImpl(int64_t Value, bool IsUnsigned, MDString *Name)
       : Value(APInt(64, Value, !IsUnsigned)), Name(Name),
         IsUnsigned(IsUnsigned) {}
diff --git a/llvm/lib/Transforms/Scalar/MergeICmps.cpp b/llvm/lib/Transforms/Scalar/MergeICmps.cpp
index 1e0906717549..2bd13556c696 100644
--- a/llvm/lib/Transforms/Scalar/MergeICmps.cpp
+++ b/llvm/lib/Transforms/Scalar/MergeICmps.cpp
@@ -74,7 +74,7 @@ namespace {
 struct BCEAtom {
   BCEAtom() = default;
   BCEAtom(GetElementPtrInst *GEP, LoadInst *LoadI, int BaseId, APInt Offset)
-      : GEP(GEP), LoadI(LoadI), BaseId(BaseId), Offset(Offset) {}
+      : GEP(GEP), LoadI(LoadI), BaseId(BaseId), Offset(std::move(Offset)) {}
 
   BCEAtom(const BCEAtom &) = delete;
   BCEAtom &operator=(const BCEAtom &) = delete;
-- 
GitLab


From ef57977f2aa32661a09fa6538f47ddee0a004d11 Mon Sep 17 00:00:00 2001
From: paperchalice <liujunchang97@outlook.com>
Date: Sat, 23 Mar 2024 15:02:27 +0800
Subject: [PATCH 040/404] [NewPM][Hexagon] Add HexagonPassRegistry.def (#86244)

Prepare for dag-isel, also migrate some test case
---
 .../Target/Hexagon/HexagonPassRegistry.def    | 21 +++++++++++++++++++
 .../Target/Hexagon/HexagonTargetMachine.cpp   |  3 +++
 .../hexagon_vector_loop_carried_reuse.ll      |  2 +-
 ...n_vector_loop_carried_reuse_commutative.ll |  2 +-
 ...agon_vector_loop_carried_reuse_constant.ll |  2 +-
 ...xagon_vector_loop_carried_reuse_invalid.ll |  2 +-
 .../CodeGen/Hexagon/hvx-loopidiom-memcpy.ll   |  1 +
 .../Hexagon/loop-idiom/hexagon-memmove1.ll    |  2 ++
 .../Hexagon/loop-idiom/hexagon-memmove2.ll    |  2 ++
 llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll |  2 +-
 .../Hexagon/loop-idiom/memmove-rt-check.ll    |  1 +
 .../Hexagon/loop-idiom/nullptr-crash.ll       |  1 +
 .../Hexagon/loop-idiom/pmpy-infinite-loop.ll  |  1 +
 .../Hexagon/loop-idiom/pmpy-long-loop.ll      |  1 +
 .../Hexagon/loop-idiom/pmpy-shiftconv-fail.ll |  1 +
 llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll  |  2 ++
 16 files changed, 41 insertions(+), 5 deletions(-)
 create mode 100644 llvm/lib/Target/Hexagon/HexagonPassRegistry.def

diff --git a/llvm/lib/Target/Hexagon/HexagonPassRegistry.def b/llvm/lib/Target/Hexagon/HexagonPassRegistry.def
new file mode 100644
index 000000000000..4f58ae6193e0
--- /dev/null
+++ b/llvm/lib/Target/Hexagon/HexagonPassRegistry.def
@@ -0,0 +1,21 @@
+//===- HexagonPassRegistry.def - Registry of Hexagon passes -----*- C++--*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is used as the registry of passes that are part of the
+// Hexagon backend.
+//
+//===----------------------------------------------------------------------===//
+
+// NOTE: NO INCLUDE GUARD DESIRED!
+
+#ifndef LOOP_PASS
+#define LOOP_PASS(NAME, CREATE_PASS)
+#endif
+LOOP_PASS("hexagon-loop-idiom", HexagonLoopIdiomRecognitionPass())
+LOOP_PASS("hexagon-vlcr", HexagonVectorLoopCarriedReusePass())
+#undef LOOP_PASS
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp
index 3c346c334d6d..e64d7e52a9aa 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp
@@ -299,6 +299,9 @@ HexagonTargetMachine::getSubtargetImpl(const Function &F) const {
 
 void HexagonTargetMachine::registerPassBuilderCallbacks(
     PassBuilder &PB, bool PopulateClassToPassNames) {
+#define GET_PASS_REGISTRY "HexagonPassRegistry.def"
+#include "llvm/Passes/TargetPassRegistry.inc"
+
   PB.registerLateLoopOptimizationsEPCallback(
       [=](LoopPassManager &LPM, OptimizationLevel Level) {
         LPM.addPass(HexagonLoopIdiomRecognitionPass());
diff --git a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll
index 0771fda02cfb..7ccee1689185 100644
--- a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll
+++ b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -hexagon-vlcr | opt -passes=adce -S | FileCheck %s
+; RUN: opt -mtriple=hexagon-- -passes='loop(hexagon-vlcr),adce' -S %s | FileCheck %s
 
 ; CHECK: %.hexagon.vlcr = tail call <32 x i32> @llvm.hexagon.V6.vmaxub.128B
 ; ModuleID = 'hexagon_vector_loop_carried_reuse.c'
diff --git a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_commutative.ll b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_commutative.ll
index 25afb9f1a137..532f7fd06793 100644
--- a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_commutative.ll
+++ b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_commutative.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -march=hexagon -hexagon-vlcr | opt -passes=adce -S | FileCheck %s
+; RUN: opt -mtriple hexagon-- -passes='loop(hexagon-vlcr),adce' -S %s | FileCheck %s
 
 ; CHECK: %v32.hexagon.vlcr = tail call <32 x i32> @llvm.hexagon.V6.vmaxub.128B
 
diff --git a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll
index 53973423732c..ecfcf53d9133 100644
--- a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll
+++ b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_constant.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -hexagon-vlcr | opt -passes=adce -S | FileCheck %s
+; RUN: opt -mtriple=hexagon-- -passes='loop(hexagon-vlcr),adce' -S %s | FileCheck %s
 
 ; CHECK-NOT: %.hexagon.vlcr
 ; ModuleID = 'hexagon_vector_loop_carried_reuse.c'
diff --git a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_invalid.ll b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_invalid.ll
index b440dba66f67..9872faeb3da0 100644
--- a/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_invalid.ll
+++ b/llvm/test/CodeGen/Hexagon/hexagon_vector_loop_carried_reuse_invalid.ll
@@ -1,4 +1,4 @@
-; RUN: opt -hexagon-vlcr < %s -S | FileCheck %s
+; RUN: opt -mtriple=hexagon-- -passes=hexagon-vlcr -S %s | FileCheck %s
 
 ; Test that reuse doesn't occur due to two shufflevectors with different masks.
 
diff --git a/llvm/test/CodeGen/Hexagon/hvx-loopidiom-memcpy.ll b/llvm/test/CodeGen/Hexagon/hvx-loopidiom-memcpy.ll
index ab7bf1b4b0e8..c53e57800edd 100644
--- a/llvm/test/CodeGen/Hexagon/hvx-loopidiom-memcpy.ll
+++ b/llvm/test/CodeGen/Hexagon/hvx-loopidiom-memcpy.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -march=hexagon -hexagon-loop-idiom -S < %s | FileCheck %s
+; RUN: opt -mtriple=hexagon-- -p hexagon-loop-idiom -disable-memcpy-idiom -S < %s | FileCheck %s
 
 ; Make sure we don't convert load/store loops into memcpy if the access type
 ; is a vector. Using vector instructions is generally better in such cases.
diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll
index c7110263c658..5ace9e6ee486 100644
--- a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll
+++ b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll
@@ -1,6 +1,8 @@
 ; Check for recognizing the "memmove" idiom.
 ; RUN: opt -hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \
 ; RUN:  | FileCheck %s
+; RUN: opt -p hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \
+; RUN:  | FileCheck %s
 ; CHECK: call void @llvm.memmove
 
 ; Function Attrs: norecurse nounwind
diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll
index 234e4f56b5d8..ed56a332f657 100644
--- a/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll
+++ b/llvm/test/CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll
@@ -1,5 +1,7 @@
 ; RUN: opt -hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \
 ; RUN:  | FileCheck %s
+; RUN: opt -p hexagon-loop-idiom -S -mtriple hexagon-unknown-elf < %s \
+; RUN:  | FileCheck %s
 
 define void @PR14241(ptr %s, i64 %size) #0 {
 ; Ensure that we don't form a memcpy for strided loops. Briefly, when we taught
diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll
index 140c676175ea..e5bcc2b9aebf 100644
--- a/llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll
+++ b/llvm/test/CodeGen/Hexagon/loop-idiom/lcssa.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -hexagon-loop-idiom < %s | opt -S -passes='loop(loop-deletion),gvn'
+; RUN: opt -mtriple hexagon-- -S -passes='loop(hexagon-loop-idiom,loop-deletion),gvn'
 ; REQUIRES: asserts
 
 ; This tests that the HexagonLoopIdiom pass does not mark LCSSA information
diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/memmove-rt-check.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/memmove-rt-check.ll
index 7a7d1d9b1a86..78f0c9e36b55 100644
--- a/llvm/test/CodeGen/Hexagon/loop-idiom/memmove-rt-check.ll
+++ b/llvm/test/CodeGen/Hexagon/loop-idiom/memmove-rt-check.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -hexagon-loop-idiom -S < %s | FileCheck %s
+; RUN: opt -p hexagon-loop-idiom -S < %s | FileCheck %s
 
 ; Make sure that we generate correct runtime checks.
 
diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll
index 37e1bb6eb7df..ce02b62911c0 100644
--- a/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll
+++ b/llvm/test/CodeGen/Hexagon/loop-idiom/nullptr-crash.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -hexagon-loop-idiom -mtriple hexagon-unknown-elf < %s
+; RUN: opt -p hexagon-loop-idiom -mtriple hexagon-unknown-elf < %s
 ; REQUIRES: asserts
 
 target triple = "hexagon"
diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-infinite-loop.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-infinite-loop.ll
index 1934ced7e7ae..74c02d63d54d 100644
--- a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-infinite-loop.ll
+++ b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-infinite-loop.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -march=hexagon -hexagon-loop-idiom -S < %s | FileCheck %s
+; RUN: opt -march=hexagon -p hexagon-loop-idiom -S < %s | FileCheck %s
 ; CHECK-LABEL: define void @fred
 
 ; Check that this test does not crash.
diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-long-loop.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-long-loop.ll
index b25010f2a90f..94b0c96c3d51 100644
--- a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-long-loop.ll
+++ b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-long-loop.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -march=hexagon -hexagon-loop-idiom -S < %s | FileCheck %s
+; RUN: opt -march=hexagon -p hexagon-loop-idiom -S < %s | FileCheck %s
 ;
 ; The number of nested selects caused the simplification loop to take
 ; more than the maximum number of iterations. This caused the compiler
diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-shiftconv-fail.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-shiftconv-fail.ll
index e4b2b5a298ed..a00b1d5876ba 100644
--- a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-shiftconv-fail.ll
+++ b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy-shiftconv-fail.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -march=hexagon -hexagon-loop-idiom -S < %s | FileCheck %s
+; RUN: opt -march=hexagon -p hexagon-loop-idiom -S < %s | FileCheck %s
 ; REQUIRES: asserts
 ;
 ; Check for sane output, this used to crash.
diff --git a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll
index 781618e58901..2461e1cfde8d 100644
--- a/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll
+++ b/llvm/test/CodeGen/Hexagon/loop-idiom/pmpy.ll
@@ -1,5 +1,7 @@
 ; RUN: opt -hexagon-loop-idiom < %s -mtriple=hexagon-unknown-unknown -S \
 ; RUN:  | FileCheck %s
+; RUN: opt -p hexagon-loop-idiom < %s -mtriple=hexagon-unknown-unknown -S \
+; RUN:  | FileCheck %s
 
 target triple = "hexagon"
 
-- 
GitLab


From 691b97c884a15a7eac641ddf67c9f2f30fb4e747 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sat, 23 Mar 2024 01:02:08 -0700
Subject: [PATCH 041/404] [ELF] Remove zero-value DT_JMPREL when IPLT is
 present while PLT isn't

The zero-value DT_JMPREL is benign but not needed.
This is also code simplification available after https://reviews.llvm.org/D65651
---
 lld/ELF/SyntheticSections.cpp                 | 16 ++-------------
 .../ELF/aarch64-gnu-ifunc-nonpreemptable.s    | 12 +++++------
 lld/test/ELF/gnu-ifunc-dyntags.s              |  8 +++-----
 lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s |  8 ++++----
 lld/test/ELF/riscv-ifunc-nonpreemptible.s     | 20 +++++++++----------
 5 files changed, 25 insertions(+), 39 deletions(-)

diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index 248ff6b4a865..10eda17f0cb3 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -1285,13 +1285,7 @@ static uint64_t addRelaSz(const RelocationBaseSection &relaDyn) {
 // output section. When this occurs we cannot just use the OutputSection
 // Size. Moreover the [DT_JMPREL, DT_JMPREL + DT_PLTRELSZ) is permitted to
 // overlap with the [DT_RELA, DT_RELA + DT_RELASZ).
-static uint64_t addPltRelSz() {
-  size_t size = in.relaPlt->getSize();
-  if (in.relaIplt->getParent() == in.relaPlt->getParent() &&
-      in.relaIplt->name == in.relaPlt->name)
-    size += in.relaIplt->getSize();
-  return size;
-}
+static uint64_t addPltRelSz() { return in.relaPlt->getSize(); }
 
 // Add remaining entries to complete .dynamic contents.
 template <class ELFT>
@@ -1407,13 +1401,7 @@ DynamicSection<ELFT>::computeContents() {
     addInt(config->useAndroidRelrTags ? DT_ANDROID_RELRENT : DT_RELRENT,
            sizeof(Elf_Relr));
   }
-  // .rel[a].plt section usually consists of two parts, containing plt and
-  // iplt relocations. It is possible to have only iplt relocations in the
-  // output. In that case relaPlt is empty and have zero offset, the same offset
-  // as relaIplt has. And we still want to emit proper dynamic tags for that
-  // case, so here we always use relaPlt as marker for the beginning of
-  // .rel[a].plt section.
-  if (isMain && (in.relaPlt->isNeeded() || in.relaIplt->isNeeded())) {
+  if (isMain && in.relaPlt->isNeeded()) {
     addInSec(DT_JMPREL, *in.relaPlt);
     entries.emplace_back(DT_PLTRELSZ, addPltRelSz());
     switch (config->emachine) {
diff --git a/lld/test/ELF/aarch64-gnu-ifunc-nonpreemptable.s b/lld/test/ELF/aarch64-gnu-ifunc-nonpreemptable.s
index 4f33dde9d1a9..54a27a8cc0d9 100644
--- a/lld/test/ELF/aarch64-gnu-ifunc-nonpreemptable.s
+++ b/lld/test/ELF/aarch64-gnu-ifunc-nonpreemptable.s
@@ -65,14 +65,14 @@ main:
 # PIE-EMPTY:
 # PIE-NEXT: <myfunc>:
 # PIE-NEXT:    10270: adrp    x16, 0x30000
-# PIE-NEXT:    10274: ldr     x17, [x16, #896]
-# PIE-NEXT:    10278: add     x16, x16, #896
-# PIE-NEXT:    1027c: br      x17
+# PIE-NEXT:           ldr     x17, [x16, #832]
+# PIE-NEXT:           add     x16, x16, #832
+# PIE-NEXT:           br      x17
 
 # PIE-RELOC:      .rela.dyn {
-# PIE-RELOC-NEXT:   0x30380 R_AARCH64_IRELATIVE - 0x10260
+# PIE-RELOC-NEXT:   0x30340 R_AARCH64_IRELATIVE - 0x10260
 # PIE-RELOC-NEXT: }
 # PIE-RELOC:      Hex dump of section '.got.plt':
-# NO-APPLY:       0x00030380 00000000 00000000
-# APPLY:          0x00030380 60020100 00000000
+# NO-APPLY:       0x00030340 00000000 00000000
+# APPLY:          0x00030340 60020100 00000000
 # PIE-RELOC-EMPTY:
diff --git a/lld/test/ELF/gnu-ifunc-dyntags.s b/lld/test/ELF/gnu-ifunc-dyntags.s
index fd80dc24f2f8..57a17245a3e8 100644
--- a/lld/test/ELF/gnu-ifunc-dyntags.s
+++ b/lld/test/ELF/gnu-ifunc-dyntags.s
@@ -9,15 +9,13 @@
 
 # CHECK:  Name          Size   VMA
 # CHECK:  .rela.dyn   00000030 0000000000000248
-# CHECK:  .got.plt    00000010 00000000000033b0
+# CHECK:  .got.plt    00000010 0000000000003370
 
 # TAGS:   Tag                Type                 Name/Value
 # TAGS:   0x0000000000000007 RELA                 0x248
 # TAGS:   0x0000000000000008 RELASZ               48 (bytes)
-# TAGS:   0x0000000000000017 JMPREL               0x0
-# TAGS:   0x0000000000000002 PLTRELSZ             0 (bytes)
-# TAGS:   0x0000000000000003 PLTGOT               0x33B0
-# TAGS:   0x0000000000000014 PLTREL               RELA
+# TAGS-NOT: JMPREL
+# TAGS-NOT: PLTREL
 
 # TAGS:      Relocations [
 # TAGS-NEXT:   Section {{.*}} .rela.dyn {
diff --git a/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s b/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s
index a93f3cecb0c6..c9a0381b610a 100644
--- a/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s
+++ b/lld/test/ELF/ppc32-ifunc-nonpreemptible-pic.s
@@ -10,16 +10,16 @@
 # RUN: llvm-readelf -x .got2 %t | FileCheck --check-prefix=HEX2 %s
 
 # RELOC:      .rela.dyn {
-# RELOC-NEXT:   0x3024C R_PPC_RELATIVE - 0x101A0
-# RELOC-NEXT:   0x30250 R_PPC_IRELATIVE - 0x10188
+# RELOC-NEXT:   0x3022C R_PPC_RELATIVE - 0x101A0
+# RELOC-NEXT:   0x30230 R_PPC_IRELATIVE - 0x10188
 # RELOC-NEXT: }
 
 # SYM: 000101a0 0 FUNC GLOBAL DEFAULT {{.*}} func
 # HEX:      Hex dump of section '.got2':
-# HEX-NEXT: 0x0003024c 00000000 ....
+# HEX-NEXT: 0x0003022c 00000000 ....
 
 # HEX2:      Hex dump of section '.got2':
-# HEX2-NEXT: 0x0003024c 000101a0 ....
+# HEX2-NEXT: 0x0003022c 000101a0 ....
 
 .section .got2,"aw"
 .long func
diff --git a/lld/test/ELF/riscv-ifunc-nonpreemptible.s b/lld/test/ELF/riscv-ifunc-nonpreemptible.s
index 21c607545110..eda5548eef8b 100644
--- a/lld/test/ELF/riscv-ifunc-nonpreemptible.s
+++ b/lld/test/ELF/riscv-ifunc-nonpreemptible.s
@@ -16,11 +16,11 @@
 # RUN: llvm-objdump -d --no-show-raw-insn %t.64 | FileCheck --check-prefix=DIS64 %s
 
 # RELOC32:      .rela.dyn {
-# RELOC32-NEXT:   0x3220 R_RISCV_IRELATIVE - 0x117C
+# RELOC32-NEXT:   0x3200 R_RISCV_IRELATIVE - 0x117C
 # RELOC32-NEXT: }
 # RELOC32-LABEL:    Hex dump of section '.got.plt':
-# NO-APPLY-RELOC32: 0x00003220 00000000
-# APPLY-RELOC32:    0x00003220 7c110000
+# NO-APPLY-RELOC32: 0x00003200 00000000
+# APPLY-RELOC32:    0x00003200 7c110000
 # RELOC32-EMPTY:
 
 # SYM32: 0001190 0 FUNC GLOBAL DEFAULT {{.*}} func
@@ -30,18 +30,18 @@
 # DIS32-NEXT:       addi a0, a0, 0x10
 # DIS32:      Disassembly of section .iplt:
 # DIS32:      <func>:
-## 32-bit: &.got.plt[func]-. = 0x3220-0x1190 = 4096*2+144
+## 32-bit: &.got.plt[func]-. = 0x3200-0x1190 = 4096*2+0x70
 # DIS32-NEXT: 1190: auipc t3, 0x2
-# DIS32-NEXT:       lw t3, 0x90(t3)
+# DIS32-NEXT:       lw t3, 0x70(t3)
 # DIS32-NEXT:       jalr t1, t3
 # DIS32-NEXT:       nop
 
 # RELOC64:      .rela.dyn {
-# RELOC64-NEXT:   0x3380 R_RISCV_IRELATIVE - 0x1260
+# RELOC64-NEXT:   0x3340 R_RISCV_IRELATIVE - 0x1260
 # RELOC64-NEXT: }
 # RELOC64-LABEL:    Hex dump of section '.got.plt':
-# NO-APPLY-RELOC64: 0x00003380 00000000 00000000
-# APPLY-RELOC64:    0x00003380 60120000 00000000
+# NO-APPLY-RELOC64: 0x00003340 00000000 00000000
+# APPLY-RELOC64:    0x00003340 60120000 00000000
 # RELOC64-EMPTY:
 
 # SYM64: 000000000001270 0 FUNC GLOBAL DEFAULT {{.*}} func
@@ -51,9 +51,9 @@
 # DIS64-NEXT:       addi a0, a0, 0xc
 # DIS64:      Disassembly of section .iplt:
 # DIS64:      <func>:
-## 64-bit: &.got.plt[func]-. = 0x3380-0x1270 = 4096*2+272
+## 64-bit: &.got.plt[func]-. = 0x3340-0x1270 = 4096*2+0xd0
 # DIS64-NEXT: 1270: auipc t3, 0x2
-# DIS64-NEXT:       ld t3, 0x110(t3)
+# DIS64-NEXT:       ld t3, 0xd0(t3)
 # DIS64-NEXT:       jalr t1, t3
 # DIS64-NEXT:       nop
 
-- 
GitLab


From 579dc7f8441a8044b92bdfa6f0db2f91301c0eed Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Sat, 23 Mar 2024 02:03:45 -0700
Subject: [PATCH 042/404] [clang-forma] Support `PointerAlignment` for pointers
 to members (#86253)

Fixes #85761.
---
 clang/lib/Format/TokenAnnotator.cpp           | 37 +++++++++++-------
 clang/unittests/Format/FormatTest.cpp         | 38 ++++++++++++-------
 clang/unittests/Format/QualifierFixerTest.cpp | 36 +++++++++---------
 3 files changed, 66 insertions(+), 45 deletions(-)

diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index a5cafcbacaa5..7757c8ff7639 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -4357,9 +4357,11 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
   if (Left.is(tok::kw_auto) && Right.isOneOf(tok::l_paren, tok::l_brace))
     return false;
 
+  const auto *BeforeLeft = Left.Previous;
+
   // operator co_await(x)
-  if (Right.is(tok::l_paren) && Left.is(tok::kw_co_await) && Left.Previous &&
-      Left.Previous->is(tok::kw_operator)) {
+  if (Right.is(tok::l_paren) && Left.is(tok::kw_co_await) && BeforeLeft &&
+      BeforeLeft->is(tok::kw_operator)) {
     return false;
   }
   // co_await (x), co_yield (x), co_return (x)
@@ -4394,8 +4396,10 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
   }
   if (Left.is(tok::colon))
     return Left.isNot(TT_ObjCMethodExpr);
-  if (Left.is(tok::coloncolon))
-    return false;
+  if (Left.is(tok::coloncolon)) {
+    return Right.is(tok::star) && Right.is(TT_PointerOrReference) &&
+           Style.PointerAlignment != FormatStyle::PAS_Left;
+  }
   if (Left.is(tok::less) || Right.isOneOf(tok::greater, tok::less)) {
     if (Style.Language == FormatStyle::LK_TextProto ||
         (Style.Language == FormatStyle::LK_Proto &&
@@ -4410,8 +4414,8 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
       return false;
   }
   if (Right.is(tok::ellipsis)) {
-    return Left.Tok.isLiteral() || (Left.is(tok::identifier) && Left.Previous &&
-                                    Left.Previous->is(tok::kw_case));
+    return Left.Tok.isLiteral() || (Left.is(tok::identifier) && BeforeLeft &&
+                                    BeforeLeft->is(tok::kw_case));
   }
   if (Left.is(tok::l_square) && Right.is(tok::amp))
     return Style.SpacesInSquareBrackets;
@@ -4479,8 +4483,8 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
     if (Right.is(tok::l_brace) && Right.is(BK_Block))
       return true;
     // for (auto a = 0, b = 0; const auto& c : {1, 2, 3})
-    if (Left.Previous && Left.Previous->isTypeOrIdentifier(IsCpp) &&
-        Right.Next && Right.Next->is(TT_RangeBasedForLoopColon)) {
+    if (BeforeLeft && BeforeLeft->isTypeOrIdentifier(IsCpp) && Right.Next &&
+        Right.Next->is(TT_RangeBasedForLoopColon)) {
       return getTokenPointerOrReferenceAlignment(Left) !=
              FormatStyle::PAS_Right;
     }
@@ -4502,12 +4506,17 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
           startsWithInitStatement(Line)))) {
       return false;
     }
-    return Left.Previous && !Left.Previous->isOneOf(
-                                tok::l_paren, tok::coloncolon, tok::l_square);
+    if (!BeforeLeft)
+      return false;
+    if (BeforeLeft->is(tok::coloncolon)) {
+      return Left.is(tok::star) &&
+             Style.PointerAlignment != FormatStyle::PAS_Right;
+    }
+    return !BeforeLeft->isOneOf(tok::l_paren, tok::l_square);
   }
   // Ensure right pointer alignment with ellipsis e.g. int *...P
-  if (Left.is(tok::ellipsis) && Left.Previous &&
-      Left.Previous->isPointerOrReference()) {
+  if (Left.is(tok::ellipsis) && BeforeLeft &&
+      BeforeLeft->isPointerOrReference()) {
     return Style.PointerAlignment != FormatStyle::PAS_Right;
   }
 
@@ -4669,13 +4678,13 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
       return Style.SpaceBeforeParensOptions.AfterFunctionDefinitionName ||
              spaceRequiredBeforeParens(Right);
     }
-    if (!Left.Previous || !Left.Previous->isOneOf(tok::period, tok::arrow)) {
+    if (!BeforeLeft || !BeforeLeft->isOneOf(tok::period, tok::arrow)) {
       if (Left.isOneOf(tok::kw_try, Keywords.kw___except, tok::kw_catch)) {
         return Style.SpaceBeforeParensOptions.AfterControlStatements ||
                spaceRequiredBeforeParens(Right);
       }
       if (Left.isOneOf(tok::kw_new, tok::kw_delete)) {
-        return ((!Line.MightBeFunctionDecl || !Left.Previous) &&
+        return ((!Line.MightBeFunctionDecl || !BeforeLeft) &&
                 Style.SpaceBeforeParens != FormatStyle::SBPO_Never) ||
                spaceRequiredBeforeParens(Right);
       }
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index bea989c8c306..cf8d6ab691d9 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -3621,8 +3621,8 @@ TEST_F(FormatTest, FormatsClasses) {
                "    : public aaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaaaaaaaaaaa,\n"
                "                                 aaaaaaaaaaaaaaaaaaaaaa> {};");
   verifyFormat("template <class R, class C>\n"
-               "struct Aaaaaaaaaaaaaaaaa<R (C::*)(int) const>\n"
-               "    : Aaaaaaaaaaaaaaaaa<R (C::*)(int)> {};");
+               "struct Aaaaaaaaaaaaaaaaa<R (C:: *)(int) const>\n"
+               "    : Aaaaaaaaaaaaaaaaa<R (C:: *)(int)> {};");
   verifyFormat("class ::A::B {};");
 }
 
@@ -11034,10 +11034,10 @@ TEST_F(FormatTest, UnderstandsBinaryOperators) {
 }
 
 TEST_F(FormatTest, UnderstandsPointersToMembers) {
-  verifyFormat("int A::*x;");
-  verifyFormat("int (S::*func)(void *);");
-  verifyFormat("void f() { int (S::*func)(void *); }");
-  verifyFormat("typedef bool *(Class::*Member)() const;");
+  verifyFormat("int A:: *x;");
+  verifyFormat("int (S:: *func)(void *);");
+  verifyFormat("void f() { int (S:: *func)(void *); }");
+  verifyFormat("typedef bool *(Class:: *Member)() const;");
   verifyFormat("void f() {\n"
                "  (a->*f)();\n"
                "  a->*x;\n"
@@ -11052,9 +11052,19 @@ TEST_F(FormatTest, UnderstandsPointersToMembers) {
   verifyFormat(
       "(aaaaaaaaaa->*bbbbbbb)(\n"
       "    aaaaaaaaaaaaaaaaaaaaaaaaaaa(aaaaaaaaaaaaaaaaaaaaaaaaaaa));");
+
   FormatStyle Style = getLLVMStyle();
+  EXPECT_EQ(Style.PointerAlignment, FormatStyle::PAS_Right);
+  verifyFormat("typedef bool *(Class:: *Member)() const;", Style);
+  verifyFormat("void f(int A:: *p) { int A:: *v = &A::B; }", Style);
+
   Style.PointerAlignment = FormatStyle::PAS_Left;
-  verifyFormat("typedef bool* (Class::*Member)() const;", Style);
+  verifyFormat("typedef bool* (Class::* Member)() const;", Style);
+  verifyFormat("void f(int A::* p) { int A::* v = &A::B; }", Style);
+
+  Style.PointerAlignment = FormatStyle::PAS_Middle;
+  verifyFormat("typedef bool * (Class:: * Member)() const;", Style);
+  verifyFormat("void f(int A:: * p) { int A:: * v = &A::B; }", Style);
 }
 
 TEST_F(FormatTest, UnderstandsUnaryOperators) {
@@ -12386,7 +12396,7 @@ TEST_F(FormatTest, FormatsFunctionTypes) {
   verifyFormat("int (*func)(void *);");
   verifyFormat("void f() { int (*func)(void *); }");
   verifyFormat("template <class CallbackClass>\n"
-               "using MyCallback = void (CallbackClass::*)(SomeObject *Data);");
+               "using Callback = void (CallbackClass:: *)(SomeObject *Data);");
 
   verifyGoogleFormat("A<void*(int*, SomeType*)>;");
   verifyGoogleFormat("void* (*a)(int);");
@@ -19149,13 +19159,13 @@ TEST_F(FormatTest, AlignConsecutiveDeclarations) {
                "int   bbbbbbb = 0;",
                Alignment);
   // http://llvm.org/PR68079
-  verifyFormat("using Fn   = int (A::*)();\n"
-               "using RFn  = int (A::*)() &;\n"
-               "using RRFn = int (A::*)() &&;",
+  verifyFormat("using Fn   = int (A:: *)();\n"
+               "using RFn  = int (A:: *)() &;\n"
+               "using RRFn = int (A:: *)() &&;",
                Alignment);
-  verifyFormat("using Fn   = int (A::*)();\n"
-               "using RFn  = int *(A::*)() &;\n"
-               "using RRFn = double (A::*)() &&;",
+  verifyFormat("using Fn   = int (A:: *)();\n"
+               "using RFn  = int *(A:: *)() &;\n"
+               "using RRFn = double (A:: *)() &&;",
                Alignment);
 
   // PAS_Right
diff --git a/clang/unittests/Format/QualifierFixerTest.cpp b/clang/unittests/Format/QualifierFixerTest.cpp
index 43476aea6633..792d8f3c3a98 100644
--- a/clang/unittests/Format/QualifierFixerTest.cpp
+++ b/clang/unittests/Format/QualifierFixerTest.cpp
@@ -305,7 +305,7 @@ TEST_F(QualifierFixerTest, RightQualifier) {
   verifyFormat("Foo inline static const;", "Foo inline const static;", Style);
   verifyFormat("Foo inline static const;", Style);
 
-  verifyFormat("Foo<T volatile>::Bar<Type const, 5> const volatile A::*;",
+  verifyFormat("Foo<T volatile>::Bar<Type const, 5> const volatile A:: *;",
                "volatile const Foo<volatile T>::Bar<const Type, 5> A::*;",
                Style);
 
@@ -523,14 +523,15 @@ TEST_F(QualifierFixerTest, RightQualifier) {
   verifyFormat("const INTPTR a;", Style);
 
   // Pointers to members
-  verifyFormat("int S::*a;", Style);
-  verifyFormat("int const S::*a;", "const int S:: *a;", Style);
-  verifyFormat("int const S::*const a;", "const int S::* const a;", Style);
-  verifyFormat("int A::*const A::*p1;", Style);
-  verifyFormat("float (C::*p)(int);", Style);
-  verifyFormat("float (C::*const p)(int);", Style);
-  verifyFormat("float (C::*p)(int) const;", Style);
-  verifyFormat("float const (C::*p)(int);", "const float (C::*p)(int);", Style);
+  verifyFormat("int S:: *a;", Style);
+  verifyFormat("int const S:: *a;", "const int S:: *a;", Style);
+  verifyFormat("int const S:: *const a;", "const int S::* const a;", Style);
+  verifyFormat("int A:: *const A:: *p1;", Style);
+  verifyFormat("float (C:: *p)(int);", Style);
+  verifyFormat("float (C:: *const p)(int);", Style);
+  verifyFormat("float (C:: *p)(int) const;", Style);
+  verifyFormat("float const (C:: *p)(int);", "const float (C::*p)(int);",
+               Style);
 }
 
 TEST_F(QualifierFixerTest, LeftQualifier) {
@@ -830,14 +831,15 @@ TEST_F(QualifierFixerTest, LeftQualifier) {
   verifyFormat("INTPTR const a;", Style);
 
   // Pointers to members
-  verifyFormat("int S::*a;", Style);
-  verifyFormat("const int S::*a;", "int const S:: *a;", Style);
-  verifyFormat("const int S::*const a;", "int const S::* const a;", Style);
-  verifyFormat("int A::*const A::*p1;", Style);
-  verifyFormat("float (C::*p)(int);", Style);
-  verifyFormat("float (C::*const p)(int);", Style);
-  verifyFormat("float (C::*p)(int) const;", Style);
-  verifyFormat("const float (C::*p)(int);", "float const (C::*p)(int);", Style);
+  verifyFormat("int S:: *a;", Style);
+  verifyFormat("const int S:: *a;", "int const S:: *a;", Style);
+  verifyFormat("const int S:: *const a;", "int const S::* const a;", Style);
+  verifyFormat("int A:: *const A:: *p1;", Style);
+  verifyFormat("float (C:: *p)(int);", Style);
+  verifyFormat("float (C:: *const p)(int);", Style);
+  verifyFormat("float (C:: *p)(int) const;", Style);
+  verifyFormat("const float (C:: *p)(int);", "float const (C::*p)(int);",
+               Style);
 }
 
 TEST_F(QualifierFixerTest, ConstVolatileQualifiersOrder) {
-- 
GitLab


From 47423e9827abfdcc6b10ce41618965861b0e69a4 Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Sat, 23 Mar 2024 02:24:56 -0700
Subject: [PATCH 043/404] [clang-format][NFC] Clean up
 IsQualifiedPointerOrReference in TokenAnnotator

---
 clang/lib/Format/TokenAnnotator.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index 7757c8ff7639..4c83a7a3a323 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -2753,10 +2753,9 @@ private:
     }
 
     // Heuristically try to determine whether the parentheses contain a type.
-    auto IsQualifiedPointerOrReference = [this](FormatToken *T) {
+    auto IsQualifiedPointerOrReference = [](FormatToken *T, bool IsCpp) {
       // This is used to handle cases such as x = (foo *const)&y;
       assert(!T->isTypeName(IsCpp) && "Should have already been checked");
-      (void)IsCpp; // Avoid -Wunused-lambda-capture when assertion is disabled.
       // Strip trailing qualifiers such as const or volatile when checking
       // whether the parens could be a cast to a pointer/reference type.
       while (T) {
@@ -2789,7 +2788,7 @@ private:
         !Tok.Previous ||
         Tok.Previous->isOneOf(TT_TemplateCloser, TT_TypeDeclarationParen) ||
         Tok.Previous->isTypeName(IsCpp) ||
-        IsQualifiedPointerOrReference(Tok.Previous);
+        IsQualifiedPointerOrReference(Tok.Previous, IsCpp);
     bool ParensCouldEndDecl =
         Tok.Next->isOneOf(tok::equal, tok::semi, tok::l_brace, tok::greater);
     if (ParensAreType && !ParensCouldEndDecl)
-- 
GitLab


From f317fd266c184bc8f9f4d7e8e03c043d3406333a Mon Sep 17 00:00:00 2001
From: Muhammad Omair Javaid <omair.javaid@linaro.org>
Date: Sat, 23 Mar 2024 15:30:26 +0500
Subject: [PATCH 044/404] Revert "[mlir][SVE] Add e2e for 1D depthwise WC
 convolution (#85225)"

This reverts commit 01b1b0c1f728e2c2639edc654424f50830295989.

Breaks following AArch64 SVE buildbots:
https://lab.llvm.org/buildbot/#/builders/184/builds/11363
https://lab.llvm.org/buildbot/#/builders/176/builds/9331
---
 .../Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir  | 60 -------------------
 1 file changed, 60 deletions(-)
 delete mode 100644 mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir

diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir
deleted file mode 100644
index 57d69383c2de..000000000000
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir
+++ /dev/null
@@ -1,60 +0,0 @@
-// DEFINE: %{compile} =  mlir-opt %s \
-// DEFINE:    -transform-interpreter -test-transform-dialect-erase-schedule \
-// DEFINE:    -one-shot-bufferize="bufferize-function-boundaries" -lower-vector-mask -cse -canonicalize -convert-vector-to-scf -arm-sve-legalize-vector-storage \
-// DEFINE:    -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm -o %t
-// DEFINE: %{entry_point} = conv
-// DEFINE: %{run} = %mcr_aarch64_cmd %t -e %{entry_point} -entry-point-result=void --march=aarch64 --mattr="+sve"\
-// DEFINE:    -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils
-
-// RUN: %{compile} | %{run} | FileCheck %s
-
-func.func @conv() {
-  // Define input/output tensors
-  %input_init = tensor.empty() : tensor<1x8x6xi32>
-  %output_init = tensor.empty() : tensor<1x7x6xi32>
-
-  %five = arith.constant 5 : i32
-  %zero = arith.constant 0 : i32
-  %input = linalg.fill ins(%five : i32) outs(%input_init : tensor<1x8x6xi32>) -> tensor<1x8x6xi32>
-  %output = linalg.fill ins(%zero : i32) outs(%output_init : tensor<1x7x6xi32>) -> tensor<1x7x6xi32>
-
-  // Define the filter tensor
-  %filter = arith.constant dense<[
-    [ 1,  2, 3, 4, 5, 6],
-    [ 11, 12, 13, 14, 15, 16]
-  ]> : tensor<2x6xi32>
-
-  // static sizes -> dynamic sizes
-  %input_dyn = tensor.cast %input_init : tensor<1x8x6xi32> to tensor<1x8x?xi32>
-  %output_dyn = tensor.cast %output : tensor<1x7x6xi32> to tensor<1x7x?xi32>
-  %filter_dyn = tensor.cast %filter : tensor<2x6xi32> to tensor<2x?xi32>
-
-  // Run the convolution
-  %res = linalg.depthwise_conv_1d_nwc_wc
-    ins(%input_dyn, %filter_dyn : tensor<1x8x?xi32>, tensor<2x?xi32>)
-    outs(%output_dyn : tensor<1x7x?xi32>) -> tensor<1x7x?xi32>
-
-  // Print the results
-  // CHECK: SVE: START OF TEST OUTPUT
-  vector.print str "SVE: START OF TEST OUTPUT\n"
-
-  // CHECK-NEXT: Unranked Memref base@ = {{.*}} rank = 3 offset = 0 sizes = [1, 7, 6] strides = [42, 6, 1] data =
-  // CHECK-COUNT-7: [60, 70, 80, 90, 100, 110]
-  %xf = tensor.cast %res : tensor<1x7x?xi32> to tensor<*xi32>
-  call @printMemrefI32(%xf) : (tensor<*xi32>) -> ()
-
-  // CHECK-NEXT: SVE: END OF TEST OUTPUT
-  vector.print str "SVE: END OF TEST OUTPUT\n"
-
-  return
-}
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.depthwise_conv_1d_nwc_wc"]} in %arg0 : (!transform.any_op) -> !transform.any_op
-    transform.structured.vectorize %0 vector_sizes [1, 7, [8], 2] : !transform.any_op
-    transform.yield
-  }
-}
-
-func.func private @printMemrefI32(%ptr : tensor<*xi32>) attributes { llvm.emit_c_interface }
-- 
GitLab


From d7c672834ec863b458af8ca493157e1e31aaf480 Mon Sep 17 00:00:00 2001
From: XChy <xxs_chy@outlook.com>
Date: Sat, 23 Mar 2024 19:00:42 +0800
Subject: [PATCH 045/404] [CodeGen][NFC] Update tests in AArch64/and-sink.ll

---
 llvm/test/CodeGen/AArch64/and-sink.ll | 60 +++++++++++++++++++++------
 1 file changed, 47 insertions(+), 13 deletions(-)

diff --git a/llvm/test/CodeGen/AArch64/and-sink.ll b/llvm/test/CodeGen/AArch64/and-sink.ll
index 4d085869de24..f298a55dab72 100644
--- a/llvm/test/CodeGen/AArch64/and-sink.ll
+++ b/llvm/test/CodeGen/AArch64/and-sink.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -mtriple=aarch64-linux-gnu -verify-machineinstrs < %s | FileCheck %s
 ; RUN: opt -S -passes='require<profile-summary>,function(codegenprepare)' -mtriple=aarch64-linux %s | FileCheck --check-prefix=CHECK-CGP %s
 ; RUN: opt -S -passes='require<profile-summary>,function(codegenprepare)' -cgpp-huge-func=0 -mtriple=aarch64-linux %s | FileCheck --check-prefix=CHECK-CGP %s
@@ -9,9 +10,18 @@
 ; Test that and is sunk into cmp block to form tbz.
 define dso_local i32 @and_sink1(i32 %a, i1 %c) {
 ; CHECK-LABEL: and_sink1:
-; CHECK: tbz w1, #0
-; CHECK: str wzr, [x{{[0-9]+}}, :lo12:A]
-; CHECK: tbnz {{w[0-9]+}}, #2
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    tbz w1, #0, .LBB0_3
+; CHECK-NEXT:  // %bb.1: // %bb0
+; CHECK-NEXT:    adrp x8, A
+; CHECK-NEXT:    str wzr, [x8, :lo12:A]
+; CHECK-NEXT:    tbnz w0, #2, .LBB0_3
+; CHECK-NEXT:  // %bb.2:
+; CHECK-NEXT:    mov w0, #1 // =0x1
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_3: // %bb2
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    ret
 
 ; CHECK-CGP-LABEL: @and_sink1(
 ; CHECK-CGP-NOT: and i32
@@ -35,12 +45,30 @@ bb2:
 ; Test that both 'and' and cmp get sunk to form tbz.
 define dso_local i32 @and_sink2(i32 %a, i1 %c, i1 %c2) {
 ; CHECK-LABEL: and_sink2:
-; CHECK: str wzr, [x{{[0-9]+}}, :lo12:A]
-; CHECK: tbz w1, #0
-; CHECK: str wzr, [x{{[0-9]+}}, :lo12:B]
-; CHECK: tbz w2, #0
-; CHECK: str wzr, [x{{[0-9]+}}, :lo12:C]
-; CHECK: tbnz {{w[0-9]+}}, #2
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    adrp x9, A
+; CHECK-NEXT:    str wzr, [x9, :lo12:A]
+; CHECK-NEXT:    tbz w1, #0, .LBB1_5
+; CHECK-NEXT:  // %bb.1: // %bb0.preheader
+; CHECK-NEXT:    adrp x8, B
+; CHECK-NEXT:    adrp x9, C
+; CHECK-NEXT:  .LBB1_2: // %bb0
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    str wzr, [x8, :lo12:B]
+; CHECK-NEXT:    tbz w2, #0, .LBB1_6
+; CHECK-NEXT:  // %bb.3: // %bb1
+; CHECK-NEXT:    // in Loop: Header=BB1_2 Depth=1
+; CHECK-NEXT:    str wzr, [x9, :lo12:C]
+; CHECK-NEXT:    tbnz w0, #2, .LBB1_2
+; CHECK-NEXT:  // %bb.4:
+; CHECK-NEXT:    mov w8, #1 // =0x1
+; CHECK-NEXT:  .LBB1_5: // %common.ret
+; CHECK-NEXT:    mov w0, w8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB1_6:
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    ret
 
 ; CHECK-CGP-LABEL: @and_sink2(
 ; CHECK-CGP-NOT: and i32
@@ -71,10 +99,16 @@ bb3:
 ; Test that 'and' is not sunk since cbz is a better alternative.
 define dso_local i32 @and_sink3(i32 %a) {
 ; CHECK-LABEL: and_sink3:
-; CHECK: and [[REG:w[0-9]+]], w0, #0x3
-; CHECK: [[LOOP:.L[A-Z0-9_]+]]:
-; CHECK: str wzr, [x{{[0-9]+}}, :lo12:A]
-; CHECK: cbz [[REG]], [[LOOP]]
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, A
+; CHECK-NEXT:    and w9, w0, #0x3
+; CHECK-NEXT:  .LBB2_1: // %bb0
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    str wzr, [x8, :lo12:A]
+; CHECK-NEXT:    cbz w9, .LBB2_1
+; CHECK-NEXT:  // %bb.2: // %bb2
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    ret
 
 ; CHECK-CGP-LABEL: @and_sink3(
 ; CHECK-CGP-NEXT: and i32
-- 
GitLab


From d365a45cb3eaa640b09874fb7984a6a69683c773 Mon Sep 17 00:00:00 2001
From: Evgenii Kudriashov <evgenii.kudriashov@intel.com>
Date: Sat, 23 Mar 2024 15:12:44 +0300
Subject: [PATCH 046/404] [GlobalISel] Introduce G_TRAP, G_DEBUGTRAP,
 G_UBSANTRAP (#84941)

Here we introduce three new GMIR instructions to cover a set of trap
intrinsics. The idea behind it is that generic intrinsics shouldn't be
used with G_INTRINSIC opcode.

These new instructions can match perfectly with existing trap ISD nodes.
It allows X86, AArch64, RISCV and Mips to reuse SelectionDAG patterns for
selection and avoid manual selection. However AMDGPU is an exception. It
selects traps during legalization regardless SelectionDAG or GlobalISel.

Since there are not many places where traps are used, this change
attempts to clean up all the usages of G_INTRINSIC with trap intrinsics. So,
there is no stage when both G_TRAP and
G_INTRINSIC_W_SIDE_EFFECTS(@llvm.trap) are allowed.
---
 llvm/docs/GlobalISel/GenericOpcode.rst        | 19 +++++
 llvm/docs/LangRef.rst                         |  6 ++
 .../llvm/CodeGen/GlobalISel/IRTranslator.h    |  4 +
 .../CodeGen/GlobalISel/MachineIRBuilder.h     |  5 ++
 llvm/include/llvm/Support/TargetOpcodes.def   |  5 ++
 llvm/include/llvm/Target/GenericOpcodes.td    | 22 ++++++
 .../Target/GlobalISel/SelectionDAGCompat.td   |  3 +
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  | 47 ++++++++----
 llvm/lib/CodeGen/MachineVerifier.cpp          | 11 +++
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |  3 +
 .../GISel/AArch64InstructionSelector.cpp      | 20 ++---
 .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 23 +++---
 llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h  |  8 +-
 llvm/lib/Target/Mips/MipsLegalizerInfo.cpp    |  8 --
 .../RISCV/GISel/RISCVInstructionSelector.cpp  | 27 -------
 .../X86/GISel/X86InstructionSelector.cpp      | 19 -----
 .../GlobalISel/irtranslator-unreachable.ll    |  2 +-
 .../AArch64/GlobalISel/legalize-exceptions.ll |  2 +-
 .../GlobalISel/legalizer-info-validation.mir  |  9 +++
 .../AArch64/GlobalISel/select-trap.mir        |  2 +-
 .../AArch64/GlobalISel/uaddo-8-16-bits.mir    | 52 ++++++-------
 .../AMDGPU/GlobalISel/legalize-trap.mir       |  4 +-
 .../trap.mir                                  |  9 ++-
 .../GlobalISel/instruction-select/trap.mir    |  4 +-
 .../X86/GlobalISel/x86-select-trap.mir        |  2 +-
 llvm/test/CodeGen/X86/isel-traps.ll           | 73 +++++++++++++++++++
 .../test/MachineVerifier/test_g_ubsantrap.mir | 18 +++++
 27 files changed, 276 insertions(+), 131 deletions(-)
 rename llvm/test/CodeGen/Mips/GlobalISel/{legalizer => instruction-select}/trap.mir (55%)
 create mode 100644 llvm/test/CodeGen/X86/isel-traps.ll
 create mode 100644 llvm/test/MachineVerifier/test_g_ubsantrap.mir

diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst
index ac6217d08e6a..cae2c21b80d7 100644
--- a/llvm/docs/GlobalISel/GenericOpcode.rst
+++ b/llvm/docs/GlobalISel/GenericOpcode.rst
@@ -939,6 +939,25 @@ The _CONVERGENT variant corresponds to an LLVM IR intrinsic marked `convergent`.
   Unlike SelectionDAG, there is no _VOID variant. Both of these are permitted
   to have zero, one, or multiple results.
 
+G_TRAP, G_DEBUGTRAP, G_UBSANTRAP
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Represents :ref:`llvm.trap <llvm.trap>`, :ref:`llvm.debugtrap <llvm.debugtrap>`
+and :ref:`llvm.ubsantrap <llvm.ubsantrap>` that generate a target dependent
+trap instructions.
+
+.. code-block:: none
+
+  G_TRAP
+
+.. code-block:: none
+
+  G_DEBUGTRAP
+
+.. code-block:: none
+
+  G_UBSANTRAP 12
+
 Variadic Arguments
 ------------------
 
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 8bc1cab01bf0..391fd2d960dd 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -26926,6 +26926,8 @@ Arguments:
 
 The argument should be an MDTuple containing any number of MDStrings.
 
+.. _llvm.trap:
+
 '``llvm.trap``' Intrinsic
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -26953,6 +26955,8 @@ This intrinsic is lowered to the target dependent trap instruction. If
 the target does not have a trap instruction, this intrinsic will be
 lowered to a call of the ``abort()`` function.
 
+.. _llvm.debugtrap:
+
 '``llvm.debugtrap``' Intrinsic
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -26980,6 +26984,8 @@ This intrinsic is lowered to code which is intended to cause an
 execution trap with the intention of requesting the attention of a
 debugger.
 
+.. _llvm.ubsantrap:
+
 '``llvm.ubsantrap``' Intrinsic
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
index 6ae7c1440907..5f28908e998a 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
@@ -243,6 +243,10 @@ private:
   bool translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder,
                         unsigned Opcode);
 
+  /// Translate an LLVM trap intrinsic (trap, debugtrap, ubsantrap).
+  bool translateTrap(const CallInst &U, MachineIRBuilder &MIRBuilder,
+                     unsigned Opcode);
+
   // Translate @llvm.experimental.vector.interleave2 and
   // @llvm.experimental.vector.deinterleave2 intrinsics for fixed-width vector
   // types into vector shuffles.
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index aaa81342845b..3ba036ae713f 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -2113,6 +2113,11 @@ public:
                                 DstMMO, SrcMMO);
   }
 
+  /// Build and insert G_TRAP or G_DEBUGTRAP
+  MachineInstrBuilder buildTrap(bool Debug = false) {
+    return buildInstr(Debug ? TargetOpcode::G_DEBUGTRAP : TargetOpcode::G_TRAP);
+  }
+
   /// Build and insert \p Dst = G_SBFX \p Src, \p LSB, \p Width.
   MachineInstrBuilder buildSbfx(const DstOp &Dst, const SrcOp &Src,
                                 const SrcOp &LSB, const SrcOp &Width) {
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index 899eaad5842a..5765926d6d93 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -837,6 +837,11 @@ HANDLE_TARGET_OPCODE(G_MEMMOVE)
 HANDLE_TARGET_OPCODE(G_MEMSET)
 HANDLE_TARGET_OPCODE(G_BZERO)
 
+/// llvm.trap, llvm.debugtrap and llvm.ubsantrap intrinsics
+HANDLE_TARGET_OPCODE(G_TRAP)
+HANDLE_TARGET_OPCODE(G_DEBUGTRAP)
+HANDLE_TARGET_OPCODE(G_UBSANTRAP)
+
 /// Vector reductions
 HANDLE_TARGET_OPCODE(G_VECREDUCE_SEQ_FADD)
 HANDLE_TARGET_OPCODE(G_VECREDUCE_SEQ_FMUL)
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index 67d405ba96fa..d0f471eb29b6 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -1575,6 +1575,28 @@ def G_BZERO : GenericInstruction {
   let mayStore = true;
 }
 
+//------------------------------------------------------------------------------
+// Trap intrinsics
+//------------------------------------------------------------------------------
+def G_TRAP : GenericInstruction {
+  let OutOperandList = (outs);
+  let InOperandList = (ins);
+  let hasSideEffects = true;
+  let mayStore = true;
+}
+
+def G_DEBUGTRAP : GenericInstruction {
+  let OutOperandList = (outs);
+  let InOperandList = (ins);
+  let hasSideEffects = true;
+}
+
+def G_UBSANTRAP : GenericInstruction {
+  let OutOperandList = (outs);
+  let InOperandList = (ins i8imm:$kind);
+  let hasSideEffects = true;
+}
+
 //------------------------------------------------------------------------------
 // Bitfield extraction.
 //------------------------------------------------------------------------------
diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
index b1f3c500a1b6..4bb9929e2cf8 100644
--- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
+++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
@@ -250,6 +250,9 @@ def : GINodeEquiv<G_ATOMICRMW_UINC_WRAP, atomic_load_uinc_wrap>;
 def : GINodeEquiv<G_ATOMICRMW_UDEC_WRAP, atomic_load_udec_wrap>;
 def : GINodeEquiv<G_FENCE, atomic_fence>;
 def : GINodeEquiv<G_PREFETCH, prefetch>;
+def : GINodeEquiv<G_TRAP, trap>;
+def : GINodeEquiv<G_DEBUGTRAP, debugtrap>;
+def : GINodeEquiv<G_UBSANTRAP, ubsantrap>;
 
 // Specifies the GlobalISel equivalents for SelectionDAG's ComplexPattern.
 // Should be used on defs that subclass GIComplexOperandMatcher<>.
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 757af3b1c4fe..ef95ceec18a1 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1771,6 +1771,32 @@ bool IRTranslator::translateMemFunc(const CallInst &CI,
   return true;
 }
 
+bool IRTranslator::translateTrap(const CallInst &CI,
+                                 MachineIRBuilder &MIRBuilder,
+                                 unsigned Opcode) {
+  StringRef TrapFuncName =
+      CI.getAttributes().getFnAttr("trap-func-name").getValueAsString();
+  if (TrapFuncName.empty()) {
+    if (Opcode == TargetOpcode::G_UBSANTRAP) {
+      uint64_t Code = cast<ConstantInt>(CI.getOperand(0))->getZExtValue();
+      MIRBuilder.buildInstr(Opcode, {}, ArrayRef<llvm::SrcOp>{Code});
+    } else {
+      MIRBuilder.buildInstr(Opcode);
+    }
+    return true;
+  }
+
+  CallLowering::CallLoweringInfo Info;
+  if (Opcode == TargetOpcode::G_UBSANTRAP)
+    Info.OrigArgs.push_back({getOrCreateVRegs(*CI.getArgOperand(0)),
+                             CI.getArgOperand(0)->getType(), 0});
+
+  Info.Callee = MachineOperand::CreateES(TrapFuncName.data());
+  Info.CB = &CI;
+  Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0};
+  return CLI->lowerCall(MIRBuilder, Info);
+}
+
 bool IRTranslator::translateVectorInterleave2Intrinsic(
     const CallInst &CI, MachineIRBuilder &MIRBuilder) {
   assert(CI.getIntrinsicID() == Intrinsic::experimental_vector_interleave2 &&
@@ -2459,22 +2485,11 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
     return true;
   }
   case Intrinsic::trap:
+    return translateTrap(CI, MIRBuilder, TargetOpcode::G_TRAP);
   case Intrinsic::debugtrap:
-  case Intrinsic::ubsantrap: {
-    StringRef TrapFuncName =
-        CI.getAttributes().getFnAttr("trap-func-name").getValueAsString();
-    if (TrapFuncName.empty())
-      break; // Use the default handling.
-    CallLowering::CallLoweringInfo Info;
-    if (ID == Intrinsic::ubsantrap) {
-      Info.OrigArgs.push_back({getOrCreateVRegs(*CI.getArgOperand(0)),
-                               CI.getArgOperand(0)->getType(), 0});
-    }
-    Info.Callee = MachineOperand::CreateES(TrapFuncName.data());
-    Info.CB = &CI;
-    Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0};
-    return CLI->lowerCall(MIRBuilder, Info);
-  }
+    return translateTrap(CI, MIRBuilder, TargetOpcode::G_DEBUGTRAP);
+  case Intrinsic::ubsantrap:
+    return translateTrap(CI, MIRBuilder, TargetOpcode::G_UBSANTRAP);
   case Intrinsic::amdgcn_cs_chain:
     return translateCallBase(CI, MIRBuilder);
   case Intrinsic::fptrunc_round: {
@@ -3047,7 +3062,7 @@ bool IRTranslator::translateUnreachable(const User &U, MachineIRBuilder &MIRBuil
     }
   }
 
-  MIRBuilder.buildIntrinsic(Intrinsic::trap, ArrayRef<Register>());
+  MIRBuilder.buildTrap();
   return true;
 }
 
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index 005efe48ac0c..9a1498d4070e 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -1867,6 +1867,17 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
 
     break;
   }
+  case TargetOpcode::G_UBSANTRAP: {
+    const MachineOperand &KindOp = MI->getOperand(0);
+    if (!MI->getOperand(0).isImm()) {
+      report("Crash kind must be an immediate", &KindOp, 0);
+      break;
+    }
+    int64_t Kind = MI->getOperand(0).getImm();
+    if (!isInt<8>(Kind))
+      report("Crash kind must be 8 bit wide", &KindOp, 0);
+    break;
+  }
   case TargetOpcode::G_VECREDUCE_SEQ_FADD:
   case TargetOpcode::G_VECREDUCE_SEQ_FMUL: {
     LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index b4b975cce007..b1f514f75207 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -8436,6 +8436,9 @@ def ubsan_trap_xform : SDNodeXForm<timm, [{
   return CurDAG->getTargetConstant(N->getZExtValue() | ('U' << 8), SDLoc(N), MVT::i32);
 }]>;
 
+def gi_ubsan_trap_xform : GICustomOperandRenderer<"renderUbsanTrap">,
+  GISDNodeXFormEquiv<ubsan_trap_xform>;
+
 def ubsan_trap_imm : TImmLeaf<i32, [{
   return isUInt<8>(Imm);
 }], ubsan_trap_xform>;
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 677dd0b502b9..a8f2c45279e6 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -479,6 +479,8 @@ private:
                           int OpIdx = -1) const;
   void renderLogicalImm64(MachineInstrBuilder &MIB, const MachineInstr &I,
                           int OpIdx = -1) const;
+  void renderUbsanTrap(MachineInstrBuilder &MIB, const MachineInstr &MI,
+                       int OpIdx) const;
   void renderFPImm16(MachineInstrBuilder &MIB, const MachineInstr &MI,
                      int OpIdx = -1) const;
   void renderFPImm32(MachineInstrBuilder &MIB, const MachineInstr &MI,
@@ -6159,16 +6161,6 @@ bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
     constrainSelectedInstRegOperands(*NewI, TII, TRI, RBI);
     break;
   }
-  case Intrinsic::trap:
-    MIB.buildInstr(AArch64::BRK, {}, {}).addImm(1);
-    break;
-  case Intrinsic::debugtrap:
-    MIB.buildInstr(AArch64::BRK, {}, {}).addImm(0xF000);
-    break;
-  case Intrinsic::ubsantrap:
-    MIB.buildInstr(AArch64::BRK, {}, {})
-        .addImm(I.getOperand(1).getImm() | ('U' << 8));
-    break;
   case Intrinsic::aarch64_neon_ld1x2: {
     LLT Ty = MRI.getType(I.getOperand(0).getReg());
     unsigned Opc = 0;
@@ -7663,6 +7655,14 @@ void AArch64InstructionSelector::renderLogicalImm64(
   MIB.addImm(Enc);
 }
 
+void AArch64InstructionSelector::renderUbsanTrap(MachineInstrBuilder &MIB,
+                                                 const MachineInstr &MI,
+                                                 int OpIdx) const {
+  assert(MI.getOpcode() == TargetOpcode::G_UBSANTRAP && OpIdx == 0 &&
+         "Expected G_UBSANTRAP");
+  MIB.addImm(MI.getOperand(0).getImm() | ('U' << 8));
+}
+
 void AArch64InstructionSelector::renderFPImm16(MachineInstrBuilder &MIB,
                                                const MachineInstr &MI,
                                                int OpIdx) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 90872516dd6d..e55d1de01b4f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -2030,6 +2030,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
   getActionDefinitionsBuilder({G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE, G_MEMSET})
       .lower();
 
+  getActionDefinitionsBuilder({G_TRAP, G_DEBUGTRAP}).custom();
+
   getActionDefinitionsBuilder({G_VASTART, G_VAARG, G_BRJT, G_JUMP_TABLE,
         G_INDEXED_LOAD, G_INDEXED_SEXTLOAD,
         G_INDEXED_ZEXTLOAD, G_INDEXED_STORE})
@@ -2134,6 +2136,10 @@ bool AMDGPULegalizerInfo::legalizeCustom(
     return legalizeGetFPEnv(MI, MRI, B);
   case TargetOpcode::G_SET_FPENV:
     return legalizeSetFPEnv(MI, MRI, B);
+  case TargetOpcode::G_TRAP:
+    return legalizeTrap(MI, MRI, B);
+  case TargetOpcode::G_DEBUGTRAP:
+    return legalizeDebugTrap(MI, MRI, B);
   default:
     return false;
   }
@@ -2925,7 +2931,7 @@ bool AMDGPULegalizerInfo::legalizeGlobalValue(
       // functions that use local objects. However, if these dead functions are
       // not eliminated, we don't want a compile time error. Just emit a warning
       // and a trap, since there should be no callable path here.
-      B.buildIntrinsic(Intrinsic::trap, ArrayRef<Register>());
+      B.buildTrap();
       B.buildUndef(DstReg);
       MI.eraseFromParent();
       return true;
@@ -6618,9 +6624,9 @@ bool AMDGPULegalizerInfo::legalizeSBufferLoad(LegalizerHelper &Helper,
 }
 
 // TODO: Move to selection
-bool AMDGPULegalizerInfo::legalizeTrapIntrinsic(MachineInstr &MI,
-                                                MachineRegisterInfo &MRI,
-                                                MachineIRBuilder &B) const {
+bool AMDGPULegalizerInfo::legalizeTrap(MachineInstr &MI,
+                                       MachineRegisterInfo &MRI,
+                                       MachineIRBuilder &B) const {
   if (!ST.isTrapHandlerEnabled() ||
       ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA)
     return legalizeTrapEndpgm(MI, MRI, B);
@@ -6726,8 +6732,9 @@ bool AMDGPULegalizerInfo::legalizeTrapHsa(
   return true;
 }
 
-bool AMDGPULegalizerInfo::legalizeDebugTrapIntrinsic(
-    MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
+bool AMDGPULegalizerInfo::legalizeDebugTrap(MachineInstr &MI,
+                                            MachineRegisterInfo &MRI,
+                                            MachineIRBuilder &B) const {
   // Is non-HSA path or trap-handler disabled? Then, report a warning
   // accordingly
   if (!ST.isTrapHandlerEnabled() ||
@@ -7270,10 +7277,6 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
   case Intrinsic::amdgcn_struct_buffer_atomic_fadd_v2bf16:
   case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd_v2bf16:
     return legalizeBufferAtomic(MI, B, IntrID);
-  case Intrinsic::trap:
-    return legalizeTrapIntrinsic(MI, MRI, B);
-  case Intrinsic::debugtrap:
-    return legalizeDebugTrapIntrinsic(MI, MRI, B);
   case Intrinsic::amdgcn_rsq_clamp:
     return legalizeRsqClampIntrinsic(MI, MRI, B);
   case Intrinsic::amdgcn_ds_fadd:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index 9661646fffc9..e5ba84a74a0f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -226,16 +226,16 @@ public:
 
   bool legalizeSBufferLoad(LegalizerHelper &Helper, MachineInstr &MI) const;
 
-  bool legalizeTrapIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI,
-                             MachineIRBuilder &B) const;
+  bool legalizeTrap(MachineInstr &MI, MachineRegisterInfo &MRI,
+                    MachineIRBuilder &B) const;
   bool legalizeTrapEndpgm(MachineInstr &MI, MachineRegisterInfo &MRI,
                           MachineIRBuilder &B) const;
   bool legalizeTrapHsaQueuePtr(MachineInstr &MI, MachineRegisterInfo &MRI,
                                MachineIRBuilder &B) const;
   bool legalizeTrapHsa(MachineInstr &MI, MachineRegisterInfo &MRI,
                        MachineIRBuilder &B) const;
-  bool legalizeDebugTrapIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI,
-                                  MachineIRBuilder &B) const;
+  bool legalizeDebugTrap(MachineInstr &MI, MachineRegisterInfo &MRI,
+                         MachineIRBuilder &B) const;
 
   bool legalizeIntrinsic(LegalizerHelper &Helper,
                          MachineInstr &MI) const override;
diff --git a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp
index 3307e840a2af..8468dd6a2211 100644
--- a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp
+++ b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp
@@ -508,16 +508,8 @@ bool MipsLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
                                           MachineInstr &MI) const {
   MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
   const MipsSubtarget &ST = MI.getMF()->getSubtarget<MipsSubtarget>();
-  const MipsInstrInfo &TII = *ST.getInstrInfo();
-  const MipsRegisterInfo &TRI = *ST.getRegisterInfo();
-  const RegisterBankInfo &RBI = *ST.getRegBankInfo();
 
   switch (cast<GIntrinsic>(MI).getIntrinsicID()) {
-  case Intrinsic::trap: {
-    MachineInstr *Trap = MIRBuilder.buildInstr(Mips::TRAP);
-    MI.eraseFromParent();
-    return constrainSelectedInstRegOperands(*Trap, TII, TRI, RBI);
-  }
   case Intrinsic::vacopy: {
     MachinePointerInfo MPO;
     LLT PtrTy = LLT::pointer(0, 32);
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
index 5738f86e7e9f..3103992a86c0 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -77,8 +77,6 @@ private:
                     MachineRegisterInfo &MRI) const;
   bool selectFPCompare(MachineInstr &MI, MachineIRBuilder &MIB,
                        MachineRegisterInfo &MRI) const;
-  bool selectIntrinsicWithSideEffects(MachineInstr &MI, MachineIRBuilder &MIB,
-                                      MachineRegisterInfo &MRI) const;
   void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
                  MachineIRBuilder &MIB) const;
   bool selectMergeValues(MachineInstr &MI, MachineIRBuilder &MIB,
@@ -686,8 +684,6 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) {
     return selectSelect(MI, MIB, MRI);
   case TargetOpcode::G_FCMP:
     return selectFPCompare(MI, MIB, MRI);
-  case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
-    return selectIntrinsicWithSideEffects(MI, MIB, MRI);
   case TargetOpcode::G_FENCE: {
     AtomicOrdering FenceOrdering =
         static_cast<AtomicOrdering>(MI.getOperand(0).getImm());
@@ -1255,29 +1251,6 @@ bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI,
   return true;
 }
 
-bool RISCVInstructionSelector::selectIntrinsicWithSideEffects(
-    MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const {
-  assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS &&
-         "Unexpected opcode");
-  // Find the intrinsic ID.
-  unsigned IntrinID = cast<GIntrinsic>(MI).getIntrinsicID();
-
-  // Select the instruction.
-  switch (IntrinID) {
-  default:
-    return false;
-  case Intrinsic::trap:
-    MIB.buildInstr(RISCV::UNIMP, {}, {});
-    break;
-  case Intrinsic::debugtrap:
-    MIB.buildInstr(RISCV::EBREAK, {}, {});
-    break;
-  }
-
-  MI.eraseFromParent();
-  return true;
-}
-
 void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
                                          SyncScope::ID FenceSSID,
                                          MachineIRBuilder &MIB) const {
diff --git a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
index 8e0f61a85566..9be3812300af 100644
--- a/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
+++ b/llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
@@ -119,8 +119,6 @@ private:
                        MachineFunction &MF) const;
   bool selectSelect(MachineInstr &I, MachineRegisterInfo &MRI,
                     MachineFunction &MF) const;
-  bool selectIntrinsicWSideEffects(MachineInstr &I, MachineRegisterInfo &MRI,
-                                   MachineFunction &MF) const;
 
   // emit insert subreg instruction and insert it before MachineInstr &I
   bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
@@ -434,8 +432,6 @@ bool X86InstructionSelector::select(MachineInstr &I) {
     return selectMulDivRem(I, MRI, MF);
   case TargetOpcode::G_SELECT:
     return selectSelect(I, MRI, MF);
-  case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
-    return selectIntrinsicWSideEffects(I, MRI, MF);
   }
 
   return false;
@@ -1834,21 +1830,6 @@ bool X86InstructionSelector::selectSelect(MachineInstr &I,
   return true;
 }
 
-bool X86InstructionSelector::selectIntrinsicWSideEffects(
-    MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const {
-
-  assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS &&
-         "unexpected instruction");
-
-  if (I.getOperand(0).getIntrinsicID() != Intrinsic::trap)
-    return false;
-
-  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TRAP));
-
-  I.eraseFromParent();
-  return true;
-}
-
 InstructionSelector *
 llvm::createX86InstructionSelector(const X86TargetMachine &TM,
                                    X86Subtarget &Subtarget,
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unreachable.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unreachable.ll
index fe9427d2678a..edae903fae84 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unreachable.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-unreachable.ll
@@ -6,7 +6,7 @@ declare void @llvm.trap()
 define void @unreachable() {
   ; CHECK-LABEL: name: unreachable
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+  ; CHECK-NEXT:   G_TRAP
   unreachable
   ret void
 }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll
index 5662de4cbdca..f7550ceb2799 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-exceptions.ll
@@ -48,7 +48,7 @@ define void @bar() personality ptr @__gxx_personality_v0 {
   ; CHECK-NEXT:   $x0 = COPY [[LOAD]](p0)
   ; CHECK-NEXT:   BL @_Unwind_Resume, csr_darwin_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0
   ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
-  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+  ; CHECK-NEXT:   G_TRAP
   %exn.slot = alloca ptr
   %ehselector.slot = alloca i32
   %1 = invoke i32 @foo(i32 42) to label %continue unwind label %cleanup
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index c9e5f8924f8a..ac3c47c8001d 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -752,6 +752,15 @@
 # DEBUG-NEXT: G_BZERO (opcode {{[0-9]+}}): 2 type indices, 1 imm index
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: G_TRAP (opcode {{[0-9]+}}): 0 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_DEBUGTRAP (opcode {{[0-9]+}}): 0 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_UBSANTRAP (opcode {{[0-9]+}}): 0 type indices, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_VECREDUCE_SEQ_FADD (opcode {{[0-9]+}}): 3 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-trap.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-trap.mir
index ad66fa5623e3..25ecce4dd92b 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-trap.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-trap.mir
@@ -26,7 +26,7 @@ body:             |
     ; CHECK-LABEL: name: foo
     ; CHECK: BRK 1
     ; CHECK: RET_ReallyLR
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+    G_TRAP
     RET_ReallyLR
 
 ...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/uaddo-8-16-bits.mir b/llvm/test/CodeGen/AArch64/GlobalISel/uaddo-8-16-bits.mir
index f4366fb7888e..b242c68e3b07 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/uaddo-8-16-bits.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/uaddo-8-16-bits.mir
@@ -26,7 +26,7 @@ body:             |
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors:
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+  ; CHECK-NEXT:   G_TRAP
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   $w0 = COPY [[ADD]](s32)
@@ -48,7 +48,7 @@ body:             |
   bb.2:
     successors:
 
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+    G_TRAP
 
   bb.3:
     %8:_(s32) = G_ZEXT %6(s8)
@@ -80,7 +80,7 @@ body:             |
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors:
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+  ; CHECK-NEXT:   G_TRAP
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   $w0 = COPY [[ADD]](s32)
@@ -102,7 +102,7 @@ body:             |
   bb.2:
     successors:
 
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+    G_TRAP
 
   bb.3:
     %8:_(s32) = G_ZEXT %6(s16)
@@ -134,7 +134,7 @@ body:             |
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors:
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+  ; CHECK-NEXT:   G_TRAP
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   liveins: $x2
@@ -165,7 +165,7 @@ body:             |
   bb.2:
     successors:
 
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+    G_TRAP
 
   bb.3:
     liveins: $x2
@@ -206,7 +206,7 @@ body:             |
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors:
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+  ; CHECK-NEXT:   G_TRAP
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   $w0 = COPY [[ADD]](s32)
@@ -228,7 +228,7 @@ body:             |
   bb.2:
     successors:
 
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+    G_TRAP
 
   bb.3:
     %8:_(s32) = G_ANYEXT %6(s16)
@@ -261,7 +261,7 @@ body:             |
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors:
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+  ; CHECK-NEXT:   G_TRAP
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   $w0 = COPY [[ADD]](s32)
@@ -284,7 +284,7 @@ body:             |
   bb.2:
     successors:
 
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+    G_TRAP
 
   bb.3:
     %8:_(s32) = G_ZEXT %6(s16)
@@ -317,7 +317,7 @@ body:             |
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors:
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+  ; CHECK-NEXT:   G_TRAP
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   $w0 = COPY [[ADD]](s32)
@@ -340,7 +340,7 @@ body:             |
   bb.2:
     successors:
 
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+    G_TRAP
 
   bb.3:
     %8:_(s32) = G_ZEXT %6(s16)
@@ -377,7 +377,7 @@ body:             |
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors:
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+  ; CHECK-NEXT:   G_TRAP
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   liveins: $x2
@@ -410,7 +410,7 @@ body:             |
   bb.2:
     successors:
 
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+    G_TRAP
 
   bb.3:
     liveins: $x2
@@ -512,7 +512,7 @@ body:             |
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   successors:
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+  ; CHECK-NEXT:   G_TRAP
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3:
   ; CHECK-NEXT:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO]](s16)
@@ -544,7 +544,7 @@ body:             |
   bb.2:
     successors:
 
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+    G_TRAP
 
   bb.3:
     %9:_(s32) = G_ZEXT %6(s16)
@@ -577,7 +577,7 @@ body:             |
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors:
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+  ; CHECK-NEXT:   G_TRAP
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO]](s8)
@@ -601,7 +601,7 @@ body:             |
   bb.2:
     successors:
 
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+    G_TRAP
 
   bb.3:
     %9:_(s32) = G_ZEXT %7(s8)
@@ -634,7 +634,7 @@ body:             |
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors:
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+  ; CHECK-NEXT:   G_TRAP
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO]](s8)
@@ -658,7 +658,7 @@ body:             |
   bb.2:
     successors:
 
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+    G_TRAP
 
   bb.3:
     %9:_(s32) = G_ZEXT %7(s8)
@@ -692,7 +692,7 @@ body:             |
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors:
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+  ; CHECK-NEXT:   G_TRAP
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO]](s8)
@@ -717,7 +717,7 @@ body:             |
   bb.2:
     successors:
 
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+    G_TRAP
 
   bb.3:
     %10:_(s32) = G_ZEXT %8(s8)
@@ -783,7 +783,7 @@ body:             |
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors:
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+  ; CHECK-NEXT:   G_TRAP
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UADDO]](s16)
@@ -804,7 +804,7 @@ body:             |
   bb.2:
     successors:
 
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+    G_TRAP
 
   bb.3:
     %6:_(s32) = G_ANYEXT %4(s16)
@@ -839,7 +839,7 @@ body:             |
   ; CHECK-NEXT:   RET_ReallyLR implicit $w0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+  ; CHECK-NEXT:   G_TRAP
   bb.1:
     successors: %bb.2(0x7ffff800), %bb.3(0x00000800)
     liveins: $w0, $w1
@@ -860,6 +860,6 @@ body:             |
     RET_ReallyLR implicit $w0
 
   bb.3:
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+    G_TRAP
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trap.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trap.mir
index b4bc64812b53..305eca792cfb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trap.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trap.mir
@@ -24,7 +24,7 @@ body: |
   bb.0:
     %0:_(s8) = G_CONSTANT i8 0
     %1:_(p1) = G_CONSTANT i64 0
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+    G_TRAP
 
   bb.1:
     G_STORE %0, %1 :: (store 1, addrspace 1)
@@ -55,7 +55,7 @@ body: |
   ; GCN-NEXT:   S_ENDPGM 0
   bb.0:
     %0:_(s8) = G_CONSTANT i8 0
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+    G_TRAP
     %1:_(p1) = G_CONSTANT i64 0
 
   bb.1:
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/trap.mir b/llvm/test/CodeGen/Mips/GlobalISel/instruction-select/trap.mir
similarity index 55%
rename from llvm/test/CodeGen/Mips/GlobalISel/legalizer/trap.mir
rename to llvm/test/CodeGen/Mips/GlobalISel/instruction-select/trap.mir
index 64388933fda8..dc99ce8d7a09 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/trap.mir
+++ b/llvm/test/CodeGen/Mips/GlobalISel/instruction-select/trap.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
 --- |
 
   declare void @llvm.trap()
@@ -9,12 +9,15 @@
 ---
 name:            f
 alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
 body:             |
   bb.1 (%ir-block.0):
     ; MIPS32-LABEL: name: f
     ; MIPS32: TRAP
-    ; MIPS32: RetRA
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+    ; MIPS32-NEXT: RetRA
+    G_TRAP
     RetRA
 
 ...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/trap.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/trap.mir
index 11789a030e6f..5f52030fc170 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/trap.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/trap.mir
@@ -14,7 +14,7 @@ body:             |
     ; CHECK-LABEL: name: test_trap
     ; CHECK: UNIMP
     ; CHECK-NEXT: PseudoRET
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+    G_TRAP
     PseudoRET
 
 ...
@@ -28,7 +28,7 @@ body:             |
     ; CHECK-LABEL: name: test_debugtrap
     ; CHECK: EBREAK
     ; CHECK-NEXT: PseudoRET
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.debugtrap)
+    G_DEBUGTRAP
     PseudoRET
 
 ...
diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86-select-trap.mir b/llvm/test/CodeGen/X86/GlobalISel/x86-select-trap.mir
index ea548c296dca..20b8b671ac5a 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/x86-select-trap.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/x86-select-trap.mir
@@ -23,6 +23,6 @@ body:             |
   bb.1 (%ir-block.0):
     ; CHECK-LABEL: name: trap
     ; CHECK: TRAP
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.trap)
+    G_TRAP
 
 ...
diff --git a/llvm/test/CodeGen/X86/isel-traps.ll b/llvm/test/CodeGen/X86/isel-traps.ll
new file mode 100644
index 000000000000..c207387166a6
--- /dev/null
+++ b/llvm/test/CodeGen/X86/isel-traps.ll
@@ -0,0 +1,73 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s                                   -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=ALL,X64
+; RUN: llc < %s -fast-isel -fast-isel-abort=1     -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=ALL,X64
+; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=ALL,GISEL-X64
+; RUN: llc < %s                                   -mtriple=i686-linux-gnu   | FileCheck %s --check-prefixes=ALL,X86
+; RUN: llc < %s -fast-isel -fast-isel-abort=1     -mtriple=i686-linux-gnu   | FileCheck %s --check-prefixes=ALL,X86
+; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=i686-linux-gnu   | FileCheck %s --check-prefixes=ALL,GISEL-X86
+
+declare void @llvm.trap()
+
+define void @test_trap() {
+; ALL-LABEL: test_trap:
+; ALL:       # %bb.0:
+; ALL-NEXT:    ud2
+; ALL-NEXT:    ret{{[l|q]}}
+  tail call void @llvm.trap()
+  ret void
+}
+
+define void @test_debugtrap() {
+; ALL-LABEL: test_debugtrap:
+; ALL:       # %bb.0:
+; ALL-NEXT:    int3
+; ALL-NEXT:    ret{{[l|q]}}
+  tail call void @llvm.debugtrap()
+  ret void
+}
+
+define void @test_ubsantrap() {
+; ALL-LABEL: test_ubsantrap:
+; ALL:       # %bb.0:
+; ALL-NEXT:    ud1l 12(%eax), %eax
+; ALL-NEXT:    ret{{[l|q]}}
+  call void @llvm.ubsantrap(i8 12)
+  ret void
+}
+
+define void @test_ubsantrap_custom() nounwind {
+; X64-LABEL: test_ubsantrap_custom:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    movl $42, %edi
+; X64-NEXT:    callq guide@PLT
+; X64-NEXT:    popq %rax
+; X64-NEXT:    retq
+;
+; GISEL-X64-LABEL: test_ubsantrap_custom:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    movl $42, %edi
+; GISEL-X64-NEXT:    callq guide
+; GISEL-X64-NEXT:    popq %rax
+; GISEL-X64-NEXT:    retq
+;
+; X86-LABEL: test_ubsantrap_custom:
+; X86:       # %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    movl $42, (%esp)
+; X86-NEXT:    calll guide
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: test_ubsantrap_custom:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $12, %esp
+; GISEL-X86-NEXT:    movl $42, %eax
+; GISEL-X86-NEXT:    movl %eax, (%esp)
+; GISEL-X86-NEXT:    calll guide
+; GISEL-X86-NEXT:    addl $12, %esp
+; GISEL-X86-NEXT:    retl
+  call void @llvm.ubsantrap(i8 42) "trap-func-name"="guide"
+  ret void
+}
diff --git a/llvm/test/MachineVerifier/test_g_ubsantrap.mir b/llvm/test/MachineVerifier/test_g_ubsantrap.mir
new file mode 100644
index 000000000000..d2b219d8650a
--- /dev/null
+++ b/llvm/test/MachineVerifier/test_g_ubsantrap.mir
@@ -0,0 +1,18 @@
+# RUN: not --crash llc -o - -mtriple=arm64 -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: aarch64-registered-target
+
+---
+name:            test_ubsantrap
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+
+    ; CHECK: Crash kind must be 8 bit wide
+    G_UBSANTRAP 4096
+
+    ; CHECK: Crash kind must be an immediate
+    %5:_(s32) = IMPLICIT_DEF
+    G_UBSANTRAP %5
+
+...
-- 
GitLab


From f886dfed3ae6cf70827cedc8d8aefde6250a239b Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Sat, 23 Mar 2024 13:54:35 +0100
Subject: [PATCH 047/404] [libc++] Don't push and pop extensions diagnostics
 when using clang modules (#85917)

Clang modules take a significant compile time hit when pushing and
popping diagnostics. Since all the headers are marked as system headers
in the modulemap, we can simply disable this pushing and popping when
building with clang modules.
---
 libcxx/include/__config | 34 +++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/libcxx/include/__config b/libcxx/include/__config
index 132cace3d5b2..af44af203e7c 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -838,21 +838,33 @@ typedef __char32_t char32_t;
 #  define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++2b-extensions")
 #endif
 
+// Clang modules take a significant compile time hit when pushing and popping diagnostics.
+// Since all the headers are marked as system headers in the modulemap, we can simply disable this
+// pushing and popping when building with clang modules.
+#  if !__has_feature(modules)
+#    define _LIBCPP_PUSH_EXTENSION_DIAGNOSTICS                                                                         \
+      _LIBCPP_DIAGNOSTIC_PUSH                                                                                          \
+      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++11-extensions")                                                           \
+      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++14-extensions")                                                           \
+      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++17-extensions")                                                           \
+      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++20-extensions")                                                           \
+      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION                                                                 \
+      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++14-extensions")                                                             \
+      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++17-extensions")                                                             \
+      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++20-extensions")                                                             \
+      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++23-extensions")
+#    define _LIBCPP_POP_EXTENSION_DIAGNOSTICS _LIBCPP_DIAGNOSTIC_POP
+#  else
+#    define _LIBCPP_PUSH_EXTENSION_DIAGNOSTICS
+#    define _LIBCPP_POP_EXTENSION_DIAGNOSTICS
+#  endif
+
 // Inline namespaces are available in Clang/GCC/MSVC regardless of C++ dialect.
 // clang-format off
-#  define _LIBCPP_BEGIN_NAMESPACE_STD _LIBCPP_DIAGNOSTIC_PUSH                                                          \
-                                      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++11-extensions")                           \
-                                      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++14-extensions")                           \
-                                      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++17-extensions")                           \
-                                      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++20-extensions")                           \
-                                      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION                                 \
-                                      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++14-extensions")                             \
-                                      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++17-extensions")                             \
-                                      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++20-extensions")                             \
-                                      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++23-extensions")                             \
+#  define _LIBCPP_BEGIN_NAMESPACE_STD _LIBCPP_PUSH_EXTENSION_DIAGNOSTICS                                               \
                                       namespace _LIBCPP_TYPE_VISIBILITY_DEFAULT std {                                  \
                                inline namespace _LIBCPP_ABI_NAMESPACE {
-#  define _LIBCPP_END_NAMESPACE_STD }} _LIBCPP_DIAGNOSTIC_POP
+#  define _LIBCPP_END_NAMESPACE_STD }} _LIBCPP_POP_EXTENSION_DIAGNOSTICS
 
 #  define _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM _LIBCPP_BEGIN_NAMESPACE_STD                                               \
                                              inline namespace __fs { namespace filesystem {
-- 
GitLab


From 57146daeaaf366050dc913db910fcc2995a3e06d Mon Sep 17 00:00:00 2001
From: Harvin Iriawan <25712785+harviniriawan@users.noreply.github.com>
Date: Sat, 23 Mar 2024 12:56:25 +0000
Subject: [PATCH 048/404] [CodeGen] Update for scalable MemoryType in MMO
 (#70452)

Remove getSizeOrUnknown call when MachineMemOperand is created.  For Scalable
TypeSize, the MemoryType created becomes a scalable_vector.

2 MMOs that have scalable memory access can then use the updated BasicAA that
understands scalable LocationSize.

Original Patch by Harvin Iriawan
Co-authored-by: David Green <david.green@arm.com>
---
 llvm/include/llvm/Analysis/MemoryLocation.h   |  7 ---
 llvm/include/llvm/CodeGen/MachineFunction.h   |  5 +-
 llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp  | 34 +++++++++----
 llvm/lib/CodeGen/MachineInstr.cpp             | 37 ++++++++++----
 llvm/lib/CodeGen/MachineOperand.cpp           | 13 ++---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 51 ++++++++++++-------
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 22 ++++----
 .../SelectionDAGAddressAnalysis.cpp           |  2 -
 .../SelectionDAG/SelectionDAGBuilder.cpp      |  3 +-
 llvm/lib/Target/AArch64/AArch64InstrInfo.cpp  |  5 +-
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   | 22 ++++++--
 llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll |  6 +--
 .../alloca-load-store-scalable-array.ll       | 36 ++++++-------
 .../alloca-load-store-scalable-struct.ll      | 12 ++---
 .../rvv/alloca-load-store-scalable-array.ll   | 12 ++---
 .../rvv/alloca-load-store-scalable-struct.ll  |  8 +--
 .../RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll |  8 +--
 17 files changed, 165 insertions(+), 118 deletions(-)

diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h
index 830eed5d60ee..7d896c44f467 100644
--- a/llvm/include/llvm/Analysis/MemoryLocation.h
+++ b/llvm/include/llvm/Analysis/MemoryLocation.h
@@ -297,13 +297,6 @@ public:
     return MemoryLocation(Ptr, LocationSize::beforeOrAfterPointer(), AATags);
   }
 
-  // Return the exact size if the exact size is known at compiletime,
-  // otherwise return LocationSize::beforeOrAfterPointer().
-  static LocationSize getSizeOrUnknown(const TypeSize &T) {
-    return T.isScalable() ? LocationSize::beforeOrAfterPointer()
-                          : LocationSize::precise(T.getFixedValue());
-  }
-
   MemoryLocation() : Ptr(nullptr), Size(LocationSize::beforeOrAfterPointer()) {}
 
   explicit MemoryLocation(const Value *Ptr, LocationSize Size,
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index dfbf7a1e7aae..c2bff2794493 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -1058,8 +1058,9 @@ public:
                                           int64_t Offset, LocationSize Size) {
     return getMachineMemOperand(
         MMO, Offset,
-        !Size.hasValue() || Size.isScalable()
-            ? LLT()
+        !Size.hasValue() ? LLT()
+        : Size.isScalable()
+            ? LLT::scalable_vector(1, 8 * Size.getValue().getKnownMinValue())
             : LLT::scalar(8 * Size.getValue().getKnownMinValue()));
   }
   MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
index 9fc8ecd60b03..fb9656c09ca3 100644
--- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
@@ -128,14 +128,14 @@ bool GISelAddressing::aliasIsKnownForLoadStore(const MachineInstr &MI1,
     // vector objects on the stack.
     // BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the
     // following situations arise:
-    if (PtrDiff >= 0 && Size1.hasValue()) {
+    if (PtrDiff >= 0 && Size1.hasValue() && !Size1.isScalable()) {
       // [----BasePtr0----]
       //                         [---BasePtr1--]
       // ========PtrDiff========>
       IsAlias = !((int64_t)Size1.getValue() <= PtrDiff);
       return true;
     }
-    if (PtrDiff < 0 && Size2.hasValue()) {
+    if (PtrDiff < 0 && Size2.hasValue() && !Size2.isScalable()) {
       //                     [----BasePtr0----]
       // [---BasePtr1--]
       // =====(-PtrDiff)====>
@@ -248,10 +248,20 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
       return false;
   }
 
+  // If NumBytes is scalable and offset is not 0, conservatively return may
+  // alias
+  if ((MUC0.NumBytes.isScalable() && MUC0.Offset != 0) ||
+      (MUC1.NumBytes.isScalable() && MUC1.Offset != 0))
+    return true;
+
+  const bool BothNotScalable =
+      !MUC0.NumBytes.isScalable() && !MUC1.NumBytes.isScalable();
+
   // Try to prove that there is aliasing, or that there is no aliasing. Either
   // way, we can return now. If nothing can be proved, proceed with more tests.
   bool IsAlias;
-  if (GISelAddressing::aliasIsKnownForLoadStore(MI, Other, IsAlias, MRI))
+  if (BothNotScalable &&
+      GISelAddressing::aliasIsKnownForLoadStore(MI, Other, IsAlias, MRI))
     return IsAlias;
 
   // The following all rely on MMO0 and MMO1 being valid.
@@ -267,12 +277,18 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
       Size1.hasValue()) {
     // Use alias analysis information.
     int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
-    int64_t Overlap0 = Size0.getValue() + SrcValOffset0 - MinOffset;
-    int64_t Overlap1 = Size1.getValue() + SrcValOffset1 - MinOffset;
-    if (AA->isNoAlias(MemoryLocation(MUC0.MMO->getValue(), Overlap0,
-                                     MUC0.MMO->getAAInfo()),
-                      MemoryLocation(MUC1.MMO->getValue(), Overlap1,
-                                     MUC1.MMO->getAAInfo())))
+    int64_t Overlap0 =
+        Size0.getValue().getKnownMinValue() + SrcValOffset0 - MinOffset;
+    int64_t Overlap1 =
+        Size1.getValue().getKnownMinValue() + SrcValOffset1 - MinOffset;
+    LocationSize Loc0 =
+        Size0.isScalable() ? Size0 : LocationSize::precise(Overlap0);
+    LocationSize Loc1 =
+        Size1.isScalable() ? Size1 : LocationSize::precise(Overlap1);
+
+    if (AA->isNoAlias(
+            MemoryLocation(MUC0.MMO->getValue(), Loc0, MUC0.MMO->getAAInfo()),
+            MemoryLocation(MUC1.MMO->getValue(), Loc1, MUC1.MMO->getAAInfo())))
       return false;
   }
 
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index fe2f9ccd33a3..8102bb971ba6 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -1306,6 +1306,7 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
   LocationSize WidthB = MMOb->getSize();
   bool KnownWidthA = WidthA.hasValue();
   bool KnownWidthB = WidthB.hasValue();
+  bool BothMMONonScalable = !WidthA.isScalable() && !WidthB.isScalable();
 
   const Value *ValA = MMOa->getValue();
   const Value *ValB = MMOb->getValue();
@@ -1321,12 +1322,14 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
       SameVal = true;
   }
 
-  if (SameVal) {
+  if (SameVal && BothMMONonScalable) {
     if (!KnownWidthA || !KnownWidthB)
       return true;
     int64_t MaxOffset = std::max(OffsetA, OffsetB);
-    LocationSize LowWidth = (MinOffset == OffsetA) ? WidthA : WidthB;
-    return (MinOffset + (int)LowWidth.getValue() > MaxOffset);
+    int64_t LowWidth = (MinOffset == OffsetA)
+                           ? WidthA.getValue().getKnownMinValue()
+                           : WidthB.getValue().getKnownMinValue();
+    return (MinOffset + LowWidth > MaxOffset);
   }
 
   if (!AA)
@@ -1338,15 +1341,29 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
   assert((OffsetA >= 0) && "Negative MachineMemOperand offset");
   assert((OffsetB >= 0) && "Negative MachineMemOperand offset");
 
-  int64_t OverlapA = KnownWidthA ? WidthA.getValue() + OffsetA - MinOffset
-                                 : MemoryLocation::UnknownSize;
-  int64_t OverlapB = KnownWidthB ? WidthB.getValue() + OffsetB - MinOffset
-                                 : MemoryLocation::UnknownSize;
+  // If Scalable Location Size has non-zero offset, Width + Offset does not work
+  // at the moment
+  if ((WidthA.isScalable() && OffsetA > 0) ||
+      (WidthB.isScalable() && OffsetB > 0))
+    return true;
+
+  int64_t OverlapA =
+      KnownWidthA ? WidthA.getValue().getKnownMinValue() + OffsetA - MinOffset
+                  : MemoryLocation::UnknownSize;
+  int64_t OverlapB =
+      KnownWidthB ? WidthB.getValue().getKnownMinValue() + OffsetB - MinOffset
+                  : MemoryLocation::UnknownSize;
+
+  LocationSize LocA = (WidthA.isScalable() || !KnownWidthA)
+                          ? WidthA
+                          : LocationSize::precise(OverlapA);
+  LocationSize LocB = (WidthB.isScalable() || !KnownWidthB)
+                          ? WidthB
+                          : LocationSize::precise(OverlapB);
 
   return !AA->isNoAlias(
-      MemoryLocation(ValA, OverlapA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()),
-      MemoryLocation(ValB, OverlapB,
-                     UseTBAA ? MMOb->getAAInfo() : AAMDNodes()));
+      MemoryLocation(ValA, LocA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()),
+      MemoryLocation(ValB, LocB, UseTBAA ? MMOb->getAAInfo() : AAMDNodes()));
 }
 
 bool MachineInstr::mayAlias(AAResults *AA, const MachineInstr &Other,
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index 937ca539513a..ace05902d5df 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1107,12 +1107,13 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags F,
                                      const MDNode *Ranges, SyncScope::ID SSID,
                                      AtomicOrdering Ordering,
                                      AtomicOrdering FailureOrdering)
-    : MachineMemOperand(ptrinfo, F,
-                        !TS.hasValue() || TS.isScalable()
-                            ? LLT()
-                            : LLT::scalar(8 * TS.getValue().getKnownMinValue()),
-                        BaseAlignment, AAInfo, Ranges, SSID, Ordering,
-                        FailureOrdering) {}
+    : MachineMemOperand(
+          ptrinfo, F,
+          !TS.hasValue() ? LLT()
+          : TS.isScalable()
+              ? LLT::scalable_vector(1, 8 * TS.getValue().getKnownMinValue())
+              : LLT::scalar(8 * TS.getValue().getKnownMinValue()),
+          BaseAlignment, AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
 
 void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
   // The Value and Offset may differ due to CSE. But the flags and size
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index dcd0310734ad..e27a8bb8fdac 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -24200,7 +24200,7 @@ static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG) {
   // TODO: Use "BaseIndexOffset" to make this more effective.
   SDValue NewAddr = DAG.getMemBasePlusOffset(Ld->getBasePtr(), Offset, DL);
 
-  LocationSize StoreSize = MemoryLocation::getSizeOrUnknown(VT.getStoreSize());
+  LocationSize StoreSize = LocationSize::precise(VT.getStoreSize());
   MachineFunction &MF = DAG.getMachineFunction();
   MachineMemOperand *MMO;
   if (Offset.isScalable()) {
@@ -27845,14 +27845,10 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
                  : (LSN->getAddressingMode() == ISD::PRE_DEC)
                      ? -1 * C->getSExtValue()
                      : 0;
-      LocationSize Size =
-          MemoryLocation::getSizeOrUnknown(LSN->getMemoryVT().getStoreSize());
-      return {LSN->isVolatile(),
-              LSN->isAtomic(),
-              LSN->getBasePtr(),
-              Offset /*base offset*/,
-              Size,
-              LSN->getMemOperand()};
+      TypeSize Size = LSN->getMemoryVT().getStoreSize();
+      return {LSN->isVolatile(),           LSN->isAtomic(),
+              LSN->getBasePtr(),           Offset /*base offset*/,
+              LocationSize::precise(Size), LSN->getMemOperand()};
     }
     if (const auto *LN = cast<LifetimeSDNode>(N))
       return {false /*isVolatile*/,
@@ -27894,6 +27890,13 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
       return false;
   }
 
+  // If NumBytes is scalable and offset is not 0, conservatively return may
+  // alias
+  if ((MUC0.NumBytes.hasValue() && MUC0.NumBytes.isScalable() &&
+       MUC0.Offset != 0) ||
+      (MUC1.NumBytes.hasValue() && MUC1.NumBytes.isScalable() &&
+       MUC1.Offset != 0))
+    return true;
   // Try to prove that there is aliasing, or that there is no aliasing. Either
   // way, we can return now. If nothing can be proved, proceed with more tests.
   bool IsAlias;
@@ -27924,18 +27927,22 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
   Align OrigAlignment1 = MUC1.MMO->getBaseAlign();
   LocationSize Size0 = MUC0.NumBytes;
   LocationSize Size1 = MUC1.NumBytes;
+
   if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
-      Size0.hasValue() && Size1.hasValue() && Size0 == Size1 &&
-      OrigAlignment0 > Size0.getValue() &&
-      SrcValOffset0 % Size0.getValue() == 0 &&
-      SrcValOffset1 % Size1.getValue() == 0) {
+      Size0.hasValue() && Size1.hasValue() && !Size0.isScalable() &&
+      !Size1.isScalable() && Size0 == Size1 &&
+      OrigAlignment0 > Size0.getValue().getKnownMinValue() &&
+      SrcValOffset0 % Size0.getValue().getKnownMinValue() == 0 &&
+      SrcValOffset1 % Size1.getValue().getKnownMinValue() == 0) {
     int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value();
     int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value();
 
     // There is no overlap between these relatively aligned accesses of
     // similar size. Return no alias.
-    if ((OffAlign0 + (int64_t)Size0.getValue()) <= OffAlign1 ||
-        (OffAlign1 + (int64_t)Size1.getValue()) <= OffAlign0)
+    if ((OffAlign0 + static_cast<int64_t>(
+                         Size0.getValue().getKnownMinValue())) <= OffAlign1 ||
+        (OffAlign1 + static_cast<int64_t>(
+                         Size1.getValue().getKnownMinValue())) <= OffAlign0)
       return false;
   }
 
@@ -27952,12 +27959,18 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
       Size0.hasValue() && Size1.hasValue()) {
     // Use alias analysis information.
     int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
-    int64_t Overlap0 = Size0.getValue() + SrcValOffset0 - MinOffset;
-    int64_t Overlap1 = Size1.getValue() + SrcValOffset1 - MinOffset;
+    int64_t Overlap0 =
+        Size0.getValue().getKnownMinValue() + SrcValOffset0 - MinOffset;
+    int64_t Overlap1 =
+        Size1.getValue().getKnownMinValue() + SrcValOffset1 - MinOffset;
+    LocationSize Loc0 =
+        Size0.isScalable() ? Size0 : LocationSize::precise(Overlap0);
+    LocationSize Loc1 =
+        Size1.isScalable() ? Size1 : LocationSize::precise(Overlap1);
     if (AA->isNoAlias(
-            MemoryLocation(MUC0.MMO->getValue(), Overlap0,
+            MemoryLocation(MUC0.MMO->getValue(), Loc0,
                            UseTBAA ? MUC0.MMO->getAAInfo() : AAMDNodes()),
-            MemoryLocation(MUC1.MMO->getValue(), Overlap1,
+            MemoryLocation(MUC1.MMO->getValue(), Loc1,
                            UseTBAA ? MUC1.MMO->getAAInfo() : AAMDNodes())))
       return false;
   }
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index e2c07e7cb997..0ab5142ab816 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -8406,9 +8406,7 @@ SDValue SelectionDAG::getMemIntrinsicNode(
     EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
     MachineMemOperand::Flags Flags, LocationSize Size,
     const AAMDNodes &AAInfo) {
-  if (Size.hasValue() && MemVT.isScalableVector())
-    Size = LocationSize::beforeOrAfterPointer();
-  else if (Size.hasValue() && !Size.getValue())
+  if (Size.hasValue() && !Size.getValue())
     Size = LocationSize::precise(MemVT.getStoreSize());
 
   MachineFunction &MF = getMachineFunction();
@@ -8571,7 +8569,7 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
   if (PtrInfo.V.isNull())
     PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
 
-  LocationSize Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize());
+  LocationSize Size = LocationSize::precise(MemVT.getStoreSize());
   MachineFunction &MF = getMachineFunction();
   MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
                                                    Alignment, AAInfo, Ranges);
@@ -8692,8 +8690,7 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
     PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
 
   MachineFunction &MF = getMachineFunction();
-  LocationSize Size =
-      MemoryLocation::getSizeOrUnknown(Val.getValueType().getStoreSize());
+  LocationSize Size = LocationSize::precise(Val.getValueType().getStoreSize());
   MachineMemOperand *MMO =
       MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);
   return getStore(Chain, dl, Val, Ptr, MMO);
@@ -8746,8 +8743,8 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
 
   MachineFunction &MF = getMachineFunction();
   MachineMemOperand *MMO = MF.getMachineMemOperand(
-      PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()),
-      Alignment, AAInfo);
+      PtrInfo, MMOFlags, LocationSize::precise(SVT.getStoreSize()), Alignment,
+      AAInfo);
   return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
 }
 
@@ -8841,7 +8838,7 @@ SDValue SelectionDAG::getLoadVP(
   if (PtrInfo.V.isNull())
     PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
 
-  LocationSize Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize());
+  LocationSize Size = LocationSize::precise(MemVT.getStoreSize());
   MachineFunction &MF = getMachineFunction();
   MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
                                                    Alignment, AAInfo, Ranges);
@@ -8994,8 +8991,8 @@ SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
 
   MachineFunction &MF = getMachineFunction();
   MachineMemOperand *MMO = MF.getMachineMemOperand(
-      PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()),
-      Alignment, AAInfo);
+      PtrInfo, MMOFlags, LocationSize::precise(SVT.getStoreSize()), Alignment,
+      AAInfo);
   return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
                          IsCompressing);
 }
@@ -11734,10 +11731,9 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
   // We check here that the size of the memory operand fits within the size of
   // the MMO. This is because the MMO might indicate only a possible address
   // range instead of specifying the affected memory addresses precisely.
-  // TODO: Make MachineMemOperands aware of scalable vectors.
   assert(
       (!MMO->getType().isValid() ||
-       memvt.getStoreSize().getKnownMinValue() <= MMO->getSize().getValue()) &&
+       TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize().getValue())) &&
       "Size mismatch!");
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
index 9670c3ac8430..f2ab88851b78 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
@@ -106,8 +106,6 @@ bool BaseIndexOffset::computeAliasing(const SDNode *Op0,
   int64_t PtrDiff;
   if (BasePtr0.equalBaseIndex(BasePtr1, DAG, PtrDiff)) {
     // If the size of memory access is unknown, do not use it to analysis.
-    // One example of unknown size memory access is to load/store scalable
-    // vector objects on the stack.
     // BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the
     // following situations arise:
     if (PtrDiff >= 0 && NumBytes0.hasValue() && !NumBytes0.isScalable()) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 84df98b8a613..ae6bd7e938e8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -4962,7 +4962,8 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
   unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
       MachinePointerInfo(AS), MachineMemOperand::MOLoad,
-      LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata(), Ranges);
+      LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata(),
+      Ranges);
 
   if (!UniformBase) {
     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 02943b8a4ab1..d0c5e6b99e9e 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -2687,10 +2687,7 @@ bool AArch64InstrInfo::getMemOperandsWithOffsetWidth(
     return false;
   // The maximum vscale is 16 under AArch64, return the maximal extent for the
   // vector.
-  Width = WidthN.isScalable()
-              ? WidthN.getKnownMinValue() * AArch64::SVEMaxBitsPerVector /
-                    AArch64::SVEBitsPerBlock
-              : WidthN.getKnownMinValue();
+  Width = LocationSize::precise(WidthN);
   BaseOps.push_back(BaseOp);
   return true;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 5a2fb0239e0a..5214595485ca 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -10364,9 +10364,15 @@ RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op,
       RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget);
   if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
       getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
+    MachineMemOperand *MMO = Load->getMemOperand();
+    MachineFunction &MF = DAG.getMachineFunction();
+    MMO = MF.getMachineMemOperand(
+        MMO, MMO->getPointerInfo(),
+        MMO->getMemoryType().isValid()
+            ? LLT::scalable_vector(1, MMO->getMemoryType().getSizeInBits())
+            : MMO->getMemoryType());
     SDValue NewLoad =
-        DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(),
-                    Load->getMemOperand());
+        DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(), MMO);
     SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
     return DAG.getMergeValues({Result, NewLoad.getValue(1)}, DL);
   }
@@ -10424,9 +10430,17 @@ RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op,
   const auto [MinVLMAX, MaxVLMAX] =
       RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget);
   if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
-      getLMUL1VT(ContainerVT).bitsLE(ContainerVT))
+      getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
+    MachineMemOperand *MMO = Store->getMemOperand();
+    MachineFunction &MF = DAG.getMachineFunction();
+    MMO = MF.getMachineMemOperand(
+        MMO, MMO->getPointerInfo(),
+        MMO->getMemoryType().isValid()
+            ? LLT::scalable_vector(1, MMO->getMemoryType().getSizeInBits())
+            : MMO->getMemoryType());
     return DAG.getStore(Store->getChain(), DL, NewValue, Store->getBasePtr(),
-                        Store->getMemOperand());
+                        MMO);
+  }
 
   SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG,
                        Subtarget);
diff --git a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
index 58299696e78f..d4d803a91cfa 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-sme2-asm.ll
@@ -3,7 +3,7 @@
 define void @UphPNR(target("aarch64.svcount") %predcnt) {
 entry:
 ; CHECK:  %0:ppr = COPY $p0
-; CHECK:  STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2)
+; CHECK:  STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr)
 ; CHECK:  %1:pnr_p8to15 = COPY %0
 ; CHECK:  INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR_p8to15 */, %1
 ; CHECK:  RET_ReallyLR
@@ -17,7 +17,7 @@ entry:
 define void @UpaPNR(target("aarch64.svcount") %predcnt) {
 entry:
 ; CHECK:  %0:ppr = COPY $p0
-; CHECK:  STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2)
+; CHECK:  STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr)
 ; CHECK:  %1:pnr = COPY %0
 ; CHECK:  INLINEASM &"ld1w {z0.s,z1.s,z2.s,z3.s}, $0/z, [x10]", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR */, %1
 ; CHECK:  RET_ReallyLR
@@ -31,7 +31,7 @@ entry:
 define void @UplPNR(target("aarch64.svcount") %predcnt) {
 entry:
 ; CHECK:  %0:ppr = COPY $p0
-; CHECK:  STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store unknown-size into %ir.predcnt.addr, align 2)
+; CHECK:  STR_PXI %0, %stack.0.predcnt.addr, 0 :: (store (<vscale x 1 x s16>) into %ir.predcnt.addr)
 ; CHECK:  %1:pnr_3b = COPY %0
 ; CHECK:  INLINEASM &"fadd z0.h, $0/m, z0.h, #0.5", 1 /* sideeffect attdialect */, {{[0-9]+}} /* reguse:PNR_3b */, %1
 ; CHECK:  RET_ReallyLR
diff --git a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll
index 9a4e01a29ecb..7244ac949ab8 100644
--- a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll
+++ b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-array.ll
@@ -14,12 +14,12 @@ define void @array_1D(ptr %addr) #0 {
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x0, #1, mul vl]
-; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x0]
-; CHECK-NEXT:    st1d { z0.d }, p0, [sp, #2, mul vl]
-; CHECK-NEXT:    st1d { z1.d }, p0, [sp, #1, mul vl]
-; CHECK-NEXT:    st1d { z2.d }, p0, [sp]
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
+; CHECK-NEXT:    st1d { z1.d }, p0, [sp, #2, mul vl]
+; CHECK-NEXT:    st1d { z2.d }, p0, [sp, #1, mul vl]
 ; CHECK-NEXT:    addvl sp, sp, #3
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
@@ -81,18 +81,18 @@ define void @array_2D(ptr %addr) #0 {
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x30, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 48 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, #5, mul vl]
-; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x0, #4, mul vl]
-; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x0]
-; CHECK-NEXT:    ld1d { z3.d }, p0/z, [x0, #3, mul vl]
-; CHECK-NEXT:    ld1d { z4.d }, p0/z, [x0, #1, mul vl]
-; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT:    st1d { z0.d }, p0, [sp, #5, mul vl]
-; CHECK-NEXT:    st1d { z1.d }, p0, [sp, #4, mul vl]
-; CHECK-NEXT:    st1d { z3.d }, p0, [sp, #3, mul vl]
-; CHECK-NEXT:    st1d { z5.d }, p0, [sp, #2, mul vl]
-; CHECK-NEXT:    st1d { z4.d }, p0, [sp, #1, mul vl]
-; CHECK-NEXT:    st1d { z2.d }, p0, [sp]
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x0, #5, mul vl]
+; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT:    ld1d { z3.d }, p0/z, [x0, #4, mul vl]
+; CHECK-NEXT:    ld1d { z4.d }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT:    ld1d { z5.d }, p0/z, [x0, #3, mul vl]
+; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
+; CHECK-NEXT:    st1d { z1.d }, p0, [sp, #5, mul vl]
+; CHECK-NEXT:    st1d { z3.d }, p0, [sp, #4, mul vl]
+; CHECK-NEXT:    st1d { z5.d }, p0, [sp, #3, mul vl]
+; CHECK-NEXT:    st1d { z4.d }, p0, [sp, #2, mul vl]
+; CHECK-NEXT:    st1d { z2.d }, p0, [sp, #1, mul vl]
 ; CHECK-NEXT:    addvl sp, sp, #6
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll
index 7292d52aaf47..f03a6f018d34 100644
--- a/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll
+++ b/llvm/test/CodeGen/AArch64/alloca-load-store-scalable-struct.ll
@@ -13,12 +13,12 @@ define void @test(ptr %addr) #0 {
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, #2, mul vl]
-; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x0, #1, mul vl]
-; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x0]
-; CHECK-NEXT:    st1d { z0.d }, p0, [sp, #2, mul vl]
-; CHECK-NEXT:    st1d { z1.d }, p0, [sp, #1, mul vl]
-; CHECK-NEXT:    st1d { z2.d }, p0, [sp]
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT:    ld1d { z2.d }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
+; CHECK-NEXT:    st1d { z1.d }, p0, [sp, #2, mul vl]
+; CHECK-NEXT:    st1d { z2.d }, p0, [sp, #1, mul vl]
 ; CHECK-NEXT:    addvl sp, sp, #3
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll
index 1d025a2f776f..1fe91c721f4d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-array.ll
@@ -18,15 +18,15 @@ define void @test(ptr %addr) {
 ; CHECK-NEXT:    add a2, a0, a1
 ; CHECK-NEXT:    vl1re64.v v8, (a2)
 ; CHECK-NEXT:    slli a2, a1, 1
-; CHECK-NEXT:    add a3, a0, a2
-; CHECK-NEXT:    vl1re64.v v9, (a3)
+; CHECK-NEXT:    vl1re64.v v9, (a0)
+; CHECK-NEXT:    add a0, a0, a2
 ; CHECK-NEXT:    vl1re64.v v10, (a0)
 ; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vs1r.v v9, (a0)
 ; CHECK-NEXT:    add a2, a0, a2
-; CHECK-NEXT:    vs1r.v v9, (a2)
-; CHECK-NEXT:    add a1, a0, a1
-; CHECK-NEXT:    vs1r.v v8, (a1)
-; CHECK-NEXT:    vs1r.v v10, (a0)
+; CHECK-NEXT:    vs1r.v v10, (a2)
+; CHECK-NEXT:    add a0, a0, a1
+; CHECK-NEXT:    vs1r.v v8, (a0)
 ; CHECK-NEXT:    csrrs a0, vlenb, zero
 ; CHECK-NEXT:    slli a0, a0, 2
 ; CHECK-NEXT:    add sp, sp, a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll
index 64031f8a9359..a9a680d54d58 100644
--- a/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/alloca-load-store-scalable-struct.ll
@@ -16,13 +16,13 @@ define <vscale x 1 x double> @test(ptr %addr, i64 %vl) {
 ; CHECK-NEXT:    sub sp, sp, a2
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
 ; CHECK-NEXT:    csrrs a2, vlenb, zero
-; CHECK-NEXT:    add a3, a0, a2
-; CHECK-NEXT:    vl1re64.v v8, (a3)
+; CHECK-NEXT:    vl1re64.v v8, (a0)
+; CHECK-NEXT:    add a0, a0, a2
 ; CHECK-NEXT:    vl1re64.v v9, (a0)
 ; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vs1r.v v8, (a0)
 ; CHECK-NEXT:    add a2, a0, a2
-; CHECK-NEXT:    vs1r.v v8, (a2)
-; CHECK-NEXT:    vs1r.v v9, (a0)
+; CHECK-NEXT:    vs1r.v v9, (a2)
 ; CHECK-NEXT:    vl1re64.v v8, (a2)
 ; CHECK-NEXT:    vl1re64.v v9, (a0)
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
index 0544204cce79..52bd15742ef4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
@@ -16,8 +16,8 @@ define void @vpmerge_vpload_store(<vscale x 2 x i32> %passthru, ptr %p, <vscale
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr = COPY $x10
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
   ; CHECK-NEXT:   $v0 = COPY [[COPY1]]
-  ; CHECK-NEXT:   [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 0 /* tu, mu */
-  ; CHECK-NEXT:   VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store unknown-size into %ir.p, align 8)
+  ; CHECK-NEXT:   [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 0 /* tu, mu */ :: (load unknown-size from %ir.p, align 8)
+  ; CHECK-NEXT:   VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p)
   ; CHECK-NEXT:   PseudoRET
   %splat = insertelement <vscale x 2 x i1> poison, i1 -1, i32 0
   %mask = shufflevector <vscale x 2 x i1> %splat, <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
@@ -37,8 +37,8 @@ define void @vpselect_vpload_store(<vscale x 2 x i32> %passthru, ptr %p, <vscale
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr = COPY $x10
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
   ; CHECK-NEXT:   $v0 = COPY [[COPY1]]
-  ; CHECK-NEXT:   [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 1 /* ta, mu */
-  ; CHECK-NEXT:   VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store unknown-size into %ir.p, align 8)
+  ; CHECK-NEXT:   [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], $v0, [[COPY]], 5 /* e32 */, 1 /* ta, mu */ :: (load unknown-size from %ir.p, align 8)
+  ; CHECK-NEXT:   VS1R_V killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]] :: (store (<vscale x 1 x s64>) into %ir.p)
   ; CHECK-NEXT:   PseudoRET
   %splat = insertelement <vscale x 2 x i1> poison, i1 -1, i32 0
   %mask = shufflevector <vscale x 2 x i1> %splat, <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
-- 
GitLab


From b68e2eba0bc8dd70b88f4271831139ee9b6ed25c Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Sat, 23 Mar 2024 15:28:22 +0100
Subject: [PATCH 049/404] [libc++] Vectorize mismatch (#73255)

```
---------------------------------------------------
Benchmark                           old         new
---------------------------------------------------
bm_mismatch<char>/1           0.835 ns      2.37 ns
bm_mismatch<char>/2            1.44 ns      2.60 ns
bm_mismatch<char>/3            2.06 ns      2.83 ns
bm_mismatch<char>/4            2.60 ns      3.29 ns
bm_mismatch<char>/5            3.15 ns      3.77 ns
bm_mismatch<char>/6            3.82 ns      4.17 ns
bm_mismatch<char>/7            4.29 ns      4.52 ns
bm_mismatch<char>/8            4.78 ns      4.86 ns
bm_mismatch<char>/16           9.06 ns      7.54 ns
bm_mismatch<char>/64           31.7 ns      19.1 ns
bm_mismatch<char>/512           249 ns      8.16 ns
bm_mismatch<char>/4096         1956 ns      44.2 ns
bm_mismatch<char>/32768       15498 ns       501 ns
bm_mismatch<char>/262144     123965 ns      4479 ns
bm_mismatch<char>/1048576    495668 ns     21306 ns
bm_mismatch<short>/1          0.710 ns      2.12 ns
bm_mismatch<short>/2           1.03 ns      2.66 ns
bm_mismatch<short>/3           1.29 ns      3.56 ns
bm_mismatch<short>/4           1.68 ns      4.29 ns
bm_mismatch<short>/5           1.96 ns      5.18 ns
bm_mismatch<short>/6           2.59 ns      5.91 ns
bm_mismatch<short>/7           2.86 ns      6.63 ns
bm_mismatch<short>/8           3.19 ns      7.33 ns
bm_mismatch<short>/16          5.48 ns      13.0 ns
bm_mismatch<short>/64          16.6 ns      4.06 ns
bm_mismatch<short>/512          130 ns      13.8 ns
bm_mismatch<short>/4096         985 ns      93.8 ns
bm_mismatch<short>/32768       7846 ns      1002 ns
bm_mismatch<short>/262144     63217 ns     10637 ns
bm_mismatch<short>/1048576   251782 ns     42471 ns
bm_mismatch<int>/1            0.716 ns      1.91 ns
bm_mismatch<int>/2             1.21 ns      2.49 ns
bm_mismatch<int>/3             1.38 ns      3.46 ns
bm_mismatch<int>/4             1.71 ns      4.04 ns
bm_mismatch<int>/5             2.00 ns      4.98 ns
bm_mismatch<int>/6             2.43 ns      5.67 ns
bm_mismatch<int>/7             3.05 ns      6.38 ns
bm_mismatch<int>/8             3.22 ns      7.09 ns
bm_mismatch<int>/16            5.18 ns      12.8 ns
bm_mismatch<int>/64            16.6 ns      5.28 ns
bm_mismatch<int>/512            129 ns      25.2 ns
bm_mismatch<int>/4096          1009 ns       201 ns
bm_mismatch<int>/32768         7776 ns      2144 ns
bm_mismatch<int>/262144       62371 ns     20551 ns
bm_mismatch<int>/1048576     254750 ns     90097 ns
```
---
 libcxx/benchmarks/CMakeLists.txt              |   1 +
 .../benchmarks/algorithms/mismatch.bench.cpp  |  31 +++
 libcxx/docs/ReleaseNotes/19.rst               |   2 +
 libcxx/include/CMakeLists.txt                 |   1 +
 libcxx/include/__algorithm/mismatch.h         |  82 ++++++-
 libcxx/include/__algorithm/simd_utils.h       | 123 ++++++++++
 libcxx/include/__bit/bit_cast.h               |   9 +
 libcxx/include/__bit/countr.h                 |  13 +-
 libcxx/include/libcxx.imp                     |   1 +
 libcxx/include/module.modulemap               |   6 +-
 .../mismatch/mismatch.pass.cpp                | 214 +++++++++++++-----
 .../mismatch/mismatch_pred.pass.cpp           | 119 ----------
 12 files changed, 413 insertions(+), 189 deletions(-)
 create mode 100644 libcxx/benchmarks/algorithms/mismatch.bench.cpp
 create mode 100644 libcxx/include/__algorithm/simd_utils.h
 delete mode 100644 libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch_pred.pass.cpp

diff --git a/libcxx/benchmarks/CMakeLists.txt b/libcxx/benchmarks/CMakeLists.txt
index 3dec6faea13a..387e013afeb6 100644
--- a/libcxx/benchmarks/CMakeLists.txt
+++ b/libcxx/benchmarks/CMakeLists.txt
@@ -183,6 +183,7 @@ set(BENCHMARK_TESTS
     algorithms/make_heap_then_sort_heap.bench.cpp
     algorithms/min.bench.cpp
     algorithms/min_max_element.bench.cpp
+    algorithms/mismatch.bench.cpp
     algorithms/pop_heap.bench.cpp
     algorithms/pstl.stable_sort.bench.cpp
     algorithms/push_heap.bench.cpp
diff --git a/libcxx/benchmarks/algorithms/mismatch.bench.cpp b/libcxx/benchmarks/algorithms/mismatch.bench.cpp
new file mode 100644
index 000000000000..9274932a764c
--- /dev/null
+++ b/libcxx/benchmarks/algorithms/mismatch.bench.cpp
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <algorithm>
+#include <benchmark/benchmark.h>
+#include <random>
+
+// TODO: Look into benchmarking aligned and unaligned memory explicitly
+// (currently things happen to be aligned because they are malloced that way)
+template <class T>
+static void bm_mismatch(benchmark::State& state) {
+  std::vector<T> vec1(state.range(), '1');
+  std::vector<T> vec2(state.range(), '1');
+  std::mt19937_64 rng(std::random_device{}());
+
+  vec1.back() = '2';
+  for (auto _ : state) {
+    benchmark::DoNotOptimize(vec1);
+    benchmark::DoNotOptimize(std::mismatch(vec1.begin(), vec1.end(), vec2.begin()));
+  }
+}
+BENCHMARK(bm_mismatch<char>)->DenseRange(1, 8)->Range(16, 1 << 20);
+BENCHMARK(bm_mismatch<short>)->DenseRange(1, 8)->Range(16, 1 << 20);
+BENCHMARK(bm_mismatch<int>)->DenseRange(1, 8)->Range(16, 1 << 20);
+
+BENCHMARK_MAIN();
diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst
index cac42f9c3c3f..dd39c1bbbc78 100644
--- a/libcxx/docs/ReleaseNotes/19.rst
+++ b/libcxx/docs/ReleaseNotes/19.rst
@@ -51,6 +51,8 @@ Improvements and New Features
 - The performance of growing ``std::vector`` has been improved for trivially relocatable types.
 - The performance of ``ranges::fill`` and ``ranges::fill_n`` has been improved for ``vector<bool>::iterator``\s,
   resulting in a performance increase of up to 1400x.
+- The ``std::mismatch`` algorithm has been optimized for integral types, which can lead up to 40x performance
+  improvements.
 
 Deprecations and Removals
 -------------------------
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 6ed8d21d98a1..982b85e4e2d6 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -217,6 +217,7 @@ set(files
   __algorithm/shift_right.h
   __algorithm/shuffle.h
   __algorithm/sift_down.h
+  __algorithm/simd_utils.h
   __algorithm/sort.h
   __algorithm/sort_heap.h
   __algorithm/stable_partition.h
diff --git a/libcxx/include/__algorithm/mismatch.h b/libcxx/include/__algorithm/mismatch.h
index d345b6048a7e..4eb693a1f2e9 100644
--- a/libcxx/include/__algorithm/mismatch.h
+++ b/libcxx/include/__algorithm/mismatch.h
@@ -11,23 +11,93 @@
 #define _LIBCPP___ALGORITHM_MISMATCH_H
 
 #include <__algorithm/comp.h>
+#include <__algorithm/simd_utils.h>
+#include <__algorithm/unwrap_iter.h>
 #include <__config>
-#include <__iterator/iterator_traits.h>
+#include <__functional/identity.h>
+#include <__type_traits/invoke.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_equality_comparable.h>
+#include <__type_traits/operation_traits.h>
+#include <__utility/move.h>
 #include <__utility/pair.h>
+#include <__utility/unreachable.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
 
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+template <class _Iter1, class _Sent1, class _Iter2, class _Pred, class _Proj1, class _Proj2>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Iter1, _Iter2>
+__mismatch_loop(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
+  while (__first1 != __last1) {
+    if (!std::__invoke(__pred, std::__invoke(__proj1, *__first1), std::__invoke(__proj2, *__first2)))
+      break;
+    ++__first1;
+    ++__first2;
+  }
+  return std::make_pair(std::move(__first1), std::move(__first2));
+}
+
+template <class _Iter1, class _Sent1, class _Iter2, class _Pred, class _Proj1, class _Proj2>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Iter1, _Iter2>
+__mismatch(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
+  return std::__mismatch_loop(__first1, __last1, __first2, __pred, __proj1, __proj2);
+}
+
+#if _LIBCPP_VECTORIZE_ALGORITHMS
+
+template <class _Tp,
+          class _Pred,
+          class _Proj1,
+          class _Proj2,
+          __enable_if_t<is_integral<_Tp>::value && __desugars_to<__equal_tag, _Pred, _Tp, _Tp>::value &&
+                            __is_identity<_Proj1>::value && __is_identity<_Proj2>::value,
+                        int> = 0>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Tp*, _Tp*>
+__mismatch(_Tp* __first1, _Tp* __last1, _Tp* __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
+  constexpr size_t __unroll_count = 4;
+  constexpr size_t __vec_size     = __native_vector_size<_Tp>;
+  using __vec                     = __simd_vector<_Tp, __vec_size>;
+  if (!__libcpp_is_constant_evaluated()) {
+    while (static_cast<size_t>(__last1 - __first1) >= __unroll_count * __vec_size) [[__unlikely__]] {
+      __vec __lhs[__unroll_count];
+      __vec __rhs[__unroll_count];
+
+      for (size_t __i = 0; __i != __unroll_count; ++__i) {
+        __lhs[__i] = std::__load_vector<__vec>(__first1 + __i * __vec_size);
+        __rhs[__i] = std::__load_vector<__vec>(__first2 + __i * __vec_size);
+      }
+
+      for (size_t __i = 0; __i != __unroll_count; ++__i) {
+        if (auto __cmp_res = __lhs[__i] == __rhs[__i]; !std::__all_of(__cmp_res)) {
+          auto __offset = __i * __vec_size + std::__find_first_not_set(__cmp_res);
+          return {__first1 + __offset, __first2 + __offset};
+        }
+      }
+
+      __first1 += __unroll_count * __vec_size;
+      __first2 += __unroll_count * __vec_size;
+    }
+  }
+  // TODO: Consider vectorizing the tail
+  return std::__mismatch_loop(__first1, __last1, __first2, __pred, __proj1, __proj2);
+}
+
+#endif // _LIBCPP_VECTORIZE_ALGORITHMS
+
 template <class _InputIterator1, class _InputIterator2, class _BinaryPredicate>
 _LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator1, _InputIterator2>
 mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _BinaryPredicate __pred) {
-  for (; __first1 != __last1; ++__first1, (void)++__first2)
-    if (!__pred(*__first1, *__first2))
-      break;
-  return pair<_InputIterator1, _InputIterator2>(__first1, __first2);
+  __identity __proj;
+  auto __res = std::__mismatch(
+      std::__unwrap_iter(__first1), std::__unwrap_iter(__last1), std::__unwrap_iter(__first2), __pred, __proj, __proj);
+  return std::make_pair(std::__rewrap_iter(__first1, __res.first), std::__rewrap_iter(__first2, __res.second));
 }
 
 template <class _InputIterator1, class _InputIterator2>
@@ -59,4 +129,6 @@ mismatch(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __fi
 
 _LIBCPP_END_NAMESPACE_STD
 
+_LIBCPP_POP_MACROS
+
 #endif // _LIBCPP___ALGORITHM_MISMATCH_H
diff --git a/libcxx/include/__algorithm/simd_utils.h b/libcxx/include/__algorithm/simd_utils.h
new file mode 100644
index 000000000000..1aedb3db010f
--- /dev/null
+++ b/libcxx/include/__algorithm/simd_utils.h
@@ -0,0 +1,123 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_SIMD_UTILS_H
+#define _LIBCPP___ALGORITHM_SIMD_UTILS_H
+
+#include <__bit/bit_cast.h>
+#include <__bit/countr.h>
+#include <__config>
+#include <__type_traits/is_arithmetic.h>
+#include <__type_traits/is_same.h>
+#include <__utility/integer_sequence.h>
+#include <cstddef>
+#include <cstdint>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+// TODO: Find out how altivec changes things and allow vectorizations there too.
+#if _LIBCPP_STD_VER >= 14 && defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 1700 && !defined(__ALTIVEC__)
+#  define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 1
+#else
+#  define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 0
+#endif
+
+#if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS && !defined(__OPTIMIZE_SIZE__)
+#  define _LIBCPP_VECTORIZE_ALGORITHMS 1
+#else
+#  define _LIBCPP_VECTORIZE_ALGORITHMS 0
+#endif
+
+#if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// This isn't specialized for 64 byte vectors on purpose. They have the potential to significantly reduce performance
+// in mixed simd/non-simd workloads and don't provide any performance improvement for currently vectorized algorithms
+// as far as benchmarks are concerned.
+#  if defined(__AVX__)
+template <class _Tp>
+inline constexpr size_t __native_vector_size = 32 / sizeof(_Tp);
+#  elif defined(__SSE__) || defined(__ARM_NEON__)
+template <class _Tp>
+inline constexpr size_t __native_vector_size = 16 / sizeof(_Tp);
+#  elif defined(__MMX__)
+template <class _Tp>
+inline constexpr size_t __native_vector_size = 8 / sizeof(_Tp);
+#  else
+template <class _Tp>
+inline constexpr size_t __native_vector_size = 1;
+#  endif
+
+template <class _ArithmeticT, size_t _Np>
+using __simd_vector __attribute__((__ext_vector_type__(_Np))) = _ArithmeticT;
+
+template <class _VecT>
+inline constexpr size_t __simd_vector_size_v = []<bool _False = false>() -> size_t {
+  static_assert(_False, "Not a vector!");
+}();
+
+template <class _Tp, size_t _Np>
+inline constexpr size_t __simd_vector_size_v<__simd_vector<_Tp, _Np>> = _Np;
+
+template <class _Tp, size_t _Np>
+_LIBCPP_HIDE_FROM_ABI _Tp __simd_vector_underlying_type_impl(__simd_vector<_Tp, _Np>) {
+  return _Tp{};
+}
+
+template <class _VecT>
+using __simd_vector_underlying_type_t = decltype(std::__simd_vector_underlying_type_impl(_VecT{}));
+
+// This isn't inlined without always_inline when loading chars.
+template <class _VecT, class _Tp>
+_LIBCPP_NODISCARD _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _VecT __load_vector(const _Tp* __ptr) noexcept {
+  return [=]<size_t... _Indices>(index_sequence<_Indices...>) _LIBCPP_ALWAYS_INLINE noexcept {
+    return _VecT{__ptr[_Indices]...};
+  }(make_index_sequence<__simd_vector_size_v<_VecT>>{});
+}
+
+template <class _Tp, size_t _Np>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool __all_of(__simd_vector<_Tp, _Np> __vec) noexcept {
+  return __builtin_reduce_and(__builtin_convertvector(__vec, __simd_vector<bool, _Np>));
+}
+
+template <class _Tp, size_t _Np>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_set(__simd_vector<_Tp, _Np> __vec) noexcept {
+  using __mask_vec = __simd_vector<bool, _Np>;
+
+  // This has MSan disabled du to https://github.com/llvm/llvm-project/issues/85876
+  auto __impl = [&]<class _MaskT>(_MaskT) _LIBCPP_NO_SANITIZE("memory") noexcept {
+    return std::__countr_zero(__builtin_bit_cast(_MaskT, __builtin_convertvector(__vec, __mask_vec)));
+  };
+
+  if constexpr (sizeof(__mask_vec) == sizeof(uint8_t)) {
+    return __impl(uint8_t{});
+  } else if constexpr (sizeof(__mask_vec) == sizeof(uint16_t)) {
+    return __impl(uint16_t{});
+  } else if constexpr (sizeof(__mask_vec) == sizeof(uint32_t)) {
+    return __impl(uint32_t{});
+  } else if constexpr (sizeof(__mask_vec) == sizeof(uint64_t)) {
+    return __impl(uint64_t{});
+  } else {
+    static_assert(sizeof(__mask_vec) == 0, "unexpected required size for mask integer type");
+    return 0;
+  }
+}
+
+template <class _Tp, size_t _Np>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_not_set(__simd_vector<_Tp, _Np> __vec) noexcept {
+  return std::__find_first_set(~__vec);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS
+
+#endif // _LIBCPP___ALGORITHM_SIMD_UTILS_H
diff --git a/libcxx/include/__bit/bit_cast.h b/libcxx/include/__bit/bit_cast.h
index f20b39ae748b..6298810f3733 100644
--- a/libcxx/include/__bit/bit_cast.h
+++ b/libcxx/include/__bit/bit_cast.h
@@ -19,6 +19,15 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+#ifndef _LIBCPP_CXX03_LANG
+
+template <class _ToType, class _FromType>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI constexpr _ToType __bit_cast(const _FromType& __from) noexcept {
+  return __builtin_bit_cast(_ToType, __from);
+}
+
+#endif // _LIBCPP_CXX03_LANG
+
 #if _LIBCPP_STD_VER >= 20
 
 template <class _ToType, class _FromType>
diff --git a/libcxx/include/__bit/countr.h b/libcxx/include/__bit/countr.h
index 0cc679f87a99..b6b3ac52ca4e 100644
--- a/libcxx/include/__bit/countr.h
+++ b/libcxx/include/__bit/countr.h
@@ -35,10 +35,8 @@ _LIBCPP_NODISCARD inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __libcpp_ct
   return __builtin_ctzll(__x);
 }
 
-#if _LIBCPP_STD_VER >= 20
-
-template <__libcpp_unsigned_integer _Tp>
-_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_zero(_Tp __t) noexcept {
+template <class _Tp>
+_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 int __countr_zero(_Tp __t) _NOEXCEPT {
   if (__t == 0)
     return numeric_limits<_Tp>::digits;
 
@@ -59,6 +57,13 @@ _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_zero(_Tp __t) n
   }
 }
 
+#if _LIBCPP_STD_VER >= 20
+
+template <__libcpp_unsigned_integer _Tp>
+_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_zero(_Tp __t) noexcept {
+  return std::__countr_zero(__t);
+}
+
 template <__libcpp_unsigned_integer _Tp>
 _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr int countr_one(_Tp __t) noexcept {
   return __t != numeric_limits<_Tp>::max() ? std::countr_zero(static_cast<_Tp>(~__t)) : numeric_limits<_Tp>::digits;
diff --git a/libcxx/include/libcxx.imp b/libcxx/include/libcxx.imp
index 77b7befd44f5..56ea58262828 100644
--- a/libcxx/include/libcxx.imp
+++ b/libcxx/include/libcxx.imp
@@ -217,6 +217,7 @@
   { include: [ "<__algorithm/shift_right.h>", "private", "<algorithm>", "public" ] },
   { include: [ "<__algorithm/shuffle.h>", "private", "<algorithm>", "public" ] },
   { include: [ "<__algorithm/sift_down.h>", "private", "<algorithm>", "public" ] },
+  { include: [ "<__algorithm/simd_utils.h>", "private", "<algorithm>", "public" ] },
   { include: [ "<__algorithm/sort.h>", "private", "<algorithm>", "public" ] },
   { include: [ "<__algorithm/sort_heap.h>", "private", "<algorithm>", "public" ] },
   { include: [ "<__algorithm/stable_partition.h>", "private", "<algorithm>", "public" ] },
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index f36a47cef009..03d18775631e 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -697,7 +697,10 @@ module std_private_algorithm_minmax                                      [system
   export *
 }
 module std_private_algorithm_minmax_element                              [system] { header "__algorithm/minmax_element.h" }
-module std_private_algorithm_mismatch                                    [system] { header "__algorithm/mismatch.h" }
+module std_private_algorithm_mismatch                                    [system] {
+  header "__algorithm/mismatch.h"
+  export std_private_algorithm_simd_utils
+}
 module std_private_algorithm_move                                        [system] { header "__algorithm/move.h" }
 module std_private_algorithm_move_backward                               [system] { header "__algorithm/move_backward.h" }
 module std_private_algorithm_next_permutation                            [system] { header "__algorithm/next_permutation.h" }
@@ -1048,6 +1051,7 @@ module std_private_algorithm_sort                                        [system
   header "__algorithm/sort.h"
   export std_private_debug_utils_strict_weak_ordering_check
 }
+module std_private_algorithm_simd_utils                                  [system] { header "__algorithm/simd_utils.h" }
 module std_private_algorithm_sort_heap                                   [system] { header "__algorithm/sort_heap.h" }
 module std_private_algorithm_stable_partition                            [system] { header "__algorithm/stable_partition.h" }
 module std_private_algorithm_stable_sort                                 [system] { header "__algorithm/stable_sort.h" }
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp
index cc588c095ccf..e7f3994d977d 100644
--- a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch.pass.cpp
@@ -16,79 +16,173 @@
 // template<InputIterator Iter1, InputIterator Iter2Pred>
 //   constexpr pair<Iter1, Iter2>   // constexpr after c++17
 //   mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2); // C++14
+//
+// template<InputIterator Iter1, InputIterator Iter2,
+//          Predicate<auto, Iter1::value_type, Iter2::value_type> Pred>
+//   requires CopyConstructible<Pred>
+//   constexpr pair<Iter1, Iter2>   // constexpr after c++17
+//   mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Pred pred);
+//
+// template<InputIterator Iter1, InputIterator Iter2, Predicate Pred>
+//   constexpr pair<Iter1, Iter2>   // constexpr after c++17
+//   mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, Pred pred); // C++14
+
+// ADDITIONAL_COMPILE_FLAGS(has-fconstexpr-steps): -fconstexpr-steps=50000000
+// ADDITIONAL_COMPILE_FLAGS(has-fconstexpr-ops-limit): -fconstexpr-ops-limit=100000000
 
 #include <algorithm>
+#include <array>
 #include <cassert>
+#include <vector>
 
 #include "test_macros.h"
 #include "test_iterators.h"
-
-#if TEST_STD_VER > 17
-TEST_CONSTEXPR bool test_constexpr() {
-    int ia[] = {1, 3, 6, 7};
-    int ib[] = {1, 3};
-    int ic[] = {1, 3, 5, 7};
-    typedef cpp17_input_iterator<int*>         II;
-    typedef bidirectional_iterator<int*> BI;
-
-    auto p1 = std::mismatch(std::begin(ia), std::end(ia), std::begin(ic));
-    if (p1.first != ia+2 || p1.second != ic+2)
-        return false;
-
-    auto p2 = std::mismatch(std::begin(ia), std::end(ia), std::begin(ic), std::end(ic));
-    if (p2.first != ia+2 || p2.second != ic+2)
-        return false;
-
-    auto p3 = std::mismatch(std::begin(ib), std::end(ib), std::begin(ic));
-    if (p3.first != ib+2 || p3.second != ic+2)
-        return false;
-
-    auto p4 = std::mismatch(std::begin(ib), std::end(ib), std::begin(ic), std::end(ic));
-    if (p4.first != ib+2 || p4.second != ic+2)
-        return false;
-
-    auto p5 = std::mismatch(II(std::begin(ib)), II(std::end(ib)), II(std::begin(ic)));
-    if (p5.first != II(ib+2) || p5.second != II(ic+2))
-        return false;
-    auto p6 = std::mismatch(BI(std::begin(ib)), BI(std::end(ib)), BI(std::begin(ic)), BI(std::end(ic)));
-    if (p6.first != BI(ib+2) || p6.second != BI(ic+2))
-        return false;
-
-    return true;
-    }
+#include "type_algorithms.h"
+
+template <class Iter, class Container1, class Container2>
+TEST_CONSTEXPR_CXX20 void check(Container1 lhs, Container2 rhs, size_t offset) {
+  if (lhs.size() == rhs.size()) {
+    assert(std::mismatch(Iter(lhs.data()), Iter(lhs.data() + lhs.size()), Iter(rhs.data())) ==
+           std::make_pair(Iter(lhs.data() + offset), Iter(rhs.data() + offset)));
+
+    assert(std::mismatch(Iter(lhs.data()),
+                         Iter(lhs.data() + lhs.size()),
+                         Iter(rhs.data()),
+                         std::equal_to<typename Container1::value_type>()) ==
+           std::make_pair(Iter(lhs.data() + offset), Iter(rhs.data() + offset)));
+  }
+
+#if TEST_STD_VER >= 14
+  assert(
+      std::mismatch(Iter(lhs.data()), Iter(lhs.data() + lhs.size()), Iter(rhs.data()), Iter(rhs.data() + rhs.size())) ==
+      std::make_pair(Iter(lhs.data() + offset), Iter(rhs.data() + offset)));
+
+  assert(std::mismatch(Iter(lhs.data()),
+                       Iter(lhs.data() + lhs.size()),
+                       Iter(rhs.data()),
+                       Iter(rhs.data() + rhs.size()),
+                       std::equal_to<typename Container1::value_type>()) ==
+         std::make_pair(Iter(lhs.data() + offset), Iter(rhs.data() + offset)));
 #endif
+}
 
-int main(int, char**)
-{
-    int ia[] = {0, 1, 2, 2, 0, 1, 2, 3};
-    const unsigned sa = sizeof(ia)/sizeof(ia[0]);
-    int ib[] = {0, 1, 2, 3, 0, 1, 2, 3};
-    const unsigned sb = sizeof(ib)/sizeof(ib[0]); ((void)sb); // unused in C++11
-
-    typedef cpp17_input_iterator<const int*> II;
-    typedef random_access_iterator<const int*>  RAI;
-
-    assert(std::mismatch(II(ia), II(ia + sa), II(ib))
-            == (std::pair<II, II>(II(ia+3), II(ib+3))));
-
-    assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib))
-            == (std::pair<RAI, RAI>(RAI(ia+3), RAI(ib+3))));
-
-#if TEST_STD_VER > 11 // We have the four iteration version
-    assert(std::mismatch(II(ia), II(ia + sa), II(ib), II(ib+sb))
-            == (std::pair<II, II>(II(ia+3), II(ib+3))));
+struct NonTrivial {
+  int i_;
+
+  TEST_CONSTEXPR_CXX20 NonTrivial(int i) : i_(i) {}
+  TEST_CONSTEXPR_CXX20 NonTrivial(NonTrivial&& other) : i_(other.i_) { other.i_ = 0; }
+
+  TEST_CONSTEXPR_CXX20 friend bool operator==(const NonTrivial& lhs, const NonTrivial& rhs) { return lhs.i_ == rhs.i_; }
+};
+
+struct ModTwoComp {
+  TEST_CONSTEXPR_CXX20 bool operator()(int lhs, int rhs) { return lhs % 2 == rhs % 2; }
+};
+
+template <class Iter>
+TEST_CONSTEXPR_CXX20 bool test() {
+  { // empty ranges
+    std::array<int, 0> lhs = {};
+    std::array<int, 0> rhs = {};
+    check<Iter>(lhs, rhs, 0);
+  }
+
+  { // same range without mismatch
+    std::array<int, 8> lhs = {0, 1, 2, 3, 0, 1, 2, 3};
+    std::array<int, 8> rhs = {0, 1, 2, 3, 0, 1, 2, 3};
+    check<Iter>(lhs, rhs, 8);
+  }
+
+  { // same range with mismatch
+    std::array<int, 8> lhs = {0, 1, 2, 2, 0, 1, 2, 3};
+    std::array<int, 8> rhs = {0, 1, 2, 3, 0, 1, 2, 3};
+    check<Iter>(lhs, rhs, 3);
+  }
+
+  { // second range is smaller
+    std::array<int, 8> lhs = {0, 1, 2, 2, 0, 1, 2, 3};
+    std::array<int, 2> rhs = {0, 1};
+    check<Iter>(lhs, rhs, 2);
+  }
+
+  { // first range is smaller
+    std::array<int, 2> lhs = {0, 1};
+    std::array<int, 8> rhs = {0, 1, 2, 2, 0, 1, 2, 3};
+    check<Iter>(lhs, rhs, 2);
+  }
+
+  { // use a custom comparator
+    std::array<int, 4> lhs = {0, 2, 3, 4};
+    std::array<int, 4> rhs = {0, 0, 4, 4};
+    assert(std::mismatch(lhs.data(), lhs.data() + lhs.size(), rhs.data(), ModTwoComp()) ==
+           std::make_pair(lhs.data() + 2, rhs.data() + 2));
+#if TEST_STD_VER >= 14
+    assert(std::mismatch(lhs.data(), lhs.data() + lhs.size(), rhs.data(), rhs.data() + rhs.size(), ModTwoComp()) ==
+           std::make_pair(lhs.data() + 2, rhs.data() + 2));
+#endif
+  }
 
-    assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib), RAI(ib+sb))
-            == (std::pair<RAI, RAI>(RAI(ia+3), RAI(ib+3))));
+  return true;
+}
 
+struct Test {
+  template <class Iter>
+  TEST_CONSTEXPR_CXX20 void operator()() {
+    test<Iter>();
+  }
+};
+
+TEST_CONSTEXPR_CXX20 bool test() {
+  types::for_each(types::cpp17_input_iterator_list<int*>(), Test());
+
+  { // use a non-integer type to also test the general case - all elements match
+    std::array<NonTrivial, 8> lhs = {1, 2, 3, 4, 5, 6, 7, 8};
+    std::array<NonTrivial, 8> rhs = {1, 2, 3, 4, 5, 6, 7, 8};
+    check<NonTrivial*>(std::move(lhs), std::move(rhs), 8);
+  }
+
+  { // use a non-integer type to also test the general case - not all elements match
+    std::array<NonTrivial, 8> lhs = {1, 2, 3, 4, 7, 6, 7, 8};
+    std::array<NonTrivial, 8> rhs = {1, 2, 3, 4, 5, 6, 7, 8};
+    check<NonTrivial*>(std::move(lhs), std::move(rhs), 4);
+  }
+
+  return true;
+}
 
-    assert(std::mismatch(II(ia), II(ia + sa), II(ib), II(ib+2))
-            == (std::pair<II, II>(II(ia+2), II(ib+2))));
+int main(int, char**) {
+  test();
+#if TEST_STD_VER >= 20
+  static_assert(test());
 #endif
 
-#if TEST_STD_VER > 17
-    static_assert(test_constexpr());
-#endif
+  { // check with a lot of elements to test the vectorization optimization
+    {
+      std::vector<char> lhs(256);
+      std::vector<char> rhs(256);
+      for (size_t i = 0; i != lhs.size(); ++i) {
+        lhs[i] = 1;
+        check<char*>(lhs, rhs, i);
+        lhs[i] = 0;
+        rhs[i] = 1;
+        check<char*>(lhs, rhs, i);
+        rhs[i] = 0;
+      }
+    }
+
+    {
+      std::vector<int> lhs(256);
+      std::vector<int> rhs(256);
+      for (size_t i = 0; i != lhs.size(); ++i) {
+        lhs[i] = 1;
+        check<int*>(lhs, rhs, i);
+        lhs[i] = 0;
+        rhs[i] = 1;
+        check<int*>(lhs, rhs, i);
+        rhs[i] = 0;
+      }
+    }
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch_pred.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch_pred.pass.cpp
deleted file mode 100644
index bda4ec7ba5ed..000000000000
--- a/libcxx/test/std/algorithms/alg.nonmodifying/mismatch/mismatch_pred.pass.cpp
+++ /dev/null
@@ -1,119 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// <algorithm>
-
-// template<InputIterator Iter1, InputIterator Iter2,
-//          Predicate<auto, Iter1::value_type, Iter2::value_type> Pred>
-//   requires CopyConstructible<Pred>
-//   constexpr pair<Iter1, Iter2>   // constexpr after c++17
-//   mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Pred pred);
-//
-// template<InputIterator Iter1, InputIterator Iter2, Predicate Pred>
-//   constexpr pair<Iter1, Iter2>   // constexpr after c++17
-//   mismatch(Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, Pred pred); // C++14
-
-#include <algorithm>
-#include <functional>
-#include <cassert>
-
-#include "test_macros.h"
-#include "test_iterators.h"
-#include "counting_predicates.h"
-
-#if TEST_STD_VER > 17
-TEST_CONSTEXPR bool eq(int a, int b) { return a == b; }
-
-TEST_CONSTEXPR bool test_constexpr() {
-    int ia[] = {1, 3, 6, 7};
-    int ib[] = {1, 3};
-    int ic[] = {1, 3, 5, 7};
-    typedef cpp17_input_iterator<int*>         II;
-    typedef bidirectional_iterator<int*> BI;
-
-    auto p1 = std::mismatch(std::begin(ia), std::end(ia), std::begin(ic), eq);
-    if (p1.first != ia+2 || p1.second != ic+2)
-        return false;
-
-    auto p2 = std::mismatch(std::begin(ia), std::end(ia), std::begin(ic), std::end(ic), eq);
-    if (p2.first != ia+2 || p2.second != ic+2)
-        return false;
-
-    auto p3 = std::mismatch(std::begin(ib), std::end(ib), std::begin(ic), eq);
-    if (p3.first != ib+2 || p3.second != ic+2)
-        return false;
-
-    auto p4 = std::mismatch(std::begin(ib), std::end(ib), std::begin(ic), std::end(ic), eq);
-    if (p4.first != ib+2 || p4.second != ic+2)
-        return false;
-
-    auto p5 = std::mismatch(II(std::begin(ib)), II(std::end(ib)), II(std::begin(ic)), eq);
-    if (p5.first != II(ib+2) || p5.second != II(ic+2))
-        return false;
-    auto p6 = std::mismatch(BI(std::begin(ib)), BI(std::end(ib)), BI(std::begin(ic)), BI(std::end(ic)), eq);
-    if (p6.first != BI(ib+2) || p6.second != BI(ic+2))
-        return false;
-
-    return true;
-    }
-#endif
-
-
-#if TEST_STD_VER > 11
-#define HAS_FOUR_ITERATOR_VERSION
-#endif
-
-int main(int, char**)
-{
-    int ia[] = {0, 1, 2, 2, 0, 1, 2, 3};
-    const unsigned sa = sizeof(ia)/sizeof(ia[0]);
-    int ib[] = {0, 1, 2, 3, 0, 1, 2, 3};
-    const unsigned sb = sizeof(ib)/sizeof(ib[0]); ((void)sb); // unused in C++11
-
-    typedef cpp17_input_iterator<const int*> II;
-    typedef random_access_iterator<const int*>  RAI;
-    typedef std::equal_to<int> EQ;
-
-    assert(std::mismatch(II(ia), II(ia + sa), II(ib), EQ())
-            == (std::pair<II, II>(II(ia+3), II(ib+3))));
-    assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib), EQ())
-            == (std::pair<RAI, RAI>(RAI(ia+3), RAI(ib+3))));
-
-    binary_counting_predicate<EQ, int> bcp((EQ()));
-    assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib), std::ref(bcp))
-            == (std::pair<RAI, RAI>(RAI(ia+3), RAI(ib+3))));
-    assert(bcp.count() > 0 && bcp.count() < sa);
-    bcp.reset();
-
-#if TEST_STD_VER >= 14
-    assert(std::mismatch(II(ia), II(ia + sa), II(ib), II(ib + sb), EQ())
-            == (std::pair<II, II>(II(ia+3), II(ib+3))));
-    assert(std::mismatch(RAI(ia), RAI(ia + sa), RAI(ib), RAI(ib + sb), EQ())
-            == (std::pair<RAI, RAI>(RAI(ia+3), RAI(ib+3))));
-
-    assert(std::mismatch(II(ia), II(ia + sa), II(ib), II(ib + sb), std::ref(bcp))
-            == (std::pair<II, II>(II(ia+3), II(ib+3))));
-    assert(bcp.count() > 0 && bcp.count() < std::min(sa, sb));
-#endif
-
-    assert(std::mismatch(ia, ia + sa, ib, EQ()) ==
-           (std::pair<int*,int*>(ia+3,ib+3)));
-
-#if TEST_STD_VER >= 14
-    assert(std::mismatch(ia, ia + sa, ib, ib + sb, EQ()) ==
-           (std::pair<int*,int*>(ia+3,ib+3)));
-    assert(std::mismatch(ia, ia + sa, ib, ib + 2, EQ()) ==
-           (std::pair<int*,int*>(ia+2,ib+2)));
-#endif
-
-#if TEST_STD_VER > 17
-    static_assert(test_constexpr());
-#endif
-
-  return 0;
-}
-- 
GitLab


From 3f5e649ff64a92a732027ef76e33a1c8b1722d84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= <mgorny@gentoo.org>
Date: Sat, 23 Mar 2024 16:46:32 +0100
Subject: [PATCH 050/404] [Libomptarget] Fix linking to LLVM dylib (#86397)

Use `LINK_COMPONENTS` parameter of `add_llvm_library` rather than
passing LLVM components directly to `target_link_libraries`, in order to
ensure that LLVM dylib is linked correctly when used. Otherwise, CMake
insists on linking to static libraries that aren't present on
distributions doing pure dylib installs, such as Gentoo.

This fixes a regression introduced
in dcbddc25250158469c5635ad2ae4095faef53dfd.
---
 .../plugins-nextgen/CMakeLists.txt            | 55 +++++++++----------
 1 file changed, 27 insertions(+), 28 deletions(-)

diff --git a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt
index c19fd80592d6..dbd82ac94517 100644
--- a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt
+++ b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt
@@ -14,41 +14,40 @@
 set(common_dir ${CMAKE_CURRENT_SOURCE_DIR}/common)
 add_subdirectory(common)
 function(add_target_library target_name lib_name)
-  llvm_map_components_to_libnames(llvm_libs
-    ${LLVM_TARGETS_TO_BUILD}
-    AggressiveInstCombine
-    Analysis
-    BinaryFormat
-    BitReader
-    BitWriter
-    CodeGen
-    Core
-    Extensions
-    InstCombine
-    Instrumentation
-    IPO
-    IRReader
-    Linker
-    MC
-    Object
-    Passes
-    Remarks
-    ScalarOpts
-    Support
-    Target
-    TargetParser
-    TransformUtils
-    Vectorize
-  )
-
   add_llvm_library(${target_name} SHARED
+    LINK_COMPONENTS
+      ${LLVM_TARGETS_TO_BUILD}
+      AggressiveInstCombine
+      Analysis
+      BinaryFormat
+      BitReader
+      BitWriter
+      CodeGen
+      Core
+      Extensions
+      InstCombine
+      Instrumentation
+      IPO
+      IRReader
+      Linker
+      MC
+      Object
+      Passes
+      Remarks
+      ScalarOpts
+      Support
+      Target
+      TargetParser
+      TransformUtils
+      Vectorize
+
     NO_INSTALL_RPATH
     BUILDTREE_ONLY
   )
 
   llvm_update_compile_flags(${target_name})
   target_link_libraries(${target_name} PRIVATE
-                        PluginCommon ${llvm_libs} ${OPENMP_PTHREAD_LIB})
+                        PluginCommon ${OPENMP_PTHREAD_LIB})
 
   target_compile_definitions(${target_name} PRIVATE TARGET_NAME=${lib_name})
   target_compile_definitions(${target_name} PRIVATE 
-- 
GitLab


From d7ce6b4d96c8879f38ba4cb5fdb1cc09d5b129e5 Mon Sep 17 00:00:00 2001
From: Mike Rice <michael.p.rice@intel.com>
Date: Sat, 23 Mar 2024 09:53:29 -0700
Subject: [PATCH 051/404] [clang-tidy] Fix result check after
 overwriteChangedFiles() (#86360)

If any return from overwriteChangedFiles is true some fixes were not
applied.
---
 clang-tools-extra/clang-tidy/ClangTidy.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang-tools-extra/clang-tidy/ClangTidy.cpp b/clang-tools-extra/clang-tidy/ClangTidy.cpp
index 40ac6918faf4..b877ea06dc05 100644
--- a/clang-tools-extra/clang-tidy/ClangTidy.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidy.cpp
@@ -233,7 +233,7 @@ public:
         if (!tooling::applyAllReplacements(Replacements.get(), Rewrite)) {
           llvm::errs() << "Can't apply replacements for file " << File << "\n";
         }
-        AnyNotWritten &= Rewrite.overwriteChangedFiles();
+        AnyNotWritten |= Rewrite.overwriteChangedFiles();
       }
 
       if (AnyNotWritten) {
-- 
GitLab


From 87c7f4a12b2a1b723a78d760761ee473f52c4cee Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sat, 23 Mar 2024 10:03:09 -0700
Subject: [PATCH 052/404] [MC] Remove unnecessary reversal of relocations. NFC

Commit f44db24e1fd948c75c87aea017646f16553d3361 (2015) enabled this
simplication.
---
 llvm/lib/MC/ELFObjectWriter.cpp                          | 9 +--------
 .../lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp | 5 ++---
 2 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp
index 3c4d3ab9a508..a6fb6b5c1a4e 100644
--- a/llvm/lib/MC/ELFObjectWriter.cpp
+++ b/llvm/lib/MC/ELFObjectWriter.cpp
@@ -939,18 +939,11 @@ void ELFWriter::writeRelocations(const MCAssembler &Asm,
                                        const MCSectionELF &Sec) {
   std::vector<ELFRelocationEntry> &Relocs = OWriter.Relocations[&Sec];
 
-  // We record relocations by pushing to the end of a vector. Reverse the vector
-  // to get the relocations in the order they were created.
-  // In most cases that is not important, but it can be for special sections
-  // (.eh_frame) or specific relocations (TLS optimizations on SystemZ).
-  std::reverse(Relocs.begin(), Relocs.end());
-
   // Sort the relocation entries. MIPS needs this.
   OWriter.TargetObjectWriter->sortRelocs(Asm, Relocs);
 
   const bool Rela = usesRela(Sec);
-  for (unsigned i = 0, e = Relocs.size(); i != e; ++i) {
-    const ELFRelocationEntry &Entry = Relocs[e - i - 1];
+  for (const ELFRelocationEntry &Entry : Relocs) {
     unsigned Index = Entry.Symbol ? Entry.Symbol->getIndex() : 0;
 
     if (is64Bit()) {
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
index 181b82f14bfe..4d6a00c14a35 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
@@ -498,10 +498,9 @@ void MipsELFObjectWriter::sortRelocs(const MCAssembler &Asm,
 
   assert(Relocs.size() == Sorted.size() && "Some relocs were not consumed");
 
-  // Overwrite the original vector with the sorted elements. The caller expects
-  // them in reverse order.
+  // Overwrite the original vector with the sorted elements.
   unsigned CopyTo = 0;
-  for (const auto &R : reverse(Sorted))
+  for (const auto &R : Sorted)
     Relocs[CopyTo++] = R.R;
 }
 
-- 
GitLab


From 3a63f737e29a2382c8ec26c0d360bd77c01a27a1 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sat, 23 Mar 2024 10:15:47 -0700
Subject: [PATCH 053/404] [MC] Refactor writeRelocations. NFC

MIPS is different and should better off use separate code.
---
 llvm/lib/MC/ELFObjectWriter.cpp | 60 ++++++++++++++++++---------------
 1 file changed, 33 insertions(+), 27 deletions(-)

diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp
index a6fb6b5c1a4e..f4c6cbc8dd44 100644
--- a/llvm/lib/MC/ELFObjectWriter.cpp
+++ b/llvm/lib/MC/ELFObjectWriter.cpp
@@ -943,40 +943,28 @@ void ELFWriter::writeRelocations(const MCAssembler &Asm,
   OWriter.TargetObjectWriter->sortRelocs(Asm, Relocs);
 
   const bool Rela = usesRela(Sec);
-  for (const ELFRelocationEntry &Entry : Relocs) {
-    unsigned Index = Entry.Symbol ? Entry.Symbol->getIndex() : 0;
-
-    if (is64Bit()) {
-      write(Entry.Offset);
-      if (OWriter.TargetObjectWriter->getEMachine() == ELF::EM_MIPS) {
-        write(uint32_t(Index));
-
+  if (OWriter.TargetObjectWriter->getEMachine() == ELF::EM_MIPS) {
+    for (const ELFRelocationEntry &Entry : Relocs) {
+      uint32_t Symidx = Entry.Symbol ? Entry.Symbol->getIndex() : 0;
+      if (is64Bit()) {
+        write(Entry.Offset);
+        write(uint32_t(Symidx));
         write(OWriter.TargetObjectWriter->getRSsym(Entry.Type));
         write(OWriter.TargetObjectWriter->getRType3(Entry.Type));
         write(OWriter.TargetObjectWriter->getRType2(Entry.Type));
         write(OWriter.TargetObjectWriter->getRType(Entry.Type));
+        if (Rela)
+          write(Entry.Addend);
       } else {
-        struct ELF::Elf64_Rela ERE64;
-        ERE64.setSymbolAndType(Index, Entry.Type);
-        write(ERE64.r_info);
-      }
-      if (Rela)
-        write(Entry.Addend);
-    } else {
-      write(uint32_t(Entry.Offset));
-
-      struct ELF::Elf32_Rela ERE32;
-      ERE32.setSymbolAndType(Index, Entry.Type);
-      write(ERE32.r_info);
-
-      if (Rela)
-        write(uint32_t(Entry.Addend));
-
-      if (OWriter.TargetObjectWriter->getEMachine() == ELF::EM_MIPS) {
+        write(uint32_t(Entry.Offset));
+        ELF::Elf32_Rela ERE32;
+        ERE32.setSymbolAndType(Symidx, Entry.Type);
+        write(ERE32.r_info);
+        if (Rela)
+          write(uint32_t(Entry.Addend));
         if (uint32_t RType =
                 OWriter.TargetObjectWriter->getRType2(Entry.Type)) {
           write(uint32_t(Entry.Offset));
-
           ERE32.setSymbolAndType(0, RType);
           write(ERE32.r_info);
           write(uint32_t(0));
@@ -984,13 +972,31 @@ void ELFWriter::writeRelocations(const MCAssembler &Asm,
         if (uint32_t RType =
                 OWriter.TargetObjectWriter->getRType3(Entry.Type)) {
           write(uint32_t(Entry.Offset));
-
           ERE32.setSymbolAndType(0, RType);
           write(ERE32.r_info);
           write(uint32_t(0));
         }
       }
     }
+    return;
+  }
+  for (const ELFRelocationEntry &Entry : Relocs) {
+    uint32_t Symidx = Entry.Symbol ? Entry.Symbol->getIndex() : 0;
+    if (is64Bit()) {
+      write(Entry.Offset);
+      ELF::Elf64_Rela ERE;
+      ERE.setSymbolAndType(Symidx, Entry.Type);
+      write(ERE.r_info);
+      if (Rela)
+        write(Entry.Addend);
+    } else {
+      write(uint32_t(Entry.Offset));
+      ELF::Elf32_Rela ERE;
+      ERE.setSymbolAndType(Symidx, Entry.Type);
+      write(ERE.r_info);
+      if (Rela)
+        write(uint32_t(Entry.Addend));
+    }
   }
 }
 
-- 
GitLab


From 39c8e87717fbc611b9e84f62edf656608ae52e5c Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sat, 23 Mar 2024 18:43:14 +0100
Subject: [PATCH 054/404] [VPlan] Move recording of Inst->VPValue to
 VPRecipeBuilder (NFCI). (#84464)

Instead of keeping a mapping of Inst->VPValues (of their corresponding
recipes) in VPlan's Value2VPValue mapping, keep it in VPRecipeBuilder
instead. After recently replacing the last user of this mapping after
initial construction, this mapping is only needed for recipe
construction (to map IR operands to VPValue operands).

By moving the mapping, VPlan's VPValue tracking can be simplified and
limited only to live-ins. It also allows removing disableValue2VPValue
and associated machinery & asserts.

PR: https://github.com/llvm/llvm-project/pull/84464
---
 .../Transforms/Vectorize/LoopVectorize.cpp    | 47 +++++++------------
 .../Transforms/Vectorize/VPRecipeBuilder.h    | 35 +++++++-------
 llvm/lib/Transforms/Vectorize/VPlan.h         | 25 ++--------
 .../Transforms/Vectorize/VPlanTransforms.cpp  | 15 +++---
 4 files changed, 46 insertions(+), 76 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2163930b02c1..28748142ae92 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7898,6 +7898,18 @@ void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
   }
 }
 
+iterator_range<mapped_iterator<Use *, std::function<VPValue *(Value *)>>>
+VPRecipeBuilder::mapToVPValues(User::op_range Operands) {
+  std::function<VPValue *(Value *)> Fn = [this](Value *Op) {
+    if (auto *I = dyn_cast<Instruction>(Op)) {
+      if (auto *R = Ingredient2Recipe.lookup(I))
+        return R->getVPSingleValue();
+    }
+    return Plan.getVPValueOrAddLiveIn(Op);
+  };
+  return map_range(Operands, Fn);
+}
+
 VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst) {
   assert(is_contained(predecessors(Dst), Src) && "Invalid edge");
 
@@ -7922,7 +7934,7 @@ VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst) {
   if (OrigLoop->isLoopExiting(Src))
     return EdgeMaskCache[Edge] = SrcMask;
 
-  VPValue *EdgeMask = Plan.getVPValueOrAddLiveIn(BI->getCondition());
+  VPValue *EdgeMask = getVPValueOrAddLiveIn(BI->getCondition(), Plan);
   assert(EdgeMask && "No Edge Mask found for condition");
 
   if (BI->getSuccessor(0) != Dst)
@@ -8383,7 +8395,7 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(Instruction *I,
     BlockInMask = getBlockInMask(I->getParent());
   }
 
-  auto *Recipe = new VPReplicateRecipe(I, Plan.mapToVPValues(I->operands()),
+  auto *Recipe = new VPReplicateRecipe(I, mapToVPValues(I->operands()),
                                        IsUniform, BlockInMask);
   return Recipe;
 }
@@ -8399,10 +8411,6 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
     if (Phi->getParent() != OrigLoop->getHeader())
       return tryToBlend(Phi, Operands);
 
-    // Always record recipes for header phis. Later first-order recurrence phis
-    // can have earlier phis as incoming values.
-    recordRecipeOf(Phi);
-
     if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands, Range)))
       return Recipe;
 
@@ -8427,14 +8435,6 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
       PhiRecipe = new VPFirstOrderRecurrencePHIRecipe(Phi, *StartV);
     }
 
-    // Record the incoming value from the backedge, so we can add the incoming
-    // value from the backedge after all recipes have been created.
-    auto *Inc = cast<Instruction>(
-        Phi->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
-    auto RecipeIter = Ingredient2Recipe.find(Inc);
-    if (RecipeIter == Ingredient2Recipe.end())
-      recordRecipeOf(Inc);
-
     PhisToFix.push_back(PhiRecipe);
     return PhiRecipe;
   }
@@ -8522,7 +8522,7 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW,
 // Add exit values to \p Plan. VPLiveOuts are added for each LCSSA phi in the
 // original exit block.
 static void addUsersInExitBlock(VPBasicBlock *HeaderVPBB, Loop *OrigLoop,
-                                VPlan &Plan) {
+                                VPRecipeBuilder &Builder, VPlan &Plan) {
   BasicBlock *ExitBB = OrigLoop->getUniqueExitBlock();
   BasicBlock *ExitingBB = OrigLoop->getExitingBlock();
   // Only handle single-exit loops with unique exit blocks for now.
@@ -8533,7 +8533,7 @@ static void addUsersInExitBlock(VPBasicBlock *HeaderVPBB, Loop *OrigLoop,
   for (PHINode &ExitPhi : ExitBB->phis()) {
     Value *IncomingValue =
         ExitPhi.getIncomingValueForBlock(ExitingBB);
-    VPValue *V = Plan.getVPValueOrAddLiveIn(IncomingValue);
+    VPValue *V = Builder.getVPValueOrAddLiveIn(IncomingValue, Plan);
     Plan.addLiveOut(&ExitPhi, V);
   }
 }
@@ -8603,9 +8603,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
     if (!getDecisionAndClampRange(applyIG, Range))
       continue;
     InterleaveGroups.insert(IG);
-    for (unsigned i = 0; i < IG->getFactor(); i++)
-      if (Instruction *Member = IG->getMember(i))
-        RecipeBuilder.recordRecipeOf(Member);
   };
 
   // ---------------------------------------------------------------------------
@@ -8647,7 +8644,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
         Operands.push_back(Plan->getVPValueOrAddLiveIn(
             Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader())));
       } else {
-        auto OpRange = Plan->mapToVPValues(Instr->operands());
+        auto OpRange = RecipeBuilder.mapToVPValues(Instr->operands());
         Operands = {OpRange.begin(), OpRange.end()};
       }
 
@@ -8662,10 +8659,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
           RecipeBuilder.tryToCreateWidenRecipe(Instr, Operands, Range, VPBB);
       if (!Recipe)
         Recipe = RecipeBuilder.handleReplication(Instr, Range);
-      for (auto *Def : Recipe->definedValues()) {
-        auto *UV = Def->getUnderlyingValue();
-        Plan->addVPValue(UV, Def);
-      }
 
       RecipeBuilder.setRecipe(Instr, Recipe);
       if (isa<VPHeaderPHIRecipe>(Recipe)) {
@@ -8697,7 +8690,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
     // and there is nothing to fix from vector loop; phis should have incoming
     // from scalar loop only.
   } else
-    addUsersInExitBlock(HeaderVPBB, OrigLoop, *Plan);
+    addUsersInExitBlock(HeaderVPBB, OrigLoop, RecipeBuilder, *Plan);
 
   assert(isa<VPRegionBlock>(Plan->getVectorLoopRegion()) &&
          !Plan->getVectorLoopRegion()->getEntryBasicBlock()->empty() &&
@@ -8765,10 +8758,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
     Plan->getVPValueOrAddLiveIn(StrideV)->replaceAllUsesWith(ConstVPV);
   }
 
-  // From this point onwards, VPlan-to-VPlan transformations may change the plan
-  // in ways that accessing values using original IR values is incorrect.
-  Plan->disableValue2VPValue();
-
   VPlanTransforms::dropPoisonGeneratingRecipes(*Plan, [this](BasicBlock *BB) {
     return Legal->blockNeedsPredication(BB);
   });
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 29a395c35731..fbb70f333685 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -52,9 +52,8 @@ class VPRecipeBuilder {
   EdgeMaskCacheTy EdgeMaskCache;
   BlockMaskCacheTy BlockMaskCache;
 
-  // VPlan-VPlan transformations support: Hold a mapping from ingredients to
-  // their recipe. To save on memory, only do so for selected ingredients,
-  // marked by having a nullptr entry in this map.
+  // VPlan construction support: Hold a mapping from ingredients to
+  // their recipe.
   DenseMap<Instruction *, VPRecipeBase *> Ingredient2Recipe;
 
   /// Cross-iteration reduction & first-order recurrence phis for which we need
@@ -117,13 +116,10 @@ public:
                                        ArrayRef<VPValue *> Operands,
                                        VFRange &Range, VPBasicBlock *VPBB);
 
-  /// Set the recipe created for given ingredient. This operation is a no-op for
-  /// ingredients that were not marked using a nullptr entry in the map.
+  /// Set the recipe created for given ingredient.
   void setRecipe(Instruction *I, VPRecipeBase *R) {
-    if (!Ingredient2Recipe.count(I))
-      return;
-    assert(Ingredient2Recipe[I] == nullptr &&
-           "Recipe already set for ingredient");
+    assert(!Ingredient2Recipe.contains(I) &&
+           "Cannot reset recipe for instruction.");
     Ingredient2Recipe[I] = R;
   }
 
@@ -146,14 +142,6 @@ public:
   /// between SRC and DST.
   VPValue *getEdgeMask(BasicBlock *Src, BasicBlock *Dst) const;
 
-  /// Mark given ingredient for recording its recipe once one is created for
-  /// it.
-  void recordRecipeOf(Instruction *I) {
-    assert((!Ingredient2Recipe.count(I) || Ingredient2Recipe[I] == nullptr) &&
-           "Recipe already set for ingredient");
-    Ingredient2Recipe[I] = nullptr;
-  }
-
   /// Return the recipe created for given ingredient.
   VPRecipeBase *getRecipe(Instruction *I) {
     assert(Ingredient2Recipe.count(I) &&
@@ -171,6 +159,19 @@ public:
   /// Add the incoming values from the backedge to reduction & first-order
   /// recurrence cross-iteration phis.
   void fixHeaderPhis();
+
+  /// Returns a range mapping the values of the range \p Operands to their
+  /// corresponding VPValues.
+  iterator_range<mapped_iterator<Use *, std::function<VPValue *(Value *)>>>
+  mapToVPValues(User::op_range Operands);
+
+  VPValue *getVPValueOrAddLiveIn(Value *V, VPlan &Plan) {
+    if (auto *I = dyn_cast<Instruction>(V)) {
+      if (auto *R = Ingredient2Recipe.lookup(I))
+        return R->getVPSingleValue();
+    }
+    return Plan.getVPValueOrAddLiveIn(V);
+  }
 };
 } // end namespace llvm
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d77c7554d50e..22173954f7ce 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2872,10 +2872,6 @@ class VPlan {
   /// definitions are VPValues that hold a pointer to their underlying IR.
   SmallVector<VPValue *, 16> VPLiveInsToFree;
 
-  /// Indicates whether it is safe use the Value2VPValue mapping or if the
-  /// mapping cannot be used any longer, because it is stale.
-  bool Value2VPValueEnabled = true;
-
   /// Values used outside the plan.
   MapVector<PHINode *, VPLiveOut *> LiveOuts;
 
@@ -2954,10 +2950,6 @@ public:
   /// Returns VF * UF of the vector loop region.
   VPValue &getVFxUF() { return VFxUF; }
 
-  /// Mark the plan to indicate that using Value2VPValue is not safe any
-  /// longer, because it may be stale.
-  void disableValue2VPValue() { Value2VPValueEnabled = false; }
-
   void addVF(ElementCount VF) { VFs.insert(VF); }
 
   void setVF(ElementCount VF) {
@@ -2987,8 +2979,7 @@ public:
   void setName(const Twine &newName) { Name = newName.str(); }
 
   void addVPValue(Value *V, VPValue *VPV) {
-    assert((Value2VPValueEnabled || VPV->isLiveIn()) &&
-           "Value2VPValue mapping may be out of date!");
+    assert(VPV->isLiveIn() && "VPV must be a live-in.");
     assert(V && "Trying to add a null Value to VPlan");
     assert(!Value2VPValue.count(V) && "Value already exists in VPlan");
     Value2VPValue[V] = VPV;
@@ -2998,8 +2989,8 @@ public:
   VPValue *getVPValue(Value *V) {
     assert(V && "Trying to get the VPValue of a null Value");
     assert(Value2VPValue.count(V) && "Value does not exist in VPlan");
-    assert((Value2VPValueEnabled || Value2VPValue[V]->isLiveIn()) &&
-           "Value2VPValue mapping may be out of date!");
+    assert(Value2VPValue[V]->isLiveIn() &&
+           "Only live-ins should be in mapping");
     return Value2VPValue[V];
   }
 
@@ -3030,16 +3021,6 @@ public:
   LLVM_DUMP_METHOD void dump() const;
 #endif
 
-  /// Returns a range mapping the values the range \p Operands to their
-  /// corresponding VPValues.
-  iterator_range<mapped_iterator<Use *, std::function<VPValue *(Value *)>>>
-  mapToVPValues(User::op_range Operands) {
-    std::function<VPValue *(Value *)> Fn = [this](Value *Op) {
-      return getVPValueOrAddLiveIn(Op);
-    };
-    return map_range(Operands, Fn);
-  }
-
   /// Returns the VPRegionBlock of the vector loop.
   VPRegionBlock *getVectorLoopRegion() {
     return cast<VPRegionBlock>(getEntry()->getSingleSuccessor());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a91ccefe4b6d..00cec1148528 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -48,15 +48,14 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
       VPRecipeBase *NewRecipe = nullptr;
       if (auto *VPPhi = dyn_cast<VPWidenPHIRecipe>(&Ingredient)) {
         auto *Phi = cast<PHINode>(VPPhi->getUnderlyingValue());
-        if (const auto *II = GetIntOrFpInductionDescriptor(Phi)) {
-          VPValue *Start = Plan->getVPValueOrAddLiveIn(II->getStartValue());
-          VPValue *Step =
-              vputils::getOrCreateVPValueForSCEVExpr(*Plan, II->getStep(), SE);
-          NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, *II);
-        } else {
-          Plan->addVPValue(Phi, VPPhi);
+        const auto *II = GetIntOrFpInductionDescriptor(Phi);
+        if (!II)
           continue;
-        }
+
+        VPValue *Start = Plan->getVPValueOrAddLiveIn(II->getStartValue());
+        VPValue *Step =
+            vputils::getOrCreateVPValueForSCEVExpr(*Plan, II->getStep(), SE);
+        NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, *II);
       } else {
         assert(isa<VPInstruction>(&Ingredient) &&
                "only VPInstructions expected here");
-- 
GitLab


From e14c6fa31a303312d7561aa0c7219d687f3f2c75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= <mgorny@gentoo.org>
Date: Sat, 23 Mar 2024 20:26:20 +0100
Subject: [PATCH 055/404] [clang] [cmake] Add cmake module dir before using
 GetDarwinLinkerVersion (#86386)

Move the code adding top-level cmake/Modules directory to
CMAKE_MODULE_PATH prior to including `GetDarwinLinkerVersion`, in order
to fix standalone builds.

Fixes a regression introduced by
3bc71c2abfa00413fd15cf0e5c08af6ec0d4768b.
---
 clang/CMakeLists.txt | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index ee783d52e4a4..284b2af24dda 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -13,6 +13,13 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
   set(CLANG_BUILT_STANDALONE TRUE)
 endif()
 
+# Make sure that our source directory is on the current cmake module path so that
+# we can include cmake files from this directory.
+list(INSERT CMAKE_MODULE_PATH 0
+  "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules"
+  "${LLVM_COMMON_CMAKE_UTILS}/Modules"
+  )
+
 # Must go below project(..)
 include(GNUInstallDirs)
 include(GetDarwinLinkerVersion)
@@ -141,13 +148,6 @@ if(CLANG_BUILT_STANDALONE)
   endif() # LLVM_INCLUDE_TESTS
 endif() # standalone
 
-# Make sure that our source directory is on the current cmake module path so that
-# we can include cmake files from this directory.
-list(INSERT CMAKE_MODULE_PATH 0
-  "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules"
-  "${LLVM_COMMON_CMAKE_UTILS}/Modules"
-  )
-
 # This allows disabling clang's XML dependency even if LLVM finds libxml2.
 # By default, clang depends on libxml2 if LLVM does.
 option(CLANG_ENABLE_LIBXML2 "Whether libclang may depend on libxml2"
-- 
GitLab


From a91cd53de34fcde469d5d02e84554a06798ad2b1 Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Sat, 23 Mar 2024 16:08:31 -0700
Subject: [PATCH 056/404] [BOLT][NFC] Refactor BAT metadata data structures

Hide the implementations of `FuncHashes` and `BBHashMap` classes,
getting rid of `at` accessors that could throw an exception.

Test Plan: NFC

Reviewers: ayermolo, maksfb, dcci, rafaelauler

Reviewed By: rafaelauler

Pull Request: https://github.com/llvm/llvm-project/pull/86353
---
 .../bolt/Profile/BoltAddressTranslation.h     | 117 +++++++++++++++---
 bolt/lib/Profile/BoltAddressTranslation.cpp   |  85 ++++++-------
 2 files changed, 137 insertions(+), 65 deletions(-)

diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h
index d583ce0b76a2..f8c35f8066f7 100644
--- a/bolt/include/bolt/Profile/BoltAddressTranslation.h
+++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h
@@ -115,17 +115,6 @@ public:
   /// Save function and basic block hashes used for metadata dump.
   void saveMetadata(BinaryContext &BC);
 
-  /// Returns BB hash by function output address (after BOLT) and basic block
-  /// input offset.
-  size_t getBBHash(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const;
-
-  /// Returns BF hash by function output address (after BOLT).
-  size_t getBFHash(uint64_t OutputAddress) const;
-
-  /// Returns BB index by function output address (after BOLT) and basic block
-  /// input offset.
-  unsigned getBBIndex(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const;
-
   /// True if a given \p Address is a function with translation table entry.
   bool isBATFunction(uint64_t Address) const { return Maps.count(Address); }
 
@@ -135,7 +124,7 @@ private:
   /// emitted for the start of the BB. More entries may be emitted to cover
   /// the location of calls or any instruction that may change control flow.
   void writeEntriesForBB(MapTy &Map, const BinaryBasicBlock &BB,
-                         uint64_t FuncAddress);
+                         uint64_t FuncInputAddress, uint64_t FuncOutputAddress);
 
   /// Write the serialized address translation table for a function.
   template <bool Cold>
@@ -158,10 +147,6 @@ private:
 
   std::map<uint64_t, MapTy> Maps;
 
-  /// Map basic block input offset to a basic block index and hash pair.
-  using BBHashMap = std::unordered_map<uint32_t, std::pair<unsigned, size_t>>;
-  std::unordered_map<uint64_t, std::pair<size_t, BBHashMap>> FuncHashes;
-
   /// Map a function to its basic blocks count
   std::unordered_map<uint64_t, size_t> NumBasicBlocksMap;
 
@@ -174,6 +159,106 @@ private:
   /// Identifies the address of a control-flow changing instructions in a
   /// translation map entry
   const static uint32_t BRANCHENTRY = 0x1;
+
+public:
+  /// Map basic block input offset to a basic block index and hash pair.
+  class BBHashMapTy {
+    class EntryTy {
+      unsigned Index;
+      size_t Hash;
+
+    public:
+      unsigned getBBIndex() const { return Index; }
+      size_t getBBHash() const { return Hash; }
+      EntryTy(unsigned Index, size_t Hash) : Index(Index), Hash(Hash) {}
+    };
+
+    std::unordered_map<uint32_t, EntryTy> Map;
+    const EntryTy &getEntry(uint32_t BBInputOffset) const {
+      auto It = Map.find(BBInputOffset);
+      assert(It != Map.end());
+      return It->second;
+    }
+
+  public:
+    bool isInputBlock(uint32_t InputOffset) const {
+      return Map.count(InputOffset);
+    }
+
+    unsigned getBBIndex(uint32_t BBInputOffset) const {
+      return getEntry(BBInputOffset).getBBIndex();
+    }
+
+    size_t getBBHash(uint32_t BBInputOffset) const {
+      return getEntry(BBInputOffset).getBBHash();
+    }
+
+    void addEntry(uint32_t BBInputOffset, unsigned BBIndex, size_t BBHash) {
+      Map.emplace(BBInputOffset, EntryTy(BBIndex, BBHash));
+    }
+
+    size_t getNumBasicBlocks() const { return Map.size(); }
+  };
+
+  /// Map function output address to its hash and basic blocks hash map.
+  class FuncHashesTy {
+    class EntryTy {
+      size_t Hash;
+      BBHashMapTy BBHashMap;
+
+    public:
+      size_t getBFHash() const { return Hash; }
+      const BBHashMapTy &getBBHashMap() const { return BBHashMap; }
+      EntryTy(size_t Hash) : Hash(Hash) {}
+    };
+
+    std::unordered_map<uint64_t, EntryTy> Map;
+    const EntryTy &getEntry(uint64_t FuncOutputAddress) const {
+      auto It = Map.find(FuncOutputAddress);
+      assert(It != Map.end());
+      return It->second;
+    }
+
+  public:
+    size_t getBFHash(uint64_t FuncOutputAddress) const {
+      return getEntry(FuncOutputAddress).getBFHash();
+    }
+
+    const BBHashMapTy &getBBHashMap(uint64_t FuncOutputAddress) const {
+      return getEntry(FuncOutputAddress).getBBHashMap();
+    }
+
+    void addEntry(uint64_t FuncOutputAddress, size_t BFHash) {
+      Map.emplace(FuncOutputAddress, EntryTy(BFHash));
+    }
+
+    size_t getNumFunctions() const { return Map.size(); };
+
+    size_t getNumBasicBlocks() const {
+      size_t NumBasicBlocks{0};
+      for (auto &I : Map)
+        NumBasicBlocks += I.second.getBBHashMap().getNumBasicBlocks();
+      return NumBasicBlocks;
+    }
+  };
+
+  /// Returns BF hash by function output address (after BOLT).
+  size_t getBFHash(uint64_t FuncOutputAddress) const {
+    return FuncHashes.getBFHash(FuncOutputAddress);
+  }
+
+  /// Returns BBHashMap by function output address (after BOLT).
+  const BBHashMapTy &getBBHashMap(uint64_t FuncOutputAddress) const {
+    return FuncHashes.getBBHashMap(FuncOutputAddress);
+  }
+
+  BBHashMapTy &getBBHashMap(uint64_t FuncOutputAddress) {
+    return const_cast<BBHashMapTy &>(
+        std::as_const(*this).getBBHashMap(FuncOutputAddress));
+  }
+
+private:
+  FuncHashesTy FuncHashes;
 };
 } // namespace bolt
 
diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp
index 31886f4c8025..57d2aff71977 100644
--- a/bolt/lib/Profile/BoltAddressTranslation.cpp
+++ b/bolt/lib/Profile/BoltAddressTranslation.cpp
@@ -22,12 +22,10 @@ const char *BoltAddressTranslation::SECTION_NAME = ".note.bolt_bat";
 
 void BoltAddressTranslation::writeEntriesForBB(MapTy &Map,
                                                const BinaryBasicBlock &BB,
-                                               uint64_t FuncAddress) {
-  uint64_t HotFuncAddress = ColdPartSource.count(FuncAddress)
-                                ? ColdPartSource[FuncAddress]
-                                : FuncAddress;
+                                               uint64_t FuncInputAddress,
+                                               uint64_t FuncOutputAddress) {
   const uint64_t BBOutputOffset =
-      BB.getOutputAddressRange().first - FuncAddress;
+      BB.getOutputAddressRange().first - FuncOutputAddress;
   const uint32_t BBInputOffset = BB.getInputOffset();
 
   // Every output BB must track back to an input BB for profile collection
@@ -42,11 +40,14 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map,
   LLVM_DEBUG(dbgs() << "BB " << BB.getName() << "\n");
   LLVM_DEBUG(dbgs() << "  Key: " << Twine::utohexstr(BBOutputOffset)
                     << " Val: " << Twine::utohexstr(BBInputOffset) << "\n");
-  LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n",
-                               getBBHash(HotFuncAddress, BBInputOffset)));
-  (void)HotFuncAddress;
-  LLVM_DEBUG(dbgs() << formatv(" Index: {0}\n",
-                               getBBIndex(HotFuncAddress, BBInputOffset)));
+  // NB: in `writeEntriesForBB` we use the input address because hashes are
+  // saved early in `saveMetadata` before output addresses are assigned.
+  const BBHashMapTy &BBHashMap = getBBHashMap(FuncInputAddress);
+  (void)BBHashMap;
+  LLVM_DEBUG(
+      dbgs() << formatv(" Hash: {0:x}\n", BBHashMap.getBBHash(BBInputOffset)));
+  LLVM_DEBUG(
+      dbgs() << formatv(" Index: {0}\n", BBHashMap.getBBIndex(BBInputOffset)));
   // In case of conflicts (same Key mapping to different Vals), the last
   // update takes precedence. Of course it is not ideal to have conflicts and
   // those happen when we have an empty BB that either contained only
@@ -63,7 +64,7 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map,
     const auto InputAddress = BB.getFunction()->getAddress() + InputOffset;
     const auto OutputAddress = IOAddressMap.lookup(InputAddress);
     assert(OutputAddress && "Unknown instruction address");
-    const auto OutputOffset = *OutputAddress - FuncAddress;
+    const auto OutputOffset = *OutputAddress - FuncOutputAddress;
 
     // Is this the first instruction in the BB? No need to duplicate the entry.
     if (OutputOffset == BBOutputOffset)
@@ -99,7 +100,7 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) {
     MapTy Map;
     for (const BinaryBasicBlock *const BB :
          Function.getLayout().getMainFragment())
-      writeEntriesForBB(Map, *BB, Function.getOutputAddress());
+      writeEntriesForBB(Map, *BB, InputAddress, OutputAddress);
     Maps.emplace(Function.getOutputAddress(), std::move(Map));
     ReverseMap.emplace(OutputAddress, InputAddress);
 
@@ -113,7 +114,7 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) {
       ColdPartSource.emplace(FF.getAddress(), Function.getOutputAddress());
       Map.clear();
       for (const BinaryBasicBlock *const BB : FF)
-        writeEntriesForBB(Map, *BB, FF.getAddress());
+        writeEntriesForBB(Map, *BB, InputAddress, FF.getAddress());
 
       Maps.emplace(FF.getAddress(), std::move(Map));
     }
@@ -125,11 +126,9 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) {
   writeMaps</*Cold=*/true>(Maps, PrevAddress, OS);
 
   BC.outs() << "BOLT-INFO: Wrote " << Maps.size() << " BAT maps\n";
-  const uint64_t NumBBHashes = std::accumulate(
-      FuncHashes.begin(), FuncHashes.end(), 0ull,
-      [](size_t Acc, const auto &B) { return Acc + B.second.second.size(); });
-  BC.outs() << "BOLT-INFO: Wrote " << FuncHashes.size() << " function and "
-            << NumBBHashes << " basic block hashes\n";
+  BC.outs() << "BOLT-INFO: Wrote " << FuncHashes.getNumFunctions()
+            << " function and " << FuncHashes.getNumBasicBlocks()
+            << " basic block hashes\n";
 }
 
 APInt BoltAddressTranslation::calculateBranchEntriesBitMask(MapTy &Map,
@@ -176,11 +175,10 @@ void BoltAddressTranslation::writeMaps(std::map<uint64_t, MapTy> &Maps,
     // Only process cold fragments in cold mode, and vice versa.
     if (Cold != ColdPartSource.count(Address))
       continue;
-    // NB: here we use the input address because hashes are saved early (in
-    // `saveMetadata`) before output addresses are assigned.
+    // NB: in `writeMaps` we use the input address because hashes are saved
+    // early in `saveMetadata` before output addresses are assigned.
     const uint64_t HotInputAddress =
         ReverseMap[Cold ? ColdPartSource[Address] : Address];
-    std::pair<size_t, BBHashMap> &FuncHashPair = FuncHashes[HotInputAddress];
     MapTy &Map = MapEntry.second;
     const uint32_t NumEntries = Map.size();
     LLVM_DEBUG(dbgs() << "Writing " << NumEntries << " entries for 0x"
@@ -194,10 +192,11 @@ void BoltAddressTranslation::writeMaps(std::map<uint64_t, MapTy> &Maps,
       PrevIndex = HotIndex;
     } else {
       // Function hash
-      LLVM_DEBUG(dbgs() << "Hash: " << formatv("{0:x}\n", FuncHashPair.first));
-      OS.write(reinterpret_cast<char *>(&FuncHashPair.first), 8);
+      size_t BFHash = getBFHash(HotInputAddress);
+      LLVM_DEBUG(dbgs() << "Hash: " << formatv("{0:x}\n", BFHash));
+      OS.write(reinterpret_cast<char *>(&BFHash), 8);
       // Number of basic blocks
-      size_t NumBasicBlocks = FuncHashPair.second.size();
+      size_t NumBasicBlocks = getBBHashMap(HotInputAddress).getNumBasicBlocks();
       LLVM_DEBUG(dbgs() << "Basic blocks: " << NumBasicBlocks << '\n');
       encodeULEB128(NumBasicBlocks, OS);
     }
@@ -221,6 +220,7 @@ void BoltAddressTranslation::writeMaps(std::map<uint64_t, MapTy> &Maps,
         });
       }
     }
+    const BBHashMapTy &BBHashMap = getBBHashMap(HotInputAddress);
     size_t Index = 0;
     uint64_t InOffset = 0;
     size_t PrevBBIndex = 0;
@@ -233,9 +233,9 @@ void BoltAddressTranslation::writeMaps(std::map<uint64_t, MapTy> &Maps,
         encodeSLEB128(KeyVal.second - InOffset, OS);
       InOffset = KeyVal.second; // Keeping InOffset as if BRANCHENTRY is encoded
       if ((InOffset & BRANCHENTRY) == 0) {
-        unsigned BBIndex;
-        size_t BBHash;
-        std::tie(BBIndex, BBHash) = FuncHashPair.second[InOffset >> 1];
+        const bool IsBlock = BBHashMap.isInputBlock(InOffset >> 1);
+        unsigned BBIndex = IsBlock ? BBHashMap.getBBIndex(InOffset >> 1) : 0;
+        size_t BBHash = IsBlock ? BBHashMap.getBBHash(InOffset >> 1) : 0;
         OS.write(reinterpret_cast<char *>(&BBHash), 8);
         // Basic block index in the input binary
         encodeULEB128(BBIndex - PrevBBIndex, OS);
@@ -295,7 +295,7 @@ void BoltAddressTranslation::parseMaps(std::vector<uint64_t> &HotFuncs,
       HotFuncs.push_back(Address);
       // Function hash
       const size_t FuncHash = DE.getU64(&Offset, &Err);
-      FuncHashes[Address].first = FuncHash;
+      FuncHashes.addEntry(Address, FuncHash);
       LLVM_DEBUG(dbgs() << formatv("{0:x}: hash {1:x}\n", Address, FuncHash));
       // Number of basic blocks
       const size_t NumBasicBlocks = DE.getULEB128(&Offset, &Err);
@@ -355,8 +355,7 @@ void BoltAddressTranslation::parseMaps(std::vector<uint64_t> &HotFuncs,
         BBIndexDelta = DE.getULEB128(&Offset, &Err);
         BBIndex += BBIndexDelta;
         // Map basic block hash to hot fragment by input offset
-        FuncHashes[HotAddress].second.emplace(InputOffset >> 1,
-                                              std::pair(BBIndex, BBHash));
+        getBBHashMap(HotAddress).addEntry(InputOffset >> 1, BBIndex, BBHash);
       }
       LLVM_DEBUG({
         dbgs() << formatv(
@@ -385,6 +384,8 @@ void BoltAddressTranslation::dump(raw_ostream &OS) {
       OS << formatv(", hash: {0:x}", getBFHash(Address));
     OS << "\n";
     OS << "BB mappings:\n";
+    const BBHashMapTy &BBHashMap =
+        getBBHashMap(HotAddress ? HotAddress : Address);
     for (const auto &Entry : MapEntry.second) {
       const bool IsBranch = Entry.second & BRANCHENTRY;
       const uint32_t Val = Entry.second >> 1; // dropping BRANCHENTRY bit
@@ -393,8 +394,7 @@ void BoltAddressTranslation::dump(raw_ostream &OS) {
       if (IsBranch)
         OS << " (branch)";
       else
-        OS << formatv(" hash: {0:x}",
-                      getBBHash(HotAddress ? HotAddress : Address, Val));
+        OS << formatv(" hash: {0:x}", BBHashMap.getBBHash(Val));
       OS << "\n";
     }
     OS << "\n";
@@ -515,27 +515,14 @@ void BoltAddressTranslation::saveMetadata(BinaryContext &BC) {
     if (BF.isIgnored() || (!BC.HasRelocations && !BF.isSimple()))
       continue;
     // Prepare function and block hashes
-    FuncHashes[BF.getAddress()].first = BF.computeHash();
+    FuncHashes.addEntry(BF.getAddress(), BF.computeHash());
     BF.computeBlockHashes();
+    BBHashMapTy &BBHashMap = getBBHashMap(BF.getAddress());
+    // Set BF/BB metadata
     for (const BinaryBasicBlock &BB : BF)
-      FuncHashes[BF.getAddress()].second.emplace(
-          BB.getInputOffset(), std::pair(BB.getIndex(), BB.getHash()));
+      BBHashMap.addEntry(BB.getInputOffset(), BB.getIndex(), BB.getHash());
   }
 }
 
-unsigned BoltAddressTranslation::getBBIndex(uint64_t FuncOutputAddress,
-                                            uint32_t BBInputOffset) const {
-  return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).first;
-}
-
-size_t BoltAddressTranslation::getBBHash(uint64_t FuncOutputAddress,
-                                         uint32_t BBInputOffset) const {
-  return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).second;
-}
-
-size_t BoltAddressTranslation::getBFHash(uint64_t OutputAddress) const {
-  return FuncHashes.at(OutputAddress).first;
-}
-
 } // namespace bolt
 } // namespace llvm
-- 
GitLab


From 90a7fc366ad05a7dd0465730c055af57c80d62d9 Mon Sep 17 00:00:00 2001
From: "Felix (Ting Wang)" <Ting.Wang.SH@ibm.com>
Date: Sun, 24 Mar 2024 08:46:45 +0800
Subject: [PATCH 057/404] [PowerPC][NFC] Add base test case for
 small-local-dynamic-tls on AIX (#84711)

---
 ...aix-small-local-dynamic-tls-largeaccess.ll |  632 ++++++++++
 .../aix-small-local-dynamic-tls-types.ll      | 1066 +++++++++++++++++
 2 files changed, 1698 insertions(+)
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll

diff --git a/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll
new file mode 100644
index 000000000000..eb16bae67150
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll
@@ -0,0 +1,632 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc  -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:      -mtriple powerpc64-ibm-aix-xcoff < %s \
+; RUN:      | FileCheck %s --check-prefix=SMALL-LOCAL-DYNAMIC-SMALLCM64
+; RUN: llc  -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:      -mtriple powerpc64-ibm-aix-xcoff --code-model=large \
+; RUN:      < %s | FileCheck %s \
+; RUN:      --check-prefix=SMALL-LOCAL-DYNAMIC-LARGECM64
+
+; Test disassembly of object.
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 \
+; RUN:      -mtriple powerpc64-ibm-aix-xcoff -xcoff-traceback-table=false \
+; RUN:      --code-model=large -filetype=obj -o %t.o < %s
+; RUN: llvm-objdump -D -r --symbol-description %t.o | FileCheck -D#NFA=2 --check-prefix=DIS %s
+
+@ElementIntTLSv1 = thread_local(localdynamic) global [8187 x i32] zeroinitializer, align 4  ; Within 32K
+@ElementIntTLS2 = thread_local(localdynamic) global [4000 x i32] zeroinitializer, align 4
+@ElementIntTLS3 = thread_local(localdynamic) global [4000 x i32] zeroinitializer, align 4
+@ElementIntTLS4 = thread_local(localdynamic) global [4000 x i32] zeroinitializer, align 4
+@ElementIntTLS5 = thread_local(localdynamic) global [4000 x i32] zeroinitializer, align 4
+@ElementIntTLSv2 = thread_local(localdynamic) global [9000 x i32] zeroinitializer, align 4  ; Beyond 32K
+
+@ElementLongTLS6 = external thread_local(localdynamic) global [60 x i64], align 8
+@ElementLongTLS2 = thread_local(localdynamic) global [3000 x i64] zeroinitializer, align 8  ; Within 32K
+@MyTLSGDVar = thread_local global [800 x i64] zeroinitializer, align 8
+@ElementLongTLS3 = thread_local(localdynamic) global [3000 x i64] zeroinitializer, align 8
+@ElementLongTLS4 = thread_local(localdynamic) global [3000 x i64] zeroinitializer, align 8
+@ElementLongTLS5 = thread_local(localdynamic) global [3000 x i64] zeroinitializer, align 8
+@ElementLongTLS = thread_local(localdynamic) local_unnamed_addr global [7800 x i64] zeroinitializer, align 8  ; Beyond 32K
+
+declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) #1
+
+; All accesses use a "faster" local-dynamic sequence directly off the module handle.
+; Exercise PPCXCOFFObjectWriter::getRelocTypeAndSignSize/fixup_ppc_half16.
+define signext i32 @test1() {
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: test1:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r6, L..C1(r2) # target-flags(ppc-tlsld) @ElementIntTLS2
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r7, L..C2(r2) # target-flags(ppc-tlsld) @ElementIntTLS3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r8, L..C3(r2) # target-flags(ppc-tlsld) @ElementIntTLS4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r9, L..C4(r2) # target-flags(ppc-tlsld) @ElementIntTLS5
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r5, L..C5(r2) # target-flags(ppc-tlsld) @ElementIntTLSv1
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 1
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r6, r3, r6
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r7, r3, r7
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r8, r3, r8
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r9, r3, r9
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stwux r4, r3, r5
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r4, 24(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 2
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r3, 320(r6)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r3, 324(r7)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 88
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r4, 328(r8)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r3, 332(r9)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 102
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: test1:
+; SMALL-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C1@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r7, L..C2@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r8, L..C3@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r9, L..C4@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r7, L..C2@l(r7)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r8, L..C3@l(r8)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r9, L..C4@l(r9)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r5, L..C1@l(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C5@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 1
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r6, L..C5@l(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r7, r3, r7
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r8, r3, r8
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r9, r3, r9
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r6, r3, r6
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stwux r4, r3, r5
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r4, 24(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 2
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r3, 320(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r3, 324(r7)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 88
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r4, 328(r8)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r3, 332(r9)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 102
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+entry:
+  %tls1 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLSv1)
+  store i32 1, ptr %tls1, align 4
+  %arrayidx1 = getelementptr inbounds [8187 x i32], ptr %tls1, i64 0, i64 6
+  store i32 4, ptr %arrayidx1, align 4
+  %tls2 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS2)
+  %arrayidx2 = getelementptr inbounds [4000 x i32], ptr %tls2, i64 0, i64 80
+  store i32 2, ptr %arrayidx2, align 4
+  %tls3 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS3)
+  %arrayidx3 = getelementptr inbounds [4000 x i32], ptr %tls3, i64 0, i64 81
+  store i32 3, ptr %arrayidx3, align 4
+  %tls4 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS4)
+  %arrayidx4 = getelementptr inbounds [4000 x i32], ptr %tls4, i64 0, i64 82
+  store i32 4, ptr %arrayidx4, align 4
+  %tls5 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS5)
+  %arrayidx5 = getelementptr inbounds [4000 x i32], ptr %tls5, i64 0, i64 83
+  store i32 88, ptr %arrayidx5, align 4
+  %load1 = load i32, ptr %tls1, align 4
+  %load2 = load i32, ptr %arrayidx1, align 4
+  %load3 = load i32, ptr %arrayidx2, align 4
+  %load4 = load i32, ptr %arrayidx3, align 4
+  %load5 = load i32, ptr %arrayidx4, align 4
+  %add = add i32 %load1, 88
+  %add6 = add i32 %add, %load2
+  %add8 = add i32 %add6, %load3
+  %add10 = add i32 %add8, %load4
+  %add12 = add i32 %add10, %load5
+  ret i32 %add12
+}
+
+; All accesses use a "faster" local-dynamic sequence directly off the module handle.
+; Exercise PPCXCOFFObjectWriter::getRelocTypeAndSignSize/fixup_ppc_half16ds.
+define i64 @test2() {
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: test2:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C6(r2) # target-flags(ppc-tlsld) @ElementLongTLS6
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mr r6, r3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 212
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r4, r6, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r3, 424(r4)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C7(r2) # target-flags(ppc-tlsld) @ElementLongTLS2
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 203
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r3, r6, r3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r4, 1200(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C8(r2) # target-flags(ppc-tlsgdm) @MyTLSGDVar
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C9(r2) # target-flags(ppc-tlsgd) @MyTLSGDVar
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_addr[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 44
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r4, 440(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C10(r2) # target-flags(ppc-tlsld) @ElementLongTLS3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 6
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r3, r6, r3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r4, 2000(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C11(r2) # target-flags(ppc-tlsld) @ElementLongTLS4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 100
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r4, r6, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r3, 6800(r4)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C12(r2) # target-flags(ppc-tlsld) @ElementLongTLS5
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 882
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r3, r6, r3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r4, 8400(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 1191
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: test2:
+; SMALL-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r7, L..C6@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 212
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mr r6, r3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C6@l(r7)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r3, r6, r3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, 424(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C7@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 203
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C7@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r3, r6, r3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, 1200(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C8@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r4, L..C9@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C8@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C9@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_addr[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 44
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, 440(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C10@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 6
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C10@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r3, r6, r3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, 2000(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C11@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 100
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C11@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r3, r6, r3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, 6800(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C12@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 882
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C12@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r3, r6, r3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, 8400(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 1191
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+entry:
+  %tls1 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS6)
+  %arrayidx = getelementptr inbounds [60 x i64], ptr %tls1, i64 0, i64 53
+  store i64 212, ptr %arrayidx, align 8
+  %tls2 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS2)
+  %arrayidx1 = getelementptr inbounds [3000 x i64], ptr %tls2, i64 0, i64 150
+  store i64 203, ptr %arrayidx1, align 8
+  %tls3 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @MyTLSGDVar)
+  %arrayidx2 = getelementptr inbounds [800 x i64], ptr %tls3, i64 0, i64 55
+  store i64 44, ptr %arrayidx2, align 8
+  %tls4 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS3)
+  %arrayidx3 = getelementptr inbounds [3000 x i64], ptr %tls4, i64 0, i64 250
+  store i64 6, ptr %arrayidx3, align 8
+  %tls5 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS4)
+  %arrayidx4 = getelementptr inbounds [3000 x i64], ptr %tls5, i64 0, i64 850
+  store i64 100, ptr %arrayidx4, align 8
+  %tls6 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ElementLongTLS5)
+  %arrayidx5 = getelementptr inbounds [3000 x i64], ptr %tls6, i64 0, i64 1050
+  store i64 882, ptr %arrayidx5, align 8
+  %load1 = load i64, ptr %arrayidx1, align 8
+  %load2 = load i64, ptr %arrayidx3, align 8
+  %load3 = load i64, ptr %arrayidx4, align 8
+  %add = add i64 %load1, 882
+  %add9 = add i64 %add, %load2
+  %add11 = add i64 %add9, %load3
+  ret i64 %add11
+}
+
+; Example of one access using the regular local-dynamic access from the TOC.
+define signext i32 @test3() {
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: test3:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r6, L..C1(r2) # target-flags(ppc-tlsld) @ElementIntTLS2
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r7, L..C2(r2) # target-flags(ppc-tlsld) @ElementIntTLS3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r8, L..C3(r2) # target-flags(ppc-tlsld) @ElementIntTLS4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r9, L..C4(r2) # target-flags(ppc-tlsld) @ElementIntTLS5
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r5, L..C13(r2) # target-flags(ppc-tlsld) @ElementIntTLSv2
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 1
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r6, r3, r6
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r7, r3, r7
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r8, r3, r8
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r9, r3, r9
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stwux r4, r3, r5
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r4, 24(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 2
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r3, 320(r6)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r3, 324(r7)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 88
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r4, 328(r8)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r3, 332(r9)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 102
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: test3:
+; SMALL-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C13@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r7, L..C2@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r8, L..C3@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r9, L..C4@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r7, L..C2@l(r7)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r8, L..C3@l(r8)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r9, L..C4@l(r9)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r5, L..C13@l(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C5@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 1
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r6, L..C5@l(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r7, r3, r7
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r8, r3, r8
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r9, r3, r9
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r6, r3, r6
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stwux r4, r3, r5
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r4, 24(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 2
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r3, 320(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r3, 324(r7)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 88
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r4, 328(r8)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r3, 332(r9)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 102
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+entry:
+  %tls1 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLSv2)
+  store i32 1, ptr %tls1, align 4
+  %arrayidx1 = getelementptr inbounds [9000 x i32], ptr %tls1, i64 0, i64 6
+  store i32 4, ptr %arrayidx1, align 4
+  %tls2 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS2)
+  %arrayidx2 = getelementptr inbounds [4000 x i32], ptr %tls2, i64 0, i64 80
+  store i32 2, ptr %arrayidx2, align 4
+  %tls3 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS3)
+  %arrayidx3 = getelementptr inbounds [4000 x i32], ptr %tls3, i64 0, i64 81
+  store i32 3, ptr %arrayidx3, align 4
+  %tls4 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS4)
+  %arrayidx4 = getelementptr inbounds [4000 x i32], ptr %tls4, i64 0, i64 82
+  store i32 4, ptr %arrayidx4, align 4
+  %tls5 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ElementIntTLS5)
+  %arrayidx5 = getelementptr inbounds [4000 x i32], ptr %tls5, i64 0, i64 83
+  store i32 88, ptr %arrayidx5, align 4
+  %load1 = load i32, ptr %tls1, align 4
+  %load2 = load i32, ptr %arrayidx1, align 4
+  %load3 = load i32, ptr %arrayidx2, align 4
+  %load4 = load i32, ptr %arrayidx3, align 4
+  %load5 = load i32, ptr %arrayidx4, align 4
+  %add = add i32 %load1, 88
+  %add9 = add i32 %add, %load2
+  %add11 = add i32 %add9, %load3
+  %add13 = add i32 %add11, %load4
+  %add15 = add i32 %add13, %load5
+  ret i32 %add15
+}
+
+; DIS:      file format aix5coff64-rs6000
+; DIS:      Disassembly of section .text:
+; DIS:      0000000000000000 (idx: [[#NFA+9]]) .test1:
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                mflr 0
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                stdu 1, -48(1)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 3, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+23]]) _$TLSML[TC]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 6, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+25]]) ElementIntTLSv1[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                std 0, 64(1)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 7, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+27]]) ElementIntTLS3[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 3, 0(3)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+23]]) _$TLSML[TC]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 8, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+29]]) ElementIntTLS4[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 9, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+31]]) ElementIntTLS5[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 7, 16(7)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+27]]) ElementIntTLS3[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 8, 24(8)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+29]]) ElementIntTLS4[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 9, 32(9)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+31]]) ElementIntTLS5[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                bla 0x0
+; DIS-NEXT: {{0*}}[[#ADDR]]: R_RBA	(idx: [[#NFA+1]]) .__tls_get_mod[PR]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 5, 8(6)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+25]]) ElementIntTLSv1[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 6, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+33]]) ElementIntTLS2[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                li 4, 1
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 6, 40(6)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+33]]) ElementIntTLS2[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                add 7, 3, 7
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                add 8, 3, 8
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                add 9, 3, 9
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                add 6, 3, 6
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                stwux 4, 3, 5
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                li 4, 4
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                stw 4, 24(3)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                li 3, 2
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                stw 3, 320(6)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                li 3, 3
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                stw 3, 324(7)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                li 3, 88
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                stw 4, 328(8)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                stw 3, 332(9)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                li 3, 102
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addi 1, 1, 48
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 0, 16(1)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                mtlr 0
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                blr
+
+; DIS:      0000000000000090 (idx: [[#NFA+11]]) .test2:
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                mflr 0
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                stdu 1, -48(1)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 3, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+23]]) _$TLSML[TC]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                std 0, 64(1)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 7, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+35]]) ElementLongTLS6[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 3, 0(3)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+23]]) _$TLSML[TC]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                bla 0x0
+; DIS-NEXT: {{0*}}[[#ADDR]]: R_RBA	(idx: [[#NFA+1]]) .__tls_get_mod[PR]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                li 4, 212
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                mr	6, 3
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 3, 48(7)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+35]]) ElementLongTLS6[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                add 3, 6, 3
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                std 4, 424(3)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 3, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+37]]) ElementLongTLS2[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                li 4, 203
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 3, 56(3)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+37]]) ElementLongTLS2[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                add 3, 6, 3
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                std 4, 1200(3)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 3, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+39]]) .MyTLSGDVar[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 4, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+41]]) MyTLSGDVar[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 3, 64(3)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+39]]) .MyTLSGDVar[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 4, 72(4)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+41]]) MyTLSGDVar[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                bla 0x0
+; DIS-NEXT: {{0*}}[[#ADDR]]: R_RBA	(idx: [[#NFA+3]]) .__tls_get_addr[PR]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                li 4, 44
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                std 4, 440(3)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 3, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+43]]) ElementLongTLS3[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                li 4, 6
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 3, 80(3)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+43]]) ElementLongTLS3[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                add 3, 6, 3
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                std 4, 2000(3)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 3, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+45]]) ElementLongTLS4[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                li 4, 100
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 3, 88(3)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+45]]) ElementLongTLS4[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                add 3, 6, 3
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                std 4, 6800(3)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 3, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+47]]) ElementLongTLS5[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                li 4, 882
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 3, 96(3)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+47]]) ElementLongTLS5[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                add 3, 6, 3
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                std 4, 8400(3)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                li 3, 1191
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addi 1, 1, 48
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 0, 16(1)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                mtlr 0
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                blr
+
+; DIS:      0000000000000140 (idx: [[#NFA+13]]) .test3:
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                mflr 0
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                stdu 1, -48(1)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 3, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+23]]) _$TLSML[TC]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 6, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+49]]) ElementIntTLSv2[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                std 0, 64(1)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 7, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+27]]) ElementIntTLS3[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 3, 0(3)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+23]]) _$TLSML[TC]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 8, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+29]]) ElementIntTLS4[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 9, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+31]]) ElementIntTLS5[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 7, 16(7)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+27]]) ElementIntTLS3[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 8, 24(8)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+29]]) ElementIntTLS4[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 9, 32(9)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+31]]) ElementIntTLS5[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                bla 0x0
+; DIS-NEXT: {{0*}}[[#ADDR]]: R_RBA	(idx: [[#NFA+1]]) .__tls_get_mod[PR]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 5, 104(6)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+49]]) ElementIntTLSv2[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 6, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: [[#NFA+33]]) ElementIntTLS2[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                li 4, 1
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 6, 40(6)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: [[#NFA+33]]) ElementIntTLS2[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                add 7, 3, 7
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                add 8, 3, 8
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                add 9, 3, 9
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                add 6, 3, 6
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                stwux 4, 3, 5
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                li 4, 4
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                stw 4, 24(3)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                li 3, 2
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                stw 3, 320(6)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                li 3, 3
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                stw 3, 324(7)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                li 3, 88
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                stw 4, 328(8)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                stw 3, 332(9)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                li 3, 102
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addi 1, 1, 48
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 0, 16(1)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                mtlr 0
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                blr
+
+; DIS:      Disassembly of section .data:
+
+; DIS:      00000000000001d0 (idx: 17) test1[DS]:
+; DIS-NEXT:      1d0: 00 00 00 00
+; DIS-NEXT: 00000000000001d0:  R_POS	(idx: [[#NFA+9]]) .test1
+; DIS-NEXT:      1d4: 00 00 00 00
+; DIS-NEXT:      1d8: 00 00 00 00
+; DIS-NEXT: 00000000000001d8:  R_POS	(idx: [[#NFA+21]]) TOC[TC0]
+; DIS-NEXT:      1dc: 00 00 02 18
+
+; DIS:      00000000000001e8 (idx: 19) test2[DS]:
+; DIS-NEXT:      1e8: 00 00 00 00
+; DIS-NEXT: 00000000000001e8:  R_POS	(idx: [[#NFA+11]]) .test2
+; DIS-NEXT:      1ec: 00 00 00 90
+; DIS-NEXT:      1f0: 00 00 00 00
+; DIS-NEXT: 00000000000001f0:  R_POS	(idx: [[#NFA+21]]) TOC[TC0]
+; DIS-NEXT:      1f4: 00 00 02 18
+
+; DIS:      0000000000000200 (idx: 21) test3[DS]:
+; DIS-NEXT:      200: 00 00 00 00
+; DIS-NEXT: 0000000000000200:  R_POS	(idx: [[#NFA+13]]) .test3
+; DIS-NEXT:      204: 00 00 01 40
+; DIS-NEXT:      208: 00 00 00 00
+; DIS-NEXT: 0000000000000208:  R_POS	(idx: [[#NFA+21]]) TOC[TC0]
+; DIS-NEXT:      20c: 00 00 02 18
+
+; DIS:      0000000000000218 (idx: 25) _$TLSML[TC]:
+; DIS-NEXT:      218: 00 00 00 00
+; DIS-NEXT: 0000000000000218:  R_TLSML	(idx: [[#NFA+23]]) _$TLSML[TC]
+; DIS-NEXT:      21c: 00 00 00 00
+
+; DIS:      0000000000000220 (idx: 27) ElementIntTLSv1[TE]:
+; DIS-NEXT:      220: 00 00 00 00
+; DIS-NEXT: 0000000000000220:  R_TLS_LD	(idx: [[#NFA+51]]) ElementIntTLSv1[TL]
+; DIS-NEXT:      224: 00 00 00 00
+
+; DIS:      0000000000000228 (idx: 29) ElementIntTLS3[TE]:
+; DIS-NEXT:      228: 00 00 00 00
+; DIS-NEXT: 0000000000000228:  R_TLS_LD	(idx: [[#NFA+55]]) ElementIntTLS3[TL]
+; DIS-NEXT:      22c: 00 00 be 6c
+
+; DIS:      0000000000000230 (idx: 31) ElementIntTLS4[TE]:
+; DIS-NEXT:      230: 00 00 00 00
+; DIS-NEXT: 0000000000000230:  R_TLS_LD	(idx: [[#NFA+57]]) ElementIntTLS4[TL]
+; DIS-NEXT:      234: 00 00 fc ec
+
+; DIS:      0000000000000238 (idx: 33) ElementIntTLS5[TE]:
+; DIS-NEXT:      238: 00 00 00 00
+; DIS-NEXT: 0000000000000238:  R_TLS_LD	(idx: [[#NFA+59]]) ElementIntTLS5[TL]
+; DIS-NEXT:      23c: 00 01 3b 6c
+
+; DIS:      0000000000000240 (idx: 35) ElementIntTLS2[TE]:
+; DIS-NEXT:      240: 00 00 00 00
+; DIS-NEXT: 0000000000000240:  R_TLS_LD	(idx: [[#NFA+53]]) ElementIntTLS2[TL]
+; DIS-NEXT:      244: 00 00 7f ec
+
+; DIS:      0000000000000248 (idx: 37) ElementLongTLS6[TE]:
+; DIS-NEXT:      248: 00 00 00 00
+; DIS-NEXT: 0000000000000248:  R_TLS_LD	(idx: [[#NFA+5]]) ElementLongTLS6[UL]
+; DIS-NEXT:      24c: 00 00 00 00
+
+; DIS:      0000000000000250 (idx: 39) ElementLongTLS2[TE]:
+; DIS-NEXT:      250: 00 00 00 00
+; DIS-NEXT: 0000000000000250:  R_TLS_LD	(idx: [[#NFA+63]]) ElementLongTLS2[TL]
+; DIS-NEXT:      254: 00 02 06 90
+
+; DIS:      0000000000000258 (idx: 41) .MyTLSGDVar[TE]:
+; DIS-NEXT:      258: 00 00 00 00
+; DIS-NEXT: 0000000000000258:  R_TLSM	(idx: [[#NFA+65]]) MyTLSGDVar[TL]
+; DIS-NEXT:      25c: 00 00 00 00
+
+; DIS:      0000000000000260 (idx: 43) MyTLSGDVar[TE]:
+; DIS-NEXT:      260: 00 00 00 00
+; DIS-NEXT: 0000000000000260:  R_TLS	(idx: [[#NFA+65]]) MyTLSGDVar[TL]
+; DIS-NEXT:      264: 00 02 64 50
+
+; DIS:      0000000000000268 (idx: 45) ElementLongTLS3[TE]:
+; DIS-NEXT:      268: 00 00 00 00
+; DIS-NEXT: 0000000000000268:  R_TLS_LD	(idx: [[#NFA+67]]) ElementLongTLS3[TL]
+; DIS-NEXT:      26c: 00 02 7d 50
+
+; DIS:      0000000000000270 (idx: 47) ElementLongTLS4[TE]:
+; DIS-NEXT:      270: 00 00 00 00
+; DIS-NEXT: 0000000000000270:  R_TLS_LD	(idx: [[#NFA+69]]) ElementLongTLS4[TL]
+; DIS-NEXT:      274: 00 02 db 10
+
+; DIS:      0000000000000278 (idx: 49) ElementLongTLS5[TE]:
+; DIS-NEXT:      278: 00 00 00 00
+; DIS-NEXT: 0000000000000278:  R_TLS_LD	(idx: [[#NFA+71]]) ElementLongTLS5[TL]
+; DIS-NEXT:      27c: 00 03 38 d0
+
+; DIS:      0000000000000280 (idx: 51) ElementIntTLSv2[TE]:
+; DIS-NEXT:      280: 00 00 00 00
+; DIS-NEXT: 0000000000000280:  R_TLS_LD	(idx: [[#NFA+61]]) ElementIntTLSv2[TL]
+; DIS-NEXT:      284: 00 01 79 ec
+
+; DIS:      Disassembly of section .tdata:
+; DIS:      0000000000000000 (idx: [[#NFA+51]]) ElementIntTLSv1[TL]:
+; DIS:      0000000000007fec (idx: [[#NFA+53]]) ElementIntTLS2[TL]:
+; DIS:      000000000000be6c (idx: [[#NFA+55]]) ElementIntTLS3[TL]:
+; DIS:      000000000000fcec (idx: [[#NFA+57]]) ElementIntTLS4[TL]:
+; DIS:      0000000000013b6c (idx: [[#NFA+59]]) ElementIntTLS5[TL]:
+; DIS:      00000000000179ec (idx: [[#NFA+61]]) ElementIntTLSv2[TL]:
+; DIS:      0000000000020690 (idx: [[#NFA+63]]) ElementLongTLS2[TL]:
+; DIS:      0000000000026450 (idx: [[#NFA+65]]) MyTLSGDVar[TL]:
+; DIS:      0000000000027d50 (idx: [[#NFA+67]]) ElementLongTLS3[TL]:
+; DIS:      000000000002db10 (idx: [[#NFA+69]]) ElementLongTLS4[TL]:
+; DIS:      00000000000338d0 (idx: [[#NFA+71]]) ElementLongTLS5[TL]:
+; DIS:      0000000000039690 (idx: [[#NFA+73]]) ElementLongTLS[TL]:
diff --git a/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll
new file mode 100644
index 000000000000..d996d86a23d8
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll
@@ -0,0 +1,1066 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc  -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:      -mtriple powerpc64-ibm-aix-xcoff < %s \
+; RUN:      | FileCheck %s --check-prefix=SMALL-LOCAL-DYNAMIC-SMALLCM64
+; RUN: llc  -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:      -mtriple powerpc64-ibm-aix-xcoff --code-model=large \
+; RUN:      < %s | FileCheck %s \
+; RUN:      --check-prefix=SMALL-LOCAL-DYNAMIC-LARGECM64
+; RUN: llc  -O0 -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:      -mtriple powerpc64-ibm-aix-xcoff < %s \
+; RUN:      | FileCheck %s --check-prefix=SMALL-LOCAL-DYNAMIC-SMALLCM64-O0
+; RUN: llc  -O0 -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:      -mtriple powerpc64-ibm-aix-xcoff --code-model=large \
+; RUN:      < %s | FileCheck %s \
+; RUN:      --check-prefix=SMALL-LOCAL-DYNAMIC-LARGECM64-O0
+
+declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) #1
+@tlv_int_init = local_unnamed_addr global i32 87, align 4
+
+@tlv_char = thread_local(localdynamic) global i8 1, align 1
+@tlv_short = thread_local(localdynamic) global i8 1, align 2
+@tlv_int = thread_local(localdynamic) global i32 1, align 4
+@internal_tlv_int = internal thread_local(localdynamic) global i32 1, align 4
+@tlv_long = thread_local(localdynamic) global i64 1, align 8
+@internal_tlv_long = internal thread_local(localdynamic) global i64 1, align 8
+@tlv_float = thread_local(localdynamic) global float 1.000000e+00, align 4
+@internal_tlv_double = internal thread_local(localdynamic) global double 1.000000e+00, align 8
+
+%struct.anon = type { i32 }
+@ThreadLocalStruct = thread_local(localdynamic) global %struct.anon zeroinitializer, align 1
+@a = thread_local(localdynamic) global [87 x i32] zeroinitializer, align 4
+
+define nonnull ptr @AddrTest1() local_unnamed_addr {
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: AddrTest1:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C1(r2) # target-flags(ppc-tlsld) @a
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r3, r3, 12
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: AddrTest1:
+; SMALL-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C1@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C1@l(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r3, r3, 12
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: AddrTest1:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C1(r2) # target-flags(ppc-tlsld) @a
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r3, r3, 12
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: AddrTest1:
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C0@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r3, r3, 12
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+entry:
+  %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @a)
+  %arrayidx = getelementptr inbounds [87 x i32], ptr %tlv_addr, i64 0, i64 3
+  ret ptr %arrayidx
+}
+
+define signext i32 @testUnaligned() {
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testUnaligned:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C2(r2) # target-flags(ppc-tlsld) @ThreadLocalStruct
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwax r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testUnaligned:
+; SMALL-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C2@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C2@l(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwax r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testUnaligned:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C2(r2) # target-flags(ppc-tlsld) @ThreadLocalStruct
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwa r3, 0(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testUnaligned:
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C2@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C2@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwa r3, 0(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+entry:
+  %tlv_addr = call align 1 ptr @llvm.threadlocal.address.p0(ptr align 1 @ThreadLocalStruct)
+  %x = getelementptr inbounds %struct.anon, ptr %tlv_addr, i32 0, i32 0
+  %value = load i32, ptr %x, align 1
+  ret i32 %value
+}
+
+define void @testChar(i8 noundef signext %x) {
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testChar:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mr r6, r3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C3(r2) # target-flags(ppc-tlsld) @tlv_char
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stbx r6, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testChar:
+; SMALL-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mr r6, r3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r7, L..C3@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C3@l(r7)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stbx r6, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testChar:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stw r3, 60(r1) # 4-byte Folded Spill
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mr r4, r3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwz r3, 60(r1) # 4-byte Folded Reload
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r5, L..C3(r2) # target-flags(ppc-tlsld) @tlv_char
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r4, r4, r5
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stb r3, 0(r4)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testChar:
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stw r3, 60(r1) # 4-byte Folded Spill
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C3@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 48(r1) # 8-byte Folded Spill
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r5, 48(r1) # 8-byte Folded Reload
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mr r4, r3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwz r3, 60(r1) # 4-byte Folded Reload
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r5, L..C3@l(r5)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r4, r4, r5
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stb r3, 0(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+entry:
+  %tlv_addr = tail call align 1 ptr @llvm.threadlocal.address.p0(ptr align 1 @tlv_char)
+  store i8 %x, ptr %tlv_addr, align 1
+  ret void
+}
+
+define void @testShort(i16 noundef signext %x) {
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testShort:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mr r6, r3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C4(r2) # target-flags(ppc-tlsld) @tlv_short
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    sthx r6, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testShort:
+; SMALL-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mr r6, r3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r7, L..C4@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C4@l(r7)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    sthx r6, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testShort:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stw r3, 60(r1) # 4-byte Folded Spill
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mr r4, r3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwz r3, 60(r1) # 4-byte Folded Reload
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r5, L..C4(r2) # target-flags(ppc-tlsld) @tlv_short
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r4, r4, r5
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    sth r3, 0(r4)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testShort:
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stw r3, 60(r1) # 4-byte Folded Spill
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C4@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 48(r1) # 8-byte Folded Spill
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r5, 48(r1) # 8-byte Folded Reload
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mr r4, r3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwz r3, 60(r1) # 4-byte Folded Reload
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r5, L..C4@l(r5)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r4, r4, r5
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    sth r3, 0(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+entry:
+  %tlv_addr = tail call align 2 ptr @llvm.threadlocal.address.p0(ptr align 2 @tlv_short)
+  store i16 %x, ptr %tlv_addr, align 2
+  ret void
+}
+
+define signext i32 @testInt1() {
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testInt1:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C5(r2) # target-flags(ppc-tlsld) @tlv_int
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwax r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testInt1:
+; SMALL-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C5@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C5@l(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwax r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testInt1:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C5(r2) # target-flags(ppc-tlsld) @tlv_int
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwa r3, 0(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testInt1:
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C5@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C5@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwa r3, 0(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+entry:
+  %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @tlv_int)
+  %value = load i32, ptr %tlv_addr, align 4
+  ret i32 %value
+}
+
+define signext i32 @testInt2() {
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testInt2:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C6(r2) # target-flags(ppc-tlsld) @internal_tlv_int
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwzx r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C7(r2) # @tlv_int_init
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwz r4, 0(r4)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r3, r4, r3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    extsw r3, r3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testInt2:
+; SMALL-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C6@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C6@l(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwzx r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r4, L..C7@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C7@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwz r4, 0(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r3, r4, r3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    extsw r3, r3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testInt2:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C6(r2) # target-flags(ppc-tlsld) @internal_tlv_int
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwz r4, 0(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C7(r2) # @tlv_int_init
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwz r3, 0(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    extsw r3, r3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testInt2:
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C6@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C6@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwz r4, 0(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C7@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C7@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwz r3, 0(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    extsw r3, r3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+entry:
+  %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @internal_tlv_int)
+  %tlv_val = load i32, ptr %tlv_addr, align 4
+  %global_val = load i32, ptr @tlv_int_init, align 4
+  %sum = add nsw i32 %global_val, %tlv_val
+  ret i32 %sum
+}
+
+define signext i64 @testLong1() {
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testLong1:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ldx r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testLong1:
+; SMALL-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C8@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C8@l(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ldx r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testLong1:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, 0(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testLong1:
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C8@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C8@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, 0(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+entry:
+  %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @tlv_long)
+  %value = load i64, ptr %tlv_addr, align 4
+  ret i64 %value
+}
+
+define void @testLong2(i64 noundef signext %x) {
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testLong2:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C9(r2) # target-flags(ppc-tlsld) @internal_tlv_long
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ldx r5, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r5, r5, 9
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdx r5, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testLong2:
+; SMALL-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C9@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C9@l(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ldx r5, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r5, r5, 9
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdx r5, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testLong2:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C9(r2) # target-flags(ppc-tlsld) @internal_tlv_long
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r4, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, 0(r4)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r3, r3, 9
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r3, 0(r4)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testLong2:
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C9@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C9@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r4, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, 0(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r3, r3, 9
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 0(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+entry:
+  %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @internal_tlv_long)
+  %value = load i64, ptr %tlv_addr, align 8
+  %add = add nsw i64 %value, 9
+  store i64 %add, ptr %tlv_addr, align 8
+  ret void
+}
+
+define i32 @testLong3() {
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testLong3:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ldx r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testLong3:
+; SMALL-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C8@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C8@l(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ldx r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testLong3:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, 0(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    clrldi r3, r3, 32
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testLong3:
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C8@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C8@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, 0(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    clrldi r3, r3, 32
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+entry:
+  %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @tlv_long)
+  %value = load i64, ptr %tlv_addr, align 8
+  %conv = trunc i64 %value to i32
+  ret i32 %conv
+}
+
+define void @testFloat1(float noundef %x) {
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testFloat1:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    vspltisw v2, 1
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    vspltisw v3, 8
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    xvcvsxwdp vs0, vs34
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lfsx f1, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    fadds f0, f1, f0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    xvcvsxwdp vs1, vs35
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    fadds f0, f0, f1
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stfsx f0, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testFloat1:
+; SMALL-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    vspltisw v2, 1
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C10@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    vspltisw v3, 8
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    xvcvsxwdp vs0, vs34
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C10@l(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lfsx f1, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    fadds f0, f1, f0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    xvcvsxwdp vs1, vs35
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    fadds f0, f0, f1
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stfsx f0, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testFloat1:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lfs f0, 0(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C11(r2) # %const.1
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lfs f1, 0(r4)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    fadds f0, f0, f1
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C12(r2) # %const.0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lfs f1, 0(r4)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    fadds f0, f0, f1
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stfs f0, 0(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testFloat1:
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C10@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C10@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lfs f0, 0(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r4, L..C11@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C11@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lfs f1, 0(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    fadds f0, f0, f1
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r4, L..C12@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C12@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lfs f1, 0(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    fadds f0, f0, f1
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stfs f0, 0(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+entry:
+  %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @tlv_float)
+  %value = load float, ptr %tlv_addr, align 4
+  %inc = fadd float %value, 1.000000e+00
+  %add = fadd float %inc, 8.000000e+00
+  store float %add, ptr %tlv_addr, align 4
+  ret void
+}
+
+define i32 @testFloat2() {
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testFloat2:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lfsx f0, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r3, r1, 60
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    xscvdpsxws f0, f0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stfiwx f0, 0, r3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwz r3, 60(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testFloat2:
+; SMALL-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C10@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C10@l(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lfsx f0, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r3, r1, 60
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    xscvdpsxws f0, f0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stfiwx f0, 0, r3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwz r3, 60(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testFloat2:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lfs f0, 0(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    fctiwz f0, f0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stfd f0, 56(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwa r3, 60(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    clrldi r3, r3, 32
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testFloat2:
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C10@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 48(r1) # 8-byte Folded Spill
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 48(r1) # 8-byte Folded Reload
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C10@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lfs f0, 0(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    fctiwz f0, f0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stfd f0, 56(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwa r3, 60(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    clrldi r3, r3, 32
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+entry:
+  %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @tlv_float)
+  %value = load float, ptr %tlv_addr, align 4
+  %conv = fptosi float %value to i32
+  ret i32 %conv
+}
+
+define void @testDouble1(double noundef %x) {
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testDouble1:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C11(r2) # target-flags(ppc-tlsld) @internal_tlv_double
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stfdx f1, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testDouble1:
+; SMALL-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C11@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C11@l(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stfdx f1, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testDouble1:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C13(r2) # target-flags(ppc-tlsld) @internal_tlv_double
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stxsdx f1, 0, r3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testDouble1:
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C13@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C13@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stxsdx f1, 0, r3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+entry:
+  %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @internal_tlv_double)
+  store double %x, ptr %tlv_addr, align 8
+  ret void
+}
+
+define i32 @testDouble2() {
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testDouble2:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C11(r2) # target-flags(ppc-tlsld) @internal_tlv_double
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lfdx f0, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r3, r1, 60
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    xscvdpsxws f0, f0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stfiwx f0, 0, r3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwz r3, 60(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testDouble2:
+; SMALL-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C11@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C11@l(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lfdx f0, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r3, r1, 60
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    xscvdpsxws f0, f0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stfiwx f0, 0, r3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwz r3, 60(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testDouble2:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C13(r2) # target-flags(ppc-tlsld) @internal_tlv_double
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lfdx f0, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    xscvdpsxws f0, f0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r3, r1, 52
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stfiwx f0, 0, r3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwz r3, 52(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    clrldi r3, r3, 32
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testDouble2:
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -80(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 96(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C13@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C13@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lfdx f0, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    xscvdpsxws f0, f0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r3, r1, 68
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stfiwx f0, 0, r3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwz r3, 68(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    clrldi r3, r3, 32
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 80
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+entry:
+  %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @internal_tlv_double)
+  %value = load double, ptr %tlv_addr, align 8
+  %conv = fptosi double %value to i32
+  ret i32 %conv
+}
-- 
GitLab


From a118769ec8dc94b8332fe50c9240fecc8065a417 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Sat, 23 Mar 2024 20:56:45 -0400
Subject: [PATCH 058/404] [gn] port 3bc71c2abf

---
 llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
index efb324713cfe..d533e79e6374 100644
--- a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
@@ -65,6 +65,7 @@ write_cmake_config("lit_common_configured") {
     "SANITIZER_USE_STATIC_CXX_ABI_PYBOOL=False",
     "SANITIZER_USE_STATIC_LLVM_UNWINDER_PYBOOL=False",
     "COMPILER_RT_HAS_AARCH64_SME_PYBOOL=False",
+    "COMPILER_RT_DARWIN_LINKER_VERSION=",
     "COMPILER_RT_HAS_LLD_PYBOOL=True",
     "COMPILER_RT_HAS_GWP_ASAN_PYBOOL=False",
     "HAVE_RPC_XDR_H=0",
-- 
GitLab


From 8e698a1d8e6984e41d2fdf159393d01951d87c21 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Sun, 24 Mar 2024 00:56:59 +0000
Subject: [PATCH 059/404] [gn build] Port b68e2eba0bc8

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 3c8b80526773..ac111f902513 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -287,6 +287,7 @@ if (current_toolchain == default_toolchain) {
       "__algorithm/shift_right.h",
       "__algorithm/shuffle.h",
       "__algorithm/sift_down.h",
+      "__algorithm/simd_utils.h",
       "__algorithm/sort.h",
       "__algorithm/sort_heap.h",
       "__algorithm/stable_partition.h",
-- 
GitLab


From d7d2f7ca62400ed4a3f8f89062d2aeec61bd29d4 Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Sat, 23 Mar 2024 19:11:49 -0700
Subject: [PATCH 060/404] [BOLT] Emit intra-function control flow in YAMLBAT

Attach branch counters to YAML profile, covering intra-function control
flow.

Depends on: https://github.com/llvm/llvm-project/pull/86353

Test Plan: Updated bolt/test/X86/bolt-address-translation-yaml.test

Reviewers: rafaelauler, dcci, ayermolo, maksfb

Reviewed By: rafaelauler

Pull Request: https://github.com/llvm/llvm-project/pull/76911
---
 .../bolt/Profile/BoltAddressTranslation.h     |  5 ++
 bolt/lib/Profile/DataAggregator.cpp           | 46 +++++++++++++++++++
 .../X86/bolt-address-translation-yaml.test    | 14 +++++-
 3 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h
index f8c35f8066f7..51fdadce8085 100644
--- a/bolt/include/bolt/Profile/BoltAddressTranslation.h
+++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h
@@ -257,6 +257,11 @@ public:
         std::as_const(*this).getBBHashMap(FuncOutputAddress));
   }
 
+  /// Returns the number of basic blocks in a function.
+  size_t getNumBasicBlocks(uint64_t OutputAddress) const {
+    return NumBasicBlocksMap.at(OutputAddress);
+  }
+
 private:
   FuncHashesTy FuncHashes;
 };
diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index 37c637a44a0e..b0f1cccc3256 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -2310,6 +2310,52 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
       BP.Functions.emplace_back(
           YAMLProfileWriter::convert(Function, /*UseDFS=*/false));
     }
+
+    for (const auto &KV : NamesToBranches) {
+      const StringRef FuncName = KV.first;
+      const FuncBranchData &Branches = KV.second;
+      yaml::bolt::BinaryFunctionProfile YamlBF;
+      BinaryData *BD = BC.getBinaryDataByName(FuncName);
+      assert(BD);
+      uint64_t FuncAddress = BD->getAddress();
+      if (!BAT->isBATFunction(FuncAddress))
+        continue;
+      // Filter out cold fragments
+      if (!BD->getSectionName().equals(BC.getMainCodeSectionName()))
+        continue;
+      BinaryFunction *BF = BC.getBinaryFunctionAtAddress(FuncAddress);
+      assert(BF);
+      YamlBF.Name = FuncName.str();
+      YamlBF.Id = BF->getFunctionNumber();
+      YamlBF.Hash = BAT->getBFHash(FuncAddress);
+      YamlBF.ExecCount = BF->getKnownExecutionCount();
+      YamlBF.NumBasicBlocks = BAT->getNumBasicBlocks(FuncAddress);
+      const BoltAddressTranslation::BBHashMapTy &BlockMap =
+          BAT->getBBHashMap(FuncAddress);
+
+      auto addSuccProfile = [&](yaml::bolt::BinaryBasicBlockProfile &YamlBB,
+                                uint64_t SuccOffset, unsigned SuccDataIdx) {
+        const llvm::bolt::BranchInfo &BI = Branches.Data.at(SuccDataIdx);
+        yaml::bolt::SuccessorInfo SI;
+        SI.Index = BlockMap.getBBIndex(SuccOffset);
+        SI.Count = BI.Branches;
+        SI.Mispreds = BI.Mispreds;
+        YamlBB.Successors.emplace_back(SI);
+      };
+
+      for (const auto &[FromOffset, SuccKV] : Branches.IntraIndex) {
+        yaml::bolt::BinaryBasicBlockProfile YamlBB;
+        if (!BlockMap.isInputBlock(FromOffset))
+          continue;
+        YamlBB.Index = BlockMap.getBBIndex(FromOffset);
+        YamlBB.Hash = BlockMap.getBBHash(FromOffset);
+        for (const auto &[SuccOffset, SuccDataIdx] : SuccKV)
+          addSuccProfile(YamlBB, SuccOffset, SuccDataIdx);
+        if (YamlBB.ExecCount || !YamlBB.Successors.empty())
+          YamlBF.Blocks.emplace_back(YamlBB);
+      }
+      BP.Functions.emplace_back(YamlBF);
+    }
   }
 
   // Write the profile.
diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test
index ee54a90a9f2a..be3bcb2a0c6d 100644
--- a/bolt/test/X86/bolt-address-translation-yaml.test
+++ b/bolt/test/X86/bolt-address-translation-yaml.test
@@ -25,6 +25,7 @@ READ-BAT-CHECK: BOLT-INFO: Parsed 5 BAT entries
 READ-BAT-CHECK: PERF2BOLT: read 79 aggregated LBR entries
 
 YAML-BAT-CHECK:      functions:
+# Function not covered by BAT - has insns in basic block
 YAML-BAT-CHECK:      - name:    main
 YAML-BAT-CHECK-NEXT:   fid:     2
 YAML-BAT-CHECK-NEXT:   hash:    0x9895746D48B2C876
@@ -35,6 +36,17 @@ YAML-BAT-CHECK-NEXT:   - bid:   0
 YAML-BAT-CHECK-NEXT:     insns: 26
 YAML-BAT-CHECK-NEXT:     hash:  0xA900AE79CFD40000
 YAML-BAT-CHECK-NEXT:     succ:  [ { bid: 3, cnt: 0 }, { bid: 1, cnt: 0 } ]
+# Function covered by BAT - doesn't have insns in basic block
+YAML-BAT-CHECK:      - name:    usqrt
+YAML-BAT-CHECK-NEXT:   fid:     [[#]]
+YAML-BAT-CHECK-NEXT:   hash:    0x99E67ED32A203023
+YAML-BAT-CHECK-NEXT:   exec:    21
+YAML-BAT-CHECK-NEXT:   nblocks: 5
+YAML-BAT-CHECK-NEXT:   blocks:
+YAML-BAT-CHECK:        - bid:   1
+YAML-BAT-CHECK-NEXT:       insns: [[#]]
+YAML-BAT-CHECK-NEXT:       hash:  0xD70DC695320E0010
+YAML-BAT-CHECK-NEXT:       succ:  {{.*}} { bid: 2, cnt: [[#]] }
 
 CHECK-BOLT-YAML:      pre-processing profile using YAML profile reader
-CHECK-BOLT-YAML-NEXT: 1 out of 16 functions in the binary (6.2%) have non-empty execution profile
+CHECK-BOLT-YAML-NEXT: 5 out of 16 functions in the binary (31.2%) have non-empty execution profile
-- 
GitLab


From 7c9b5228da94a44f5e3948814d896de537d162bb Mon Sep 17 00:00:00 2001
From: Owen Anderson <resistor@mac.com>
Date: Sat, 23 Mar 2024 21:49:29 -0500
Subject: [PATCH 061/404] Only check assertions that were meant to apply to the
 normal case of non-splat vector SREM expansion when we aren't hitting the
 special case. (#86238)

Fixes https://github.com/llvm/llvm-project/issues/84830
Introduced in https://github.com/llvm/llvm-project/pull/82706
---
 llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp | 10 +++++-----
 llvm/test/CodeGen/AArch64/srem-vec-crash.ll      | 15 +++++++++++++++
 2 files changed, 20 insertions(+), 5 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/srem-vec-crash.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index da29b1d5b312..8be03b66e155 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -6916,6 +6916,11 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
     // Q = floor((2 * A) / (2^K))
     APInt Q = (2 * A).udiv(APInt::getOneBitSet(W, K));
 
+    assert(APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) &&
+           "We are expecting that A is always less than all-ones for SVT");
+    assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&
+           "We are expecting that K is always less than all-ones for ShSVT");
+
     // If D was a power of two, apply the alternate constant derivation.
     if (D0.isOne()) {
       // A = 2^(W-1)
@@ -6924,11 +6929,6 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
       Q = APInt::getAllOnes(W - K).zext(W);
     }
 
-    assert(APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) &&
-           "We are expecting that A is always less than all-ones for SVT");
-    assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&
-           "We are expecting that K is always less than all-ones for ShSVT");
-
     // If the divisor is 1 the result can be constant-folded. Likewise, we
     // don't care about INT_MIN lanes, those can be set to undef if appropriate.
     if (D.isOne()) {
diff --git a/llvm/test/CodeGen/AArch64/srem-vec-crash.ll b/llvm/test/CodeGen/AArch64/srem-vec-crash.ll
new file mode 100644
index 000000000000..0fce8de30d4d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/srem-vec-crash.ll
@@ -0,0 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=aarch64-unknown-unknown < %s | FileCheck %s
+
+define i32 @pr84830(i1 %arg) {
+; CHECK-LABEL: pr84830:
+; CHECK:       // %bb.0: // %bb
+; CHECK-NEXT:    mov w0, #1 // =0x1
+; CHECK-NEXT:    ret
+bb:
+  %new0 = srem i1 %arg, true
+  %last = zext i1 %new0 to i32
+  %i = icmp ne i32 %last, 0
+  %i1 = select i1 %i, i32 0, i32 1
+  ret i32 %i1
+}
-- 
GitLab


From 74799f424063a2d751e0f9ea698db1f4efd0d8b2 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Sat, 23 Mar 2024 19:50:15 -0700
Subject: [PATCH 062/404] [memprof] Add call stack IDs to IndexedAllocationInfo
 (#85888)

The indexed MemProf file has a huge amount of redundancy.  In a large
internal application, 82% of call stacks, stored in
IndexedAllocationInfo::CallStack, are duplicates.

We should work toward deduplicating call stacks by referring to them
with unique IDs with actual call stacks stored in a separate data
structure, much like we refer to memprof::Frame with memprof::FrameId.

At the same time, we need to facilitate a graceful transition from the
current version of the MemProf format to the next.  We should be able
to read (but not write) the current version of the MemProf file even
after we move onto the next one.

With those goals in mind, I propose to have an integer ID next to
CallStack in IndexedAllocationInfo to refer to a call stack in a
succinct manner.  We'll gradually increase the areas of the compiler
where IDs and call stacks have one-to-one correspondence and
eventually remove the existing CallStack field.

This patch adds call stack ID, named CSId, to IndexedAllocationInfo
and teaches the raw profile reader to compute unique call stack IDs
and store them in the new field.  It does not introduce any user of
the call stack IDs yet, except in verifyFunctionProfileData.
---
 llvm/include/llvm/ProfileData/MemProf.h      | 23 ++++++++++++++++--
 llvm/lib/ProfileData/MemProf.cpp             | 25 ++++++++++++++++++++
 llvm/lib/ProfileData/RawMemProfReader.cpp    |  6 ++++-
 llvm/unittests/ProfileData/InstrProfTest.cpp |  3 ++-
 llvm/unittests/ProfileData/MemProfTest.cpp   |  7 ++++--
 5 files changed, 58 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h
index 37c19094bc2a..75ea0e49d453 100644
--- a/llvm/include/llvm/ProfileData/MemProf.h
+++ b/llvm/include/llvm/ProfileData/MemProf.h
@@ -1,6 +1,7 @@
 #ifndef LLVM_PROFILEDATA_MEMPROF_H_
 #define LLVM_PROFILEDATA_MEMPROF_H_
 
+#include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/IR/GlobalValue.h"
@@ -252,18 +253,26 @@ struct Frame {
   }
 };
 
+// A type representing the index into the table of call stacks.
+using CallStackId = uint64_t;
+
 // Holds allocation information in a space efficient format where frames are
 // represented using unique identifiers.
 struct IndexedAllocationInfo {
   // The dynamic calling context for the allocation in bottom-up (leaf-to-root)
   // order. Frame contents are stored out-of-line.
+  // TODO: Remove once we fully transition to CSId.
   llvm::SmallVector<FrameId> CallStack;
+  // Conceptually the same as above.  We are going to keep both CallStack and
+  // CallStackId while we are transitioning from CallStack to CallStackId.
+  CallStackId CSId = 0;
   // The statistics obtained from the runtime for the allocation.
   PortableMemInfoBlock Info;
 
   IndexedAllocationInfo() = default;
-  IndexedAllocationInfo(ArrayRef<FrameId> CS, const MemInfoBlock &MB)
-      : CallStack(CS.begin(), CS.end()), Info(MB) {}
+  IndexedAllocationInfo(ArrayRef<FrameId> CS, CallStackId CSId,
+                        const MemInfoBlock &MB)
+      : CallStack(CS.begin(), CS.end()), CSId(CSId), Info(MB) {}
 
   // Returns the size in bytes when this allocation info struct is serialized.
   size_t serializedSize() const {
@@ -622,6 +631,16 @@ public:
     return Frame::deserialize(D);
   }
 };
+
+// Compute a CallStackId for a given call stack.
+CallStackId hashCallStack(ArrayRef<FrameId> CS);
+
+// Verify that each CallStackId is computed with hashCallStack.  This function
+// is intended to help transition from CallStack to CSId in
+// IndexedAllocationInfo.
+void verifyFunctionProfileData(
+    const llvm::MapVector<GlobalValue::GUID, IndexedMemProfRecord>
+        &FunctionProfileData);
 } // namespace memprof
 } // namespace llvm
 
diff --git a/llvm/lib/ProfileData/MemProf.cpp b/llvm/lib/ProfileData/MemProf.cpp
index 0461f0e9f840..bffa4ed19be8 100644
--- a/llvm/lib/ProfileData/MemProf.cpp
+++ b/llvm/lib/ProfileData/MemProf.cpp
@@ -3,8 +3,10 @@
 #include "llvm/IR/Function.h"
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/ProfileData/SampleProf.h"
+#include "llvm/Support/BLAKE3.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/EndianStream.h"
+#include "llvm/Support/HashBuilder.h"
 
 namespace llvm {
 namespace memprof {
@@ -117,5 +119,28 @@ Expected<MemProfSchema> readMemProfSchema(const unsigned char *&Buffer) {
   return Result;
 }
 
+CallStackId hashCallStack(ArrayRef<FrameId> CS) {
+  llvm::HashBuilder<llvm::TruncatedBLAKE3<8>, llvm::endianness::little>
+      HashBuilder;
+  for (FrameId F : CS)
+    HashBuilder.add(F);
+  llvm::BLAKE3Result<8> Hash = HashBuilder.final();
+  CallStackId CSId;
+  std::memcpy(&CSId, Hash.data(), sizeof(Hash));
+  return CSId;
+}
+
+void verifyFunctionProfileData(
+    const llvm::MapVector<GlobalValue::GUID, IndexedMemProfRecord>
+        &FunctionProfileData) {
+  for (const auto &[GUID, Record] : FunctionProfileData) {
+    (void)GUID;
+    for (const auto &AS : Record.AllocSites) {
+      assert(AS.CSId == hashCallStack(AS.CallStack));
+      (void)AS;
+    }
+  }
+}
+
 } // namespace memprof
 } // namespace llvm
diff --git a/llvm/lib/ProfileData/RawMemProfReader.cpp b/llvm/lib/ProfileData/RawMemProfReader.cpp
index 60c37c417aa0..5dc1ff897815 100644
--- a/llvm/lib/ProfileData/RawMemProfReader.cpp
+++ b/llvm/lib/ProfileData/RawMemProfReader.cpp
@@ -446,6 +446,8 @@ Error RawMemProfReader::mapRawProfileToRecords() {
       Callstack.append(Frames.begin(), Frames.end());
     }
 
+    CallStackId CSId = hashCallStack(Callstack);
+
     // We attach the memprof record to each function bottom-up including the
     // first non-inline frame.
     for (size_t I = 0; /*Break out using the condition below*/; I++) {
@@ -453,7 +455,7 @@ Error RawMemProfReader::mapRawProfileToRecords() {
       auto Result =
           FunctionProfileData.insert({F.Function, IndexedMemProfRecord()});
       IndexedMemProfRecord &Record = Result.first->second;
-      Record.AllocSites.emplace_back(Callstack, Entry.second);
+      Record.AllocSites.emplace_back(Callstack, CSId, Entry.second);
 
       if (!F.IsInlineFrame)
         break;
@@ -471,6 +473,8 @@ Error RawMemProfReader::mapRawProfileToRecords() {
     }
   }
 
+  verifyFunctionProfileData(FunctionProfileData);
+
   return Error::success();
 }
 
diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp
index cd4552a039b3..c9323420bda7 100644
--- a/llvm/unittests/ProfileData/InstrProfTest.cpp
+++ b/llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -366,7 +366,8 @@ IndexedMemProfRecord makeRecord(
     const MemInfoBlock &Block = MemInfoBlock()) {
   llvm::memprof::IndexedMemProfRecord MR;
   for (const auto &Frames : AllocFrames)
-    MR.AllocSites.emplace_back(Frames, Block);
+    MR.AllocSites.emplace_back(Frames, llvm::memprof::hashCallStack(Frames),
+                               Block);
   for (const auto &Frames : CallSiteFrames)
     MR.CallSites.push_back(Frames);
   return MR;
diff --git a/llvm/unittests/ProfileData/MemProfTest.cpp b/llvm/unittests/ProfileData/MemProfTest.cpp
index f5e4a4aff2ed..1cca44e9b037 100644
--- a/llvm/unittests/ProfileData/MemProfTest.cpp
+++ b/llvm/unittests/ProfileData/MemProfTest.cpp
@@ -280,7 +280,8 @@ TEST(MemProf, RecordSerializationRoundTrip) {
   IndexedMemProfRecord Record;
   for (const auto &ACS : AllocCallStacks) {
     // Use the same info block for both allocation sites.
-    Record.AllocSites.emplace_back(ACS, Info);
+    Record.AllocSites.emplace_back(ACS, llvm::memprof::hashCallStack(ACS),
+                                   Info);
   }
   Record.CallSites.assign(CallSites);
 
@@ -376,7 +377,9 @@ TEST(MemProf, BaseMemProfReader) {
   Block.AllocCount = 1U, Block.TotalAccessDensity = 4,
   Block.TotalLifetime = 200001;
   std::array<FrameId, 2> CallStack{F1.hash(), F2.hash()};
-  FakeRecord.AllocSites.emplace_back(/*CS=*/CallStack, /*MB=*/Block);
+  FakeRecord.AllocSites.emplace_back(
+      /*CS=*/CallStack, /*CSId=*/llvm::memprof::hashCallStack(CallStack),
+      /*MB=*/Block);
   ProfData.insert({F1.hash(), FakeRecord});
 
   MemProfReader Reader(FrameIdMap, ProfData);
-- 
GitLab


From a45e58af1b381cf3c0374332386b8291ec5310f4 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Sun, 24 Mar 2024 12:48:19 +0900
Subject: [PATCH 063/404] [mlir][bufferization] Add `BufferViewFlowOpInterface`
 (#78718)

This commit adds the `BufferViewFlowOpInterface` to the bufferization
dialect. This interface can be implemented by ops that operate on
buffers to indicate that a buffer op result and/or region entry block
argument may be the same buffer as a buffer operand (or a view thereof).
This interface is queried by the `BufferViewFlowAnalysis`.

The new interface has two interface methods:
* `populateDependencies`: Implementations use the provided callback to
declare dependencies between operands and op results/region entry block
arguments. E.g., for `%r = arith.select %c, %m1, %m2 : memref<5xf32>`,
the interface implementation should declare two dependencies: %m1 -> %r
and %m2 -> %r.
* `mayBeTerminalBuffer`: An SSA value is a terminal buffer if the buffer
view flow analysis stops at the specified value. E.g., because the value
is a newly allocated buffer or because no further information is
available about the origin of the buffer.

Ops that implement the `RegionBranchOpInterface` or `BranchOpInterface`
do not have to implement the `BufferViewFlowOpInterface`. The buffer
dependencies can be inferred from those two interfaces.

This commit makes the `BufferViewFlowAnalysis` more accurate. For
unknown ops, it conservatively used to declare all combinations of
operands and op results/region entry block arguments as dependencies
(false positives). This is no longer the case. While the analysis is
still a "maybe" analysis with false positives (e.g., when analyzing ops
such as `arith.select` or `scf.if` where the taken branch is not known
at compile time), results and region entry block arguments of unknown
ops are now marked as terminal buffers.

This commit addresses a TODO in `BufferViewFlowAnalysis.cpp`:
```
// TODO: We should have an op interface instead of a hard-coded list of
// interfaces/ops.
```
It is no longer needed to hard-code ops.
---
 .../BufferViewFlowOpInterfaceImpl.h           | 20 +++++
 .../IR/BufferViewFlowOpInterface.h            | 27 +++++++
 .../IR/BufferViewFlowOpInterface.td           | 73 +++++++++++++++++++
 .../Dialect/Bufferization/IR/CMakeLists.txt   |  1 +
 .../Transforms/BufferViewFlowAnalysis.h       |  6 ++
 .../BufferViewFlowOpInterfaceImpl.h           | 20 +++++
 mlir/include/mlir/InitAllDialects.h           |  4 +
 .../BufferViewFlowOpInterfaceImpl.cpp         | 44 +++++++++++
 .../Dialect/Arith/Transforms/CMakeLists.txt   |  1 +
 .../IR/BufferViewFlowOpInterface.cpp          | 18 +++++
 .../Dialect/Bufferization/IR/CMakeLists.txt   |  1 +
 .../Transforms/BufferViewFlowAnalysis.cpp     | 72 ++++++++++++++----
 .../BufferViewFlowOpInterfaceImpl.cpp         | 48 ++++++++++++
 .../Dialect/MemRef/Transforms/CMakeLists.txt  |  2 +
 .../llvm-project-overlay/mlir/BUILD.bazel     | 40 ++++++++++
 15 files changed, 363 insertions(+), 14 deletions(-)
 create mode 100644 mlir/include/mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h
 create mode 100644 mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h
 create mode 100644 mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td
 create mode 100644 mlir/include/mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h
 create mode 100644 mlir/lib/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.cpp
 create mode 100644 mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp
 create mode 100644 mlir/lib/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.cpp

diff --git a/mlir/include/mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h b/mlir/include/mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h
new file mode 100644
index 000000000000..a2b3a9bb655b
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h
@@ -0,0 +1,20 @@
+//===- BufferViewFlowOpInterfaceImpl.h - Buffer View Analysis ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_ARITH_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H
+#define MLIR_DIALECT_ARITH_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H
+
+namespace mlir {
+class DialectRegistry;
+
+namespace arith {
+void registerBufferViewFlowOpInterfaceExternalModels(DialectRegistry &registry);
+} // namespace arith
+} // namespace mlir
+
+#endif // MLIR_DIALECT_ARITH_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H
diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h b/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h
new file mode 100644
index 000000000000..84e67fe72b62
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h
@@ -0,0 +1,27 @@
+//===- BufferViewFlowOpInterface.h - Buffer View Flow Analysis --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_BUFFERIZATION_IR_BUFFERVIEWFLOWOPINTERFACE_H_
+#define MLIR_DIALECT_BUFFERIZATION_IR_BUFFERVIEWFLOWOPINTERFACE_H_
+
+#include "mlir/IR/OpDefinition.h"
+#include "mlir/Support/LLVM.h"
+
+namespace mlir {
+class ValueRange;
+
+namespace bufferization {
+
+using RegisterDependenciesFn = std::function<void(ValueRange, ValueRange)>;
+
+} // namespace bufferization
+} // namespace mlir
+
+#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h.inc"
+
+#endif // MLIR_DIALECT_BUFFERIZATION_IR_BUFFERVIEWFLOWOPINTERFACE_H_
diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td
new file mode 100644
index 000000000000..58885d742266
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td
@@ -0,0 +1,73 @@
+//===-- BufferViewFlowOpInterface.td - Buffer View Flow ----*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef BUFFER_VIEW_FLOW_OP_INTERFACE
+#define BUFFER_VIEW_FLOW_OP_INTERFACE
+
+include "mlir/IR/OpBase.td"
+
+def BufferViewFlowOpInterface :
+    OpInterface<"BufferViewFlowOpInterface"> {
+  let description = [{
+    An op interface for the buffer view flow analysis. This interface describes
+    buffer dependencies between operands and op results/region entry block
+    arguments.
+  }];
+  let cppNamespace = "::mlir::bufferization";
+  let methods = [
+      InterfaceMethod<
+        /*desc=*/[{
+          Populate buffer dependencies between operands and op results/region
+          entry block arguments.
+
+          Implementations should register dependencies between an operand ("X")
+          and an op result/region entry block argument ("Y") if Y may depend
+          on X. Y depends on X if Y and X are the same buffer or if Y is a
+          subview of X.
+
+          Example:
+          ```
+          %r = arith.select %c, %m1, %m2 : memref<5xf32>
+          ```
+          In the above example, %0 may depend on %m1 or %m2 and a correct
+          interface implementation should call:
+          - "registerDependenciesFn(%m1, %r)".
+          - "registerDependenciesFn(%m2, %r)"
+        }],
+        /*retType=*/"void",
+        /*methodName=*/"populateDependencies",
+        /*args=*/(ins
+            "::mlir::bufferization::RegisterDependenciesFn"
+                :$registerDependenciesFn)
+      >,
+      InterfaceMethod<
+        /*desc=*/[{
+          Return "true" if the given value may be a terminal buffer. A buffer
+          value is "terminal" if it cannot be traced back any further in the
+          buffer view flow analysis.
+
+          Examples: A buffer could be terminal because:
+          - it is a newly allocated buffer (e.g., "memref.alloc"),
+          - or: because there is not enough compile-time information available
+            to make a definite decision (e.g., "memref.realloc" may reallocate
+            but we do not know for sure; another example are call ops where we
+            would have to analyze the body of the callee).
+
+          Implementations can assume that the given SSA value is an OpResult of
+          this operation or a region entry block argument of this operation.
+        }],
+        /*retType=*/"bool",
+        /*methodName=*/"mayBeTerminalBuffer",
+        /*args=*/(ins "Value":$value),
+        /*methodBody=*/"",
+        /*defaultImplementation=*/"return false;"
+      >,
+  ];
+}
+
+#endif  // BUFFER_VIEW_FLOW_OP_INTERFACE
diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/Bufferization/IR/CMakeLists.txt
index 31a553f9a32f..13a5bc370a4f 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/CMakeLists.txt
@@ -3,6 +3,7 @@ add_mlir_doc(BufferizationOps BufferizationOps Dialects/ -gen-dialect-doc)
 add_mlir_interface(AllocationOpInterface)
 add_mlir_interface(BufferDeallocationOpInterface)
 add_mlir_interface(BufferizableOpInterface)
+add_mlir_interface(BufferViewFlowOpInterface)
 
 set(LLVM_TARGET_DEFINITIONS BufferizationEnums.td)
 mlir_tablegen(BufferizationEnums.h.inc -gen-enum-decls)
diff --git a/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h b/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h
index 24825db69f90..9e43265c5dfe 100644
--- a/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h
+++ b/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h
@@ -63,6 +63,9 @@ public:
   /// results have to be changed.
   void rename(Value from, Value to);
 
+  /// Returns "true" if the given value may be a terminal.
+  bool mayBeTerminalBuffer(Value value) const;
+
 private:
   /// This function constructs a mapping from values to its immediate
   /// dependencies.
@@ -70,6 +73,9 @@ private:
 
   /// Maps values to all immediate dependencies this value can have.
   ValueMapT dependencies;
+
+  /// A set of all SSA values that may be terminal buffers.
+  DenseSet<Value> terminals;
 };
 
 } // namespace mlir
diff --git a/mlir/include/mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h b/mlir/include/mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h
new file mode 100644
index 000000000000..714518a21e97
--- /dev/null
+++ b/mlir/include/mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h
@@ -0,0 +1,20 @@
+//===- BufferViewFlowOpInterfaceImpl.h - Buffer View Analysis ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_MEMREF_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H
+#define MLIR_DIALECT_MEMREF_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H
+
+namespace mlir {
+class DialectRegistry;
+
+namespace memref {
+void registerBufferViewFlowOpInterfaceExternalModels(DialectRegistry &registry);
+} // namespace memref
+} // namespace mlir
+
+#endif // MLIR_DIALECT_MEMREF_TRANSFORMS_BUFFERVIEWFLOWOPINTERFACEIMPL_H
diff --git a/mlir/include/mlir/InitAllDialects.h b/mlir/include/mlir/InitAllDialects.h
index 9bbf12d13254..c558dc53cc7f 100644
--- a/mlir/include/mlir/InitAllDialects.h
+++ b/mlir/include/mlir/InitAllDialects.h
@@ -21,6 +21,7 @@
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Arith/IR/ValueBoundsOpInterfaceImpl.h"
 #include "mlir/Dialect/Arith/Transforms/BufferDeallocationOpInterfaceImpl.h"
+#include "mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h"
 #include "mlir/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.h"
 #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
 #include "mlir/Dialect/ArmSME/IR/ArmSME.h"
@@ -52,6 +53,7 @@
 #include "mlir/Dialect/MemRef/IR/MemRefMemorySlot.h"
 #include "mlir/Dialect/MemRef/IR/ValueBoundsOpInterfaceImpl.h"
 #include "mlir/Dialect/MemRef/Transforms/AllocationOpInterfaceImpl.h"
+#include "mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h"
 #include "mlir/Dialect/MemRef/Transforms/RuntimeOpVerification.h"
 #include "mlir/Dialect/Mesh/IR/MeshDialect.h"
 #include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
@@ -148,6 +150,7 @@ inline void registerAllDialects(DialectRegistry &registry) {
   affine::registerValueBoundsOpInterfaceExternalModels(registry);
   arith::registerBufferDeallocationOpInterfaceExternalModels(registry);
   arith::registerBufferizableOpInterfaceExternalModels(registry);
+  arith::registerBufferViewFlowOpInterfaceExternalModels(registry);
   arith::registerValueBoundsOpInterfaceExternalModels(registry);
   bufferization::func_ext::registerBufferizableOpInterfaceExternalModels(
       registry);
@@ -157,6 +160,7 @@ inline void registerAllDialects(DialectRegistry &registry) {
   gpu::registerBufferDeallocationOpInterfaceExternalModels(registry);
   linalg::registerAllDialectInterfaceImplementations(registry);
   memref::registerAllocationOpInterfaceExternalModels(registry);
+  memref::registerBufferViewFlowOpInterfaceExternalModels(registry);
   memref::registerRuntimeVerifiableOpInterfaceExternalModels(registry);
   memref::registerValueBoundsOpInterfaceExternalModels(registry);
   memref::registerMemorySlotExternalModels(registry);
diff --git a/mlir/lib/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.cpp
new file mode 100644
index 000000000000..9df9df86b64f
--- /dev/null
+++ b/mlir/lib/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.cpp
@@ -0,0 +1,44 @@
+//===- BufferViewFlowOpInterfaceImpl.cpp - Buffer View Flow Analysis ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Arith/Transforms/BufferViewFlowOpInterfaceImpl.h"
+
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h"
+
+using namespace mlir;
+using namespace mlir::bufferization;
+
+namespace mlir {
+namespace arith {
+namespace {
+
+struct SelectOpInterface
+    : public BufferViewFlowOpInterface::ExternalModel<SelectOpInterface,
+                                                      SelectOp> {
+  void
+  populateDependencies(Operation *op,
+                       RegisterDependenciesFn registerDependenciesFn) const {
+    auto selectOp = cast<SelectOp>(op);
+
+    // Either one of the true/false value may be selected at runtime.
+    registerDependenciesFn(selectOp.getTrueValue(), selectOp.getResult());
+    registerDependenciesFn(selectOp.getFalseValue(), selectOp.getResult());
+  }
+};
+
+} // namespace
+} // namespace arith
+} // namespace mlir
+
+void arith::registerBufferViewFlowOpInterfaceExternalModels(
+    DialectRegistry &registry) {
+  registry.addExtension(+[](MLIRContext *ctx, arith::ArithDialect *dialect) {
+    SelectOp::attachInterface<SelectOpInterface>(*ctx);
+  });
+}
diff --git a/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt
index 02240601bcd3..12659eaba1fa 100644
--- a/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Arith/Transforms/CMakeLists.txt
@@ -2,6 +2,7 @@ add_mlir_dialect_library(MLIRArithTransforms
   BufferDeallocationOpInterfaceImpl.cpp
   BufferizableOpInterfaceImpl.cpp
   Bufferize.cpp
+  BufferViewFlowOpInterfaceImpl.cpp
   EmulateUnsupportedFloats.cpp
   EmulateWideInt.cpp
   EmulateNarrowType.cpp
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp
new file mode 100644
index 000000000000..ea726a4bfc3f
--- /dev/null
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp
@@ -0,0 +1,18 @@
+//===- BufferViewFlowOpInterface.cpp - Buffer View Flow Analysis ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h"
+#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
+
+namespace mlir {
+namespace bufferization {
+
+#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp.inc"
+
+} // namespace bufferization
+} // namespace mlir
diff --git a/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt b/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt
index 9895db9d93ce..63dcc1eb233e 100644
--- a/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt
@@ -4,6 +4,7 @@ add_mlir_dialect_library(MLIRBufferizationDialect
   BufferDeallocationOpInterface.cpp
   BufferizationOps.cpp
   BufferizationDialect.cpp
+  BufferViewFlowOpInterface.cpp
   UnstructuredControlFlow.cpp
 
   ADDITIONAL_HEADER_DIRS
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp
index 88ef1b639fc5..9a36057425f3 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp
@@ -8,12 +8,16 @@
 
 #include "mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h"
 
+#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h"
+#include "mlir/Interfaces/CallInterfaces.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
+#include "mlir/Interfaces/FunctionInterfaces.h"
 #include "mlir/Interfaces/ViewLikeInterface.h"
 #include "llvm/ADT/SetOperations.h"
 #include "llvm/ADT/SetVector.h"
 
 using namespace mlir;
+using namespace mlir::bufferization;
 
 /// Constructs a new alias analysis using the op provided.
 BufferViewFlowAnalysis::BufferViewFlowAnalysis(Operation *op) { build(op); }
@@ -65,18 +69,44 @@ void BufferViewFlowAnalysis::rename(Value from, Value to) {
 void BufferViewFlowAnalysis::build(Operation *op) {
   // Registers all dependencies of the given values.
   auto registerDependencies = [&](ValueRange values, ValueRange dependencies) {
-    for (auto [value, dep] : llvm::zip(values, dependencies))
+    for (auto [value, dep] : llvm::zip_equal(values, dependencies))
       this->dependencies[value].insert(dep);
   };
 
+  // Mark all buffer results and buffer region entry block arguments of the
+  // given op as terminals.
+  auto populateTerminalValues = [&](Operation *op) {
+    for (Value v : op->getResults())
+      if (isa<BaseMemRefType>(v.getType()))
+        this->terminals.insert(v);
+    for (Region &r : op->getRegions())
+      for (BlockArgument v : r.getArguments())
+        if (isa<BaseMemRefType>(v.getType()))
+          this->terminals.insert(v);
+  };
+
   op->walk([&](Operation *op) {
-    // TODO: We should have an op interface instead of a hard-coded list of
-    // interfaces/ops.
+    // Query BufferViewFlowOpInterface. If the op does not implement that
+    // interface, try to infer the dependencies from other interfaces that the
+    // op may implement.
+    if (auto bufferViewFlowOp = dyn_cast<BufferViewFlowOpInterface>(op)) {
+      bufferViewFlowOp.populateDependencies(registerDependencies);
+      for (Value v : op->getResults())
+        if (isa<BaseMemRefType>(v.getType()) &&
+            bufferViewFlowOp.mayBeTerminalBuffer(v))
+          this->terminals.insert(v);
+      for (Region &r : op->getRegions())
+        for (BlockArgument v : r.getArguments())
+          if (isa<BaseMemRefType>(v.getType()) &&
+              bufferViewFlowOp.mayBeTerminalBuffer(v))
+            this->terminals.insert(v);
+      return WalkResult::advance();
+    }
 
     // Add additional dependencies created by view changes to the alias list.
     if (auto viewInterface = dyn_cast<ViewLikeOpInterface>(op)) {
-      dependencies[viewInterface.getViewSource()].insert(
-          viewInterface->getResult(0));
+      registerDependencies(viewInterface.getViewSource(),
+                           viewInterface->getResult(0));
       return WalkResult::advance();
     }
 
@@ -131,16 +161,30 @@ void BufferViewFlowAnalysis::build(Operation *op) {
       return WalkResult::advance();
     }
 
-    // Unknown op: Assume that all operands alias with all results.
-    for (Value operand : op->getOperands()) {
-      if (!isa<BaseMemRefType>(operand.getType()))
-        continue;
-      for (Value result : op->getResults()) {
-        if (!isa<BaseMemRefType>(result.getType()))
-          continue;
-        registerDependencies({operand}, {result});
-      }
+    // Region terminators are handled together with RegionBranchOpInterface.
+    if (isa<RegionBranchTerminatorOpInterface>(op))
+      return WalkResult::advance();
+
+    if (isa<CallOpInterface>(op)) {
+      // This is an intra-function analysis. We have no information about other
+      // functions. Conservatively assume that each operand may alias with each
+      // result. Also mark the results are terminals because the function could
+      // return newly allocated buffers.
+      populateTerminalValues(op);
+      for (Value operand : op->getOperands())
+        for (Value result : op->getResults())
+          registerDependencies({operand}, {result});
+      return WalkResult::advance();
     }
+
+    // We have no information about unknown ops.
+    populateTerminalValues(op);
+
     return WalkResult::advance();
   });
 }
+
+bool BufferViewFlowAnalysis::mayBeTerminalBuffer(Value value) const {
+  assert(isa<BaseMemRefType>(value.getType()) && "expected memref");
+  return terminals.contains(value);
+}
diff --git a/mlir/lib/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.cpp b/mlir/lib/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.cpp
new file mode 100644
index 000000000000..bbb269bd0016
--- /dev/null
+++ b/mlir/lib/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.cpp
@@ -0,0 +1,48 @@
+//===- BufferViewFlowOpInterfaceImpl.cpp - Buffer View Flow Analysis ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/MemRef/Transforms/BufferViewFlowOpInterfaceImpl.h"
+
+#include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h"
+#include "mlir/Dialect/MemRef/IR/MemRef.h"
+
+using namespace mlir;
+using namespace mlir::bufferization;
+
+namespace mlir {
+namespace memref {
+namespace {
+
+struct ReallocOpInterface
+    : public BufferViewFlowOpInterface::ExternalModel<ReallocOpInterface,
+                                                      ReallocOp> {
+  void
+  populateDependencies(Operation *op,
+                       RegisterDependenciesFn registerDependenciesFn) const {
+    auto reallocOp = cast<ReallocOp>(op);
+    // memref.realloc may return the source operand.
+    registerDependenciesFn(reallocOp.getSource(), reallocOp.getResult());
+  }
+
+  bool mayBeTerminalBuffer(Operation *op, Value value) const {
+    // The return value of memref.realloc is a terminal buffer because the op
+    // may return a newly allocated buffer.
+    return true;
+  }
+};
+
+} // namespace
+} // namespace memref
+} // namespace mlir
+
+void memref::registerBufferViewFlowOpInterfaceExternalModels(
+    DialectRegistry &registry) {
+  registry.addExtension(+[](MLIRContext *ctx, memref::MemRefDialect *dialect) {
+    ReallocOp::attachInterface<ReallocOpInterface>(*ctx);
+  });
+}
diff --git a/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt b/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt
index 08b7eab726eb..f150ac7ac2d6 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_mlir_dialect_library(MLIRMemRefTransforms
   AllocationOpInterfaceImpl.cpp
+  BufferViewFlowOpInterfaceImpl.cpp
   ComposeSubView.cpp
   ExpandOps.cpp
   ExpandRealloc.cpp
@@ -27,6 +28,7 @@ add_mlir_dialect_library(MLIRMemRefTransforms
   MLIRArithDialect
   MLIRArithTransforms
   MLIRBufferizationDialect
+  MLIRBufferizationTransforms
   MLIRDialectUtils
   MLIRFuncDialect
   MLIRGPUDialect
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 5b6e4678a05e..88b46bdb326c 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -10828,6 +10828,36 @@ gentbl_cc_library(
     ],
 )
 
+td_library(
+    name = "BufferViewFlowOpInterfaceTdFiles",
+    srcs = [
+        "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td",
+    ],
+    includes = ["include"],
+    deps = [
+        ":OpBaseTdFiles",
+    ],
+)
+
+gentbl_cc_library(
+    name = "BufferViewFlowOpInterfaceIncGen",
+    tbl_outs = [
+        (
+            ["-gen-op-interface-decls"],
+            "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h.inc",
+        ),
+        (
+            ["-gen-op-interface-defs"],
+            "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp.inc",
+        ),
+    ],
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td",
+    deps = [
+        ":BufferViewFlowOpInterfaceTdFiles",
+    ],
+)
+
 td_library(
     name = "SubsetOpInterfaceTdFiles",
     srcs = [
@@ -12977,6 +13007,8 @@ cc_library(
         ":ArithTransforms",
         ":ArithUtils",
         ":BufferizationDialect",
+        ":BufferizationInterfaces",
+        ":BufferizationTransforms",
         ":ControlFlowDialect",
         ":DialectUtils",
         ":FuncDialect",
@@ -13369,6 +13401,7 @@ td_library(
     includes = ["include"],
     deps = [
         ":AllocationOpInterfaceTdFiles",
+        ":BufferViewFlowOpInterfaceTdFiles",
         ":BufferizableOpInterfaceTdFiles",
         ":CopyOpInterfaceTdFiles",
         ":DestinationStyleOpInterfaceTdFiles",
@@ -13515,11 +13548,13 @@ cc_library(
     ],
     hdrs = [
         "include/mlir/Dialect/Bufferization/IR/BufferDeallocationOpInterface.h",
+        "include/mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.h",
         "include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h",
     ],
     includes = ["include"],
     deps = [
         ":BufferDeallocationOpInterfaceIncGen",
+        ":BufferViewFlowOpInterfaceIncGen",
         ":BufferizableOpInterfaceIncGen",
         ":BufferizationEnumsIncGen",
         ":IR",
@@ -13532,6 +13567,7 @@ cc_library(
     name = "BufferizationDialect",
     srcs = [
         "lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp",
+        "lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp",
         "lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp",
         "lib/Dialect/Bufferization/IR/BufferizationDialect.cpp",
         "lib/Dialect/Bufferization/IR/BufferizationOps.cpp",
@@ -13549,10 +13585,12 @@ cc_library(
         ":Analysis",
         ":ArithDialect",
         ":BufferDeallocationOpInterfaceIncGen",
+        ":BufferViewFlowOpInterfaceIncGen",
         ":BufferizableOpInterfaceIncGen",
         ":BufferizationBaseIncGen",
         ":BufferizationInterfaces",
         ":BufferizationOpsIncGen",
+        ":CallOpInterfaces",
         ":ControlFlowInterfaces",
         ":CopyOpInterface",
         ":DestinationStyleOpInterface",
@@ -13602,9 +13640,11 @@ cc_library(
         ":BufferizationDialect",
         ":BufferizationInterfaces",
         ":BufferizationPassIncGen",
+        ":CallOpInterfaces",
         ":ControlFlowDialect",
         ":ControlFlowInterfaces",
         ":FuncDialect",
+        ":FunctionInterfaces",
         ":IR",
         ":LoopLikeInterface",
         ":MemRefDialect",
-- 
GitLab


From 5d7fd6a04a6748936dece9d90481b2ba4ec97e53 Mon Sep 17 00:00:00 2001
From: yingopq <115543042+yingopq@users.noreply.github.com>
Date: Sun, 24 Mar 2024 14:35:42 +0800
Subject: [PATCH 064/404] [Mips] Restore wrong deletion of instruction 'and' in
 unsigned min/max processing. (#85902)

Fix #61881
---
 llvm/lib/Target/Mips/MipsExpandPseudo.cpp |  9 ++++
 llvm/test/CodeGen/Mips/atomic-min-max.ll  | 56 +++++++++++++++++++++++
 2 files changed, 65 insertions(+)

diff --git a/llvm/lib/Target/Mips/MipsExpandPseudo.cpp b/llvm/lib/Target/Mips/MipsExpandPseudo.cpp
index bded59439a73..c30129743a96 100644
--- a/llvm/lib/Target/Mips/MipsExpandPseudo.cpp
+++ b/llvm/lib/Target/Mips/MipsExpandPseudo.cpp
@@ -500,6 +500,15 @@ bool MipsExpandPseudo::expandAtomicBinOpSubword(
               .addReg(Incr, RegState::Kill)
               .addImm(ShiftImm);
         }
+      } else {
+        // and OldVal, OldVal, Mask
+        // and Incr, Incr, Mask
+        BuildMI(loopMBB, DL, TII->get(Mips::AND), OldVal)
+            .addReg(OldVal)
+            .addReg(Mask);
+        BuildMI(loopMBB, DL, TII->get(Mips::AND), Incr)
+            .addReg(Incr)
+            .addReg(Mask);
       }
     }
     // unsigned: sltu Scratch4, oldVal, Incr
diff --git a/llvm/test/CodeGen/Mips/atomic-min-max.ll b/llvm/test/CodeGen/Mips/atomic-min-max.ll
index bc3643f3947a..a96581bdb39a 100644
--- a/llvm/test/CodeGen/Mips/atomic-min-max.ll
+++ b/llvm/test/CodeGen/Mips/atomic-min-max.ll
@@ -2146,6 +2146,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS32-NEXT:  $BB6_1: # %entry
 ; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    and $2, $2, $8
+; MIPS32-NEXT:    and $7, $7, $8
 ; MIPS32-NEXT:    sltu $5, $2, $7
 ; MIPS32-NEXT:    move $3, $2
 ; MIPS32-NEXT:    movn $3, $7, $5
@@ -2186,6 +2188,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSEL-NEXT:  $BB6_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
+; MIPSEL-NEXT:    and $2, $2, $8
+; MIPSEL-NEXT:    and $7, $7, $8
 ; MIPSEL-NEXT:    sltu $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movn $3, $7, $5
@@ -2225,6 +2229,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSELR6-NEXT:  $BB6_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
+; MIPSELR6-NEXT:    and $2, $2, $8
+; MIPSELR6-NEXT:    and $7, $7, $8
 ; MIPSELR6-NEXT:    sltu $5, $2, $7
 ; MIPSELR6-NEXT:    seleqz $3, $2, $5
 ; MIPSELR6-NEXT:    selnez $5, $7, $5
@@ -2263,6 +2269,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMEL-NEXT:  $BB6_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
+; MMEL-NEXT:    and $2, $2, $8
+; MMEL-NEXT:    and $7, $7, $8
 ; MMEL-NEXT:    sltu $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movn $3, $7, $5
@@ -2300,6 +2308,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMELR6-NEXT:  $BB6_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
+; MMELR6-NEXT:    and $2, $2, $8
+; MMELR6-NEXT:    and $7, $7, $8
 ; MMELR6-NEXT:    sltu $5, $2, $7
 ; MMELR6-NEXT:    seleqz $3, $2, $5
 ; MMELR6-NEXT:    selnez $5, $7, $5
@@ -2417,6 +2427,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64EL-NEXT:  .LBB6_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
+; MIPS64EL-NEXT:    and $2, $2, $8
+; MIPS64EL-NEXT:    and $7, $7, $8
 ; MIPS64EL-NEXT:    sltu $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movn $3, $7, $5
@@ -2456,6 +2468,8 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB6_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
+; MIPS64ELR6-NEXT:    and $2, $2, $8
+; MIPS64ELR6-NEXT:    and $7, $7, $8
 ; MIPS64ELR6-NEXT:    sltu $5, $2, $7
 ; MIPS64ELR6-NEXT:    seleqz $3, $2, $5
 ; MIPS64ELR6-NEXT:    selnez $5, $7, $5
@@ -2655,6 +2669,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS32-NEXT:  $BB7_1: # %entry
 ; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    and $2, $2, $8
+; MIPS32-NEXT:    and $7, $7, $8
 ; MIPS32-NEXT:    sltu $5, $2, $7
 ; MIPS32-NEXT:    move $3, $2
 ; MIPS32-NEXT:    movz $3, $7, $5
@@ -2696,6 +2712,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSEL-NEXT:  $BB7_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
+; MIPSEL-NEXT:    and $2, $2, $8
+; MIPSEL-NEXT:    and $7, $7, $8
 ; MIPSEL-NEXT:    sltu $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movz $3, $7, $5
@@ -2735,6 +2753,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSELR6-NEXT:  $BB7_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
+; MIPSELR6-NEXT:    and $2, $2, $8
+; MIPSELR6-NEXT:    and $7, $7, $8
 ; MIPSELR6-NEXT:    sltu $5, $2, $7
 ; MIPSELR6-NEXT:    selnez $3, $2, $5
 ; MIPSELR6-NEXT:    seleqz $5, $7, $5
@@ -2773,6 +2793,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMEL-NEXT:  $BB7_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
+; MMEL-NEXT:    and $2, $2, $8
+; MMEL-NEXT:    and $7, $7, $8
 ; MMEL-NEXT:    sltu $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movz $3, $7, $5
@@ -2810,6 +2832,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMELR6-NEXT:  $BB7_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
+; MMELR6-NEXT:    and $2, $2, $8
+; MMELR6-NEXT:    and $7, $7, $8
 ; MMELR6-NEXT:    sltu $5, $2, $7
 ; MMELR6-NEXT:    selnez $3, $2, $5
 ; MMELR6-NEXT:    seleqz $5, $7, $5
@@ -2927,6 +2951,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64EL-NEXT:  .LBB7_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
+; MIPS64EL-NEXT:    and $2, $2, $8
+; MIPS64EL-NEXT:    and $7, $7, $8
 ; MIPS64EL-NEXT:    sltu $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movz $3, $7, $5
@@ -2966,6 +2992,8 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB7_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
+; MIPS64ELR6-NEXT:    and $2, $2, $8
+; MIPS64ELR6-NEXT:    and $7, $7, $8
 ; MIPS64ELR6-NEXT:    sltu $5, $2, $7
 ; MIPS64ELR6-NEXT:    selnez $3, $2, $5
 ; MIPS64ELR6-NEXT:    seleqz $5, $7, $5
@@ -4244,6 +4272,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS32-NEXT:  $BB10_1: # %entry
 ; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    and $2, $2, $8
+; MIPS32-NEXT:    and $7, $7, $8
 ; MIPS32-NEXT:    sltu $5, $2, $7
 ; MIPS32-NEXT:    move $3, $2
 ; MIPS32-NEXT:    movn $3, $7, $5
@@ -4284,6 +4314,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSEL-NEXT:  $BB10_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
+; MIPSEL-NEXT:    and $2, $2, $8
+; MIPSEL-NEXT:    and $7, $7, $8
 ; MIPSEL-NEXT:    sltu $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movn $3, $7, $5
@@ -4323,6 +4355,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSELR6-NEXT:  $BB10_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
+; MIPSELR6-NEXT:    and $2, $2, $8
+; MIPSELR6-NEXT:    and $7, $7, $8
 ; MIPSELR6-NEXT:    sltu $5, $2, $7
 ; MIPSELR6-NEXT:    seleqz $3, $2, $5
 ; MIPSELR6-NEXT:    selnez $5, $7, $5
@@ -4361,6 +4395,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMEL-NEXT:  $BB10_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
+; MMEL-NEXT:    and $2, $2, $8
+; MMEL-NEXT:    and $7, $7, $8
 ; MMEL-NEXT:    sltu $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movn $3, $7, $5
@@ -4398,6 +4434,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMELR6-NEXT:  $BB10_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
+; MMELR6-NEXT:    and $2, $2, $8
+; MMELR6-NEXT:    and $7, $7, $8
 ; MMELR6-NEXT:    sltu $5, $2, $7
 ; MMELR6-NEXT:    seleqz $3, $2, $5
 ; MMELR6-NEXT:    selnez $5, $7, $5
@@ -4515,6 +4553,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64EL-NEXT:  .LBB10_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
+; MIPS64EL-NEXT:    and $2, $2, $8
+; MIPS64EL-NEXT:    and $7, $7, $8
 ; MIPS64EL-NEXT:    sltu $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movn $3, $7, $5
@@ -4554,6 +4594,8 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB10_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
+; MIPS64ELR6-NEXT:    and $2, $2, $8
+; MIPS64ELR6-NEXT:    and $7, $7, $8
 ; MIPS64ELR6-NEXT:    sltu $5, $2, $7
 ; MIPS64ELR6-NEXT:    seleqz $3, $2, $5
 ; MIPS64ELR6-NEXT:    selnez $5, $7, $5
@@ -4753,6 +4795,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS32-NEXT:  $BB11_1: # %entry
 ; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    and $2, $2, $8
+; MIPS32-NEXT:    and $7, $7, $8
 ; MIPS32-NEXT:    sltu $5, $2, $7
 ; MIPS32-NEXT:    move $3, $2
 ; MIPS32-NEXT:    movz $3, $7, $5
@@ -4793,6 +4837,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSEL-NEXT:  $BB11_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
+; MIPSEL-NEXT:    and $2, $2, $8
+; MIPSEL-NEXT:    and $7, $7, $8
 ; MIPSEL-NEXT:    sltu $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movz $3, $7, $5
@@ -4832,6 +4878,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSELR6-NEXT:  $BB11_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
+; MIPSELR6-NEXT:    and $2, $2, $8
+; MIPSELR6-NEXT:    and $7, $7, $8
 ; MIPSELR6-NEXT:    sltu $5, $2, $7
 ; MIPSELR6-NEXT:    selnez $3, $2, $5
 ; MIPSELR6-NEXT:    seleqz $5, $7, $5
@@ -4870,6 +4918,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMEL-NEXT:  $BB11_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
+; MMEL-NEXT:    and $2, $2, $8
+; MMEL-NEXT:    and $7, $7, $8
 ; MMEL-NEXT:    sltu $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movz $3, $7, $5
@@ -4907,6 +4957,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMELR6-NEXT:  $BB11_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
+; MMELR6-NEXT:    and $2, $2, $8
+; MMELR6-NEXT:    and $7, $7, $8
 ; MMELR6-NEXT:    sltu $5, $2, $7
 ; MMELR6-NEXT:    selnez $3, $2, $5
 ; MMELR6-NEXT:    seleqz $5, $7, $5
@@ -5024,6 +5076,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64EL-NEXT:  .LBB11_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
+; MIPS64EL-NEXT:    and $2, $2, $8
+; MIPS64EL-NEXT:    and $7, $7, $8
 ; MIPS64EL-NEXT:    sltu $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movz $3, $7, $5
@@ -5063,6 +5117,8 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB11_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
+; MIPS64ELR6-NEXT:    and $2, $2, $8
+; MIPS64ELR6-NEXT:    and $7, $7, $8
 ; MIPS64ELR6-NEXT:    sltu $5, $2, $7
 ; MIPS64ELR6-NEXT:    selnez $3, $2, $5
 ; MIPS64ELR6-NEXT:    seleqz $5, $7, $5
-- 
GitLab


From 4acd84e7ccce6a2865f60cd2adc37a335d4f35ce Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Sun, 24 Mar 2024 01:22:48 -0700
Subject: [PATCH 065/404] Revert "[compiler-rt] Also consider SIGPROF as a
 synchronous signal" (#86416)

Reverting #85188 with follow up patches.

This reverts commit 362d26366d0175f01ffb6085eb747a6e40f01147.
This reverts commit c9bdeabdf4b46fbf1f6a9fcbf9cd61d460b18c08.
This reverts commit 6bc6e1ace9fa8453e164fa04b5d9acd5a77e089a.
This reverts commit 01fa550ff654d6724e6da54c877032baeddff14b.
This reverts commit ddcbab37ac0e5743a8d39be3dd48d967f4c85504.
---
 compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp | 4 +---
 compiler-rt/test/tsan/signal_errno.cpp               | 6 +++---
 compiler-rt/test/tsan/signal_reset.cpp               | 8 ++++----
 compiler-rt/test/tsan/signal_sync.cpp                | 4 ++--
 compiler-rt/test/tsan/signal_thread.cpp              | 4 ++--
 compiler-rt/test/tsan/signal_thread2.cpp             | 4 ++--
 6 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
index 810ce69663d0..8ffc703b05ea 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
@@ -126,7 +126,6 @@ const int SIGFPE = 8;
 const int SIGSEGV = 11;
 const int SIGPIPE = 13;
 const int SIGTERM = 15;
-const int SIGPROF = 27;
 #if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_NETBSD
 const int SIGBUS = 10;
 const int SIGSYS = 12;
@@ -2180,8 +2179,7 @@ void sighandler(int sig, __sanitizer_siginfo *info, void *ctx) {
     return;
   }
   // Don't mess with synchronous signals.
-  const bool sync = is_sync_signal(sctx, sig, info) ||
-                    (sig == SIGPROF && thr->is_inited && !thr->is_dead);
+  const bool sync = is_sync_signal(sctx, sig, info);
   if (sync ||
       // If we are in blocking function, we can safely process it now
       // (but check if we are in a recursive interceptor,
diff --git a/compiler-rt/test/tsan/signal_errno.cpp b/compiler-rt/test/tsan/signal_errno.cpp
index 99d4b6d84ca4..7e1fd4b0c5a5 100644
--- a/compiler-rt/test/tsan/signal_errno.cpp
+++ b/compiler-rt/test/tsan/signal_errno.cpp
@@ -18,7 +18,7 @@ static void MyHandler(int, siginfo_t *s, void *c) {
 
 static void* sendsignal(void *p) {
   barrier_wait(&barrier);
-  pthread_kill(mainth, SIGALRM);
+  pthread_kill(mainth, SIGPROF);
   return 0;
 }
 
@@ -37,7 +37,7 @@ int main() {
   mainth = pthread_self();
   struct sigaction act = {};
   act.sa_sigaction = &MyHandler;
-  sigaction(SIGALRM, &act, 0);
+  sigaction(SIGPROF, &act, 0);
   pthread_t th;
   pthread_create(&th, 0, sendsignal, 0);
   loop();
@@ -46,7 +46,7 @@ int main() {
 }
 
 // CHECK: WARNING: ThreadSanitizer: signal handler spoils errno
-// CHECK:   Signal 14 handler invoked at:
+// CHECK:   Signal 27 handler invoked at:
 // CHECK:     #0 MyHandler(int, {{(__)?}}siginfo{{(_t)?}}*, void*) {{.*}}signal_errno.cpp
 // CHECK:     main
 // CHECK: SUMMARY: ThreadSanitizer: signal handler spoils errno{{.*}}MyHandler
diff --git a/compiler-rt/test/tsan/signal_reset.cpp b/compiler-rt/test/tsan/signal_reset.cpp
index d76b7e5f3b5f..82758d882382 100644
--- a/compiler-rt/test/tsan/signal_reset.cpp
+++ b/compiler-rt/test/tsan/signal_reset.cpp
@@ -28,12 +28,12 @@ static void* reset(void *p) {
   struct sigaction act = {};
   for (int i = 0; i < 1000000; i++) {
     act.sa_handler = &handler;
-    if (sigaction(SIGALRM, &act, 0)) {
+    if (sigaction(SIGPROF, &act, 0)) {
       perror("sigaction");
       exit(1);
     }
     act.sa_handler = SIG_IGN;
-    if (sigaction(SIGALRM, &act, 0)) {
+    if (sigaction(SIGPROF, &act, 0)) {
       perror("sigaction");
       exit(1);
     }
@@ -44,7 +44,7 @@ static void* reset(void *p) {
 int main() {
   struct sigaction act = {};
   act.sa_handler = SIG_IGN;
-  if (sigaction(SIGALRM, &act, 0)) {
+  if (sigaction(SIGPROF, &act, 0)) {
     perror("sigaction");
     exit(1);
   }
@@ -53,7 +53,7 @@ int main() {
   t.it_value.tv_sec = 0;
   t.it_value.tv_usec = 10;
   t.it_interval = t.it_value;
-  if (setitimer(ITIMER_REAL, &t, 0)) {
+  if (setitimer(ITIMER_PROF, &t, 0)) {
     perror("setitimer");
     exit(1);
   }
diff --git a/compiler-rt/test/tsan/signal_sync.cpp b/compiler-rt/test/tsan/signal_sync.cpp
index 878b3f3b88b9..b529a1859f52 100644
--- a/compiler-rt/test/tsan/signal_sync.cpp
+++ b/compiler-rt/test/tsan/signal_sync.cpp
@@ -30,7 +30,7 @@ int main() {
 
   struct sigaction act = {};
   act.sa_handler = &handler;
-  if (sigaction(SIGVTALRM, &act, 0)) {
+  if (sigaction(SIGPROF, &act, 0)) {
     perror("sigaction");
     exit(1);
   }
@@ -39,7 +39,7 @@ int main() {
   t.it_value.tv_sec = 0;
   t.it_value.tv_usec = 10;
   t.it_interval = t.it_value;
-  if (setitimer(ITIMER_VIRTUAL, &t, 0)) {
+  if (setitimer(ITIMER_PROF, &t, 0)) {
     perror("setitimer");
     exit(1);
   }
diff --git a/compiler-rt/test/tsan/signal_thread.cpp b/compiler-rt/test/tsan/signal_thread.cpp
index 7bba8159bf38..aa91d1ddeb10 100644
--- a/compiler-rt/test/tsan/signal_thread.cpp
+++ b/compiler-rt/test/tsan/signal_thread.cpp
@@ -24,7 +24,7 @@ static void* thr(void *p) {
 int main() {
   struct sigaction act = {};
   act.sa_handler = &handler;
-  if (sigaction(SIGVTALRM, &act, 0)) {
+  if (sigaction(SIGPROF, &act, 0)) {
     perror("sigaction");
     exit(1);
   }
@@ -33,7 +33,7 @@ int main() {
   t.it_value.tv_sec = 0;
   t.it_value.tv_usec = 10;
   t.it_interval = t.it_value;
-  if (setitimer(ITIMER_VIRTUAL, &t, 0)) {
+  if (setitimer(ITIMER_PROF, &t, 0)) {
     perror("setitimer");
     exit(1);
   }
diff --git a/compiler-rt/test/tsan/signal_thread2.cpp b/compiler-rt/test/tsan/signal_thread2.cpp
index 5236628e13b6..9bde4f70b39d 100644
--- a/compiler-rt/test/tsan/signal_thread2.cpp
+++ b/compiler-rt/test/tsan/signal_thread2.cpp
@@ -40,7 +40,7 @@ static void *thr(void *p) {
 int main() {
   struct sigaction act = {};
   act.sa_handler = &handler;
-  if (sigaction(SIGALRM, &act, 0)) {
+  if (sigaction(SIGPROF, &act, 0)) {
     perror("sigaction");
     exit(1);
   }
@@ -49,7 +49,7 @@ int main() {
   t.it_value.tv_sec = 0;
   t.it_value.tv_usec = 10;
   t.it_interval = t.it_value;
-  if (setitimer(ITIMER_REAL, &t, 0)) {
+  if (setitimer(ITIMER_PROF, &t, 0)) {
     perror("setitimer");
     exit(1);
   }
-- 
GitLab


From 4d69855e9d380ecb7c1f7a64c7b37258fe36f525 Mon Sep 17 00:00:00 2001
From: AtariDreams <83477269+AtariDreams@users.noreply.github.com>
Date: Sun, 24 Mar 2024 06:48:04 -0400
Subject: [PATCH 066/404] [flang] Silence MSVC warning about shifts (NFC)
 (#83737)

Yes, 64-bit shifts are intended.
---
 flang/lib/Optimizer/Builder/IntrinsicCall.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index eb8f5135ff12..ea1ef1f08aba 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -3883,7 +3883,7 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType,
   int pos = 3 + highSignificandSize;
   mlir::Value index = builder.create<mlir::arith::AndIOp>(
       loc, builder.create<mlir::arith::ShRUIOp>(loc, intVal, signShift),
-      createIntegerConstant(1 << pos));
+      createIntegerConstant(1ULL << pos));
 
   // [e] exponent != 0
   mlir::Value exponent =
@@ -3895,7 +3895,7 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType,
           loc,
           builder.create<mlir::arith::CmpIOp>(
               loc, mlir::arith::CmpIPredicate::ne, exponent, zero),
-          createIntegerConstant(1 << --pos), zero));
+          createIntegerConstant(1ULL << --pos), zero));
 
   // [m] exponent == 1..1 (max exponent)
   index = builder.create<mlir::arith::OrIOp>(
@@ -3904,7 +3904,7 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType,
           loc,
           builder.create<mlir::arith::CmpIOp>(
               loc, mlir::arith::CmpIPredicate::eq, exponent, exponentMask),
-          createIntegerConstant(1 << --pos), zero));
+          createIntegerConstant(1ULL << --pos), zero));
 
   // [l] low-order significand != 0
   index = builder.create<mlir::arith::OrIOp>(
@@ -3916,7 +3916,7 @@ mlir::Value IntrinsicLibrary::genIeeeClass(mlir::Type resultType,
               builder.create<mlir::arith::AndIOp>(loc, intVal,
                                                   lowSignificandMask),
               zero),
-          createIntegerConstant(1 << --pos), zero));
+          createIntegerConstant(1ULL << --pos), zero));
 
   // [h] high-order significand (1 or 2 bits)
   index = builder.create<mlir::arith::OrIOp>(
-- 
GitLab


From 6c6fe4b2aea8631001b11abee62146d4aca01cee Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Sun, 24 Mar 2024 11:33:51 +0000
Subject: [PATCH 067/404] [X86] known-never-zero.ll - add 32-bit test coverage

Enabled vector coverage as well: i686+SSE2 and x64_64+AVX

Should improve test quality for #85722
---
 llvm/test/CodeGen/X86/known-never-zero.ll | 1708 ++++++++++++++-------
 1 file changed, 1161 insertions(+), 547 deletions(-)

diff --git a/llvm/test/CodeGen/X86/known-never-zero.ll b/llvm/test/CodeGen/X86/known-never-zero.ll
index cc9862769f2b..423516bc3271 100644
--- a/llvm/test/CodeGen/X86/known-never-zero.ll
+++ b/llvm/test/CodeGen/X86/known-never-zero.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=CHECK
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=X64
 
 ;; Use cttz to test if we properly prove never-zero. There is a very
 ;; simple transform from cttz -> cttz_zero_undef if its operand is
@@ -18,41 +19,70 @@ declare i32 @llvm.fshl.i32(i32, i32, i32)
 declare i32 @llvm.fshr.i32(i32, i32, i32)
 
 define i32 @or_known_nonzero(i32 %x) {
-; CHECK-LABEL: or_known_nonzero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    orl $1, %edi
-; CHECK-NEXT:    rep bsfl %edi, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: or_known_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl $1, %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: or_known_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    orl $1, %edi
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    retq
   %z = or i32 %x, 1
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @or_maybe_zero(i32 %x, i32 %y) {
-; CHECK-LABEL: or_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    orl %esi, %edi
-; CHECK-NEXT:    je .LBB1_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %edi, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB1_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: or_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    je .LBB1_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB1_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: or_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    orl %esi, %edi
+; X64-NEXT:    je .LBB1_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB1_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %z = or i32 %x, %y
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @select_known_nonzero(i1 %c, i32 %x) {
-; CHECK-LABEL: select_known_nonzero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    orl $1, %esi
-; CHECK-NEXT:    testb $1, %dil
-; CHECK-NEXT:    movl $122, %eax
-; CHECK-NEXT:    cmovnel %esi, %eax
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: select_known_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl $1, %eax
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    movl $122, %ecx
+; X86-NEXT:    cmovnel %eax, %ecx
+; X86-NEXT:    rep bsfl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: select_known_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    orl $1, %esi
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    movl $122, %eax
+; X64-NEXT:    cmovnel %esi, %eax
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
   %y = or i32 %x, 1
   %z = select i1 %c, i32 %y, i32 122
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -60,20 +90,36 @@ define i32 @select_known_nonzero(i1 %c, i32 %x) {
 }
 
 define i32 @select_maybe_zero(i1 %c, i32 %x) {
-; CHECK-LABEL: select_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    orl $1, %esi
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb $1, %dil
-; CHECK-NEXT:    cmovnel %esi, %eax
-; CHECK-NEXT:    testl %eax, %eax
-; CHECK-NEXT:    je .LBB3_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB3_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: select_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    orl $1, %ecx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB3_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB3_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: select_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    orl $1, %esi
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    testb $1, %dil
+; X64-NEXT:    cmovnel %esi, %eax
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    je .LBB3_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB3_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %y = or i32 %x, 1
   %z = select i1 %c, i32 %y, i32 0
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -81,28 +127,45 @@ define i32 @select_maybe_zero(i1 %c, i32 %x) {
 }
 
 define i32 @shl_known_nonzero_1s_bit_set(i32 %x) {
-; CHECK-LABEL: shl_known_nonzero_1s_bit_set:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    movl $123, %eax
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    shll %cl, %eax
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: shl_known_nonzero_1s_bit_set:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $123, %eax
+; X86-NEXT:    shll %cl, %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: shl_known_nonzero_1s_bit_set:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    movl $123, %eax
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shll %cl, %eax
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
   %z = shl i32 123, %x
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @shl_known_nonzero_nsw(i32 %x, i32 %yy) {
-; CHECK-LABEL: shl_known_nonzero_nsw:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    orl $256, %esi # imm = 0x100
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    shll %cl, %esi
-; CHECK-NEXT:    rep bsfl %esi, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: shl_known_nonzero_nsw:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $256, %eax # imm = 0x100
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shll %cl, %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: shl_known_nonzero_nsw:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    orl $256, %esi # imm = 0x100
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shll %cl, %esi
+; X64-NEXT:    rep bsfl %esi, %eax
+; X64-NEXT:    retq
   %y = or i32 %yy, 256
   %z = shl nsw i32 %y, %x
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -110,14 +173,23 @@ define i32 @shl_known_nonzero_nsw(i32 %x, i32 %yy) {
 }
 
 define i32 @shl_known_nonzero_nuw(i32 %x, i32 %yy) {
-; CHECK-LABEL: shl_known_nonzero_nuw:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    orl $256, %esi # imm = 0x100
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    shll %cl, %esi
-; CHECK-NEXT:    rep bsfl %esi, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: shl_known_nonzero_nuw:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $256, %eax # imm = 0x100
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shll %cl, %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: shl_known_nonzero_nuw:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    orl $256, %esi # imm = 0x100
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shll %cl, %esi
+; X64-NEXT:    rep bsfl %esi, %eax
+; X64-NEXT:    retq
   %y = or i32 %yy, 256
   %z = shl nuw i32 %y, %x
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -125,67 +197,116 @@ define i32 @shl_known_nonzero_nuw(i32 %x, i32 %yy) {
 }
 
 define i32 @shl_maybe_zero(i32 %x, i32 %y) {
-; CHECK-LABEL: shl_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    shll %cl, %esi
-; CHECK-NEXT:    testl %esi, %esi
-; CHECK-NEXT:    je .LBB7_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %esi, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB7_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: shl_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shll %cl, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB7_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB7_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: shl_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shll %cl, %esi
+; X64-NEXT:    testl %esi, %esi
+; X64-NEXT:    je .LBB7_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %esi, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB7_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %z = shl nuw nsw i32 %y, %x
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @uaddsat_known_nonzero(i32 %x) {
-; CHECK-LABEL: uaddsat_known_nonzero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    incl %edi
-; CHECK-NEXT:    movl $-1, %eax
-; CHECK-NEXT:    cmovnel %edi, %eax
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: uaddsat_known_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    incl %eax
+; X86-NEXT:    movl $-1, %ecx
+; X86-NEXT:    cmovnel %eax, %ecx
+; X86-NEXT:    rep bsfl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: uaddsat_known_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    incl %edi
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovnel %edi, %eax
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
   %z = call i32 @llvm.uadd.sat.i32(i32 %x, i32 1)
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @uaddsat_maybe_zero(i32 %x, i32 %y) {
-; CHECK-LABEL: uaddsat_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    addl %esi, %edi
-; CHECK-NEXT:    movl $-1, %eax
-; CHECK-NEXT:    cmovael %edi, %eax
-; CHECK-NEXT:    testl %eax, %eax
-; CHECK-NEXT:    je .LBB9_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB9_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: uaddsat_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovael %ecx, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB9_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB9_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: uaddsat_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    addl %esi, %edi
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %edi, %eax
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    je .LBB9_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB9_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %z = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y)
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @umax_known_nonzero(i32 %x, i32 %y) {
-; CHECK-LABEL: umax_known_nonzero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %esi, %ecx
-; CHECK-NEXT:    movl $4, %eax
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    shll %cl, %eax
-; CHECK-NEXT:    cmpl %eax, %edi
-; CHECK-NEXT:    cmoval %edi, %eax
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: umax_known_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $4, %edx
+; X86-NEXT:    shll %cl, %edx
+; X86-NEXT:    cmpl %edx, %eax
+; X86-NEXT:    cmoval %eax, %edx
+; X86-NEXT:    rep bsfl %edx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: umax_known_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %ecx
+; X64-NEXT:    movl $4, %eax
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shll %cl, %eax
+; X64-NEXT:    cmpl %eax, %edi
+; X64-NEXT:    cmoval %edi, %eax
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
   %yy = shl nuw i32 4, %y
   %z = call i32 @llvm.umax.i32(i32 %x, i32 %yy)
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -193,35 +314,62 @@ define i32 @umax_known_nonzero(i32 %x, i32 %y) {
 }
 
 define i32 @umax_maybe_zero(i32 %x, i32 %y) {
-; CHECK-LABEL: umax_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    cmpl %esi, %edi
-; CHECK-NEXT:    cmoval %edi, %esi
-; CHECK-NEXT:    testl %esi, %esi
-; CHECK-NEXT:    je .LBB11_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %esi, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB11_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: umax_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    cmoval %ecx, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB11_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB11_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: umax_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    cmpl %esi, %edi
+; X64-NEXT:    cmoval %edi, %esi
+; X64-NEXT:    testl %esi, %esi
+; X64-NEXT:    je .LBB11_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %esi, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB11_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %z = call i32 @llvm.umax.i32(i32 %x, i32 %y)
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @umin_known_nonzero(i32 %xx, i32 %yy) {
-; CHECK-LABEL: umin_known_nonzero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    movl $4, %eax
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    shll %cl, %eax
-; CHECK-NEXT:    addl $4, %esi
-; CHECK-NEXT:    cmpl %esi, %eax
-; CHECK-NEXT:    cmovbl %eax, %esi
-; CHECK-NEXT:    rep bsfl %esi, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: umin_known_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $4, %edx
+; X86-NEXT:    shll %cl, %edx
+; X86-NEXT:    addl $4, %eax
+; X86-NEXT:    cmpl %eax, %edx
+; X86-NEXT:    cmovbl %edx, %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: umin_known_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    movl $4, %eax
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shll %cl, %eax
+; X64-NEXT:    addl $4, %esi
+; X64-NEXT:    cmpl %esi, %eax
+; X64-NEXT:    cmovbl %eax, %esi
+; X64-NEXT:    rep bsfl %esi, %eax
+; X64-NEXT:    retq
   %x = shl nuw i32 4, %xx
   %y = add nuw nsw i32 %yy, 4
   %z = call i32 @llvm.umin.i32(i32 %x, i32 %y)
@@ -230,36 +378,63 @@ define i32 @umin_known_nonzero(i32 %xx, i32 %yy) {
 }
 
 define i32 @umin_maybe_zero(i32 %x, i32 %y) {
-; CHECK-LABEL: umin_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    cmpl $54, %edi
-; CHECK-NEXT:    movl $54, %eax
-; CHECK-NEXT:    cmovbl %edi, %eax
-; CHECK-NEXT:    testl %eax, %eax
-; CHECK-NEXT:    je .LBB13_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB13_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: umin_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpl $54, %ecx
+; X86-NEXT:    movl $54, %eax
+; X86-NEXT:    cmovbl %ecx, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB13_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB13_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: umin_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    cmpl $54, %edi
+; X64-NEXT:    movl $54, %eax
+; X64-NEXT:    cmovbl %edi, %eax
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    je .LBB13_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB13_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %z = call i32 @llvm.umin.i32(i32 %x, i32 54)
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @smin_known_nonzero(i32 %xx, i32 %yy) {
-; CHECK-LABEL: smin_known_nonzero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    movl $4, %eax
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    shll %cl, %eax
-; CHECK-NEXT:    addl $4, %esi
-; CHECK-NEXT:    cmpl %esi, %eax
-; CHECK-NEXT:    cmovll %eax, %esi
-; CHECK-NEXT:    rep bsfl %esi, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: smin_known_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $4, %edx
+; X86-NEXT:    shll %cl, %edx
+; X86-NEXT:    addl $4, %eax
+; X86-NEXT:    cmpl %eax, %edx
+; X86-NEXT:    cmovll %edx, %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: smin_known_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    movl $4, %eax
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shll %cl, %eax
+; X64-NEXT:    addl $4, %esi
+; X64-NEXT:    cmpl %esi, %eax
+; X64-NEXT:    cmovll %eax, %esi
+; X64-NEXT:    rep bsfl %esi, %eax
+; X64-NEXT:    retq
   %x = shl nuw i32 4, %xx
   %y = add nuw nsw i32 %yy, 4
   %z = call i32 @llvm.smin.i32(i32 %x, i32 %y)
@@ -268,36 +443,63 @@ define i32 @smin_known_nonzero(i32 %xx, i32 %yy) {
 }
 
 define i32 @smin_maybe_zero(i32 %x, i32 %y) {
-; CHECK-LABEL: smin_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    cmpl $54, %edi
-; CHECK-NEXT:    movl $54, %eax
-; CHECK-NEXT:    cmovll %edi, %eax
-; CHECK-NEXT:    testl %eax, %eax
-; CHECK-NEXT:    je .LBB15_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB15_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: smin_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmpl $54, %ecx
+; X86-NEXT:    movl $54, %eax
+; X86-NEXT:    cmovll %ecx, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB15_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB15_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: smin_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    cmpl $54, %edi
+; X64-NEXT:    movl $54, %eax
+; X64-NEXT:    cmovll %edi, %eax
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    je .LBB15_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB15_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %z = call i32 @llvm.smin.i32(i32 %x, i32 54)
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @smax_known_nonzero(i32 %xx, i32 %yy) {
-; CHECK-LABEL: smax_known_nonzero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    movl $4, %eax
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    shll %cl, %eax
-; CHECK-NEXT:    addl $4, %esi
-; CHECK-NEXT:    cmpl %esi, %eax
-; CHECK-NEXT:    cmovgl %eax, %esi
-; CHECK-NEXT:    rep bsfl %esi, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: smax_known_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $4, %edx
+; X86-NEXT:    shll %cl, %edx
+; X86-NEXT:    addl $4, %eax
+; X86-NEXT:    cmpl %eax, %edx
+; X86-NEXT:    cmovgl %edx, %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: smax_known_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    movl $4, %eax
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shll %cl, %eax
+; X64-NEXT:    addl $4, %esi
+; X64-NEXT:    cmpl %esi, %eax
+; X64-NEXT:    cmovgl %eax, %esi
+; X64-NEXT:    rep bsfl %esi, %eax
+; X64-NEXT:    retq
   %x = shl nuw i32 4, %xx
   %y = add nuw nsw i32 %yy, 4
   %z = call i32 @llvm.smax.i32(i32 %x, i32 %y)
@@ -306,35 +508,61 @@ define i32 @smax_known_nonzero(i32 %xx, i32 %yy) {
 }
 
 define i32 @smax_maybe_zero(i32 %x, i32 %y) {
-; CHECK-LABEL: smax_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    cmpl $55, %edi
-; CHECK-NEXT:    movl $54, %eax
-; CHECK-NEXT:    cmovgel %edi, %eax
-; CHECK-NEXT:    bsfl %eax, %ecx
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    cmovnel %ecx, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: smax_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl $55, %eax
+; X86-NEXT:    movl $54, %ecx
+; X86-NEXT:    cmovgel %eax, %ecx
+; X86-NEXT:    bsfl %ecx, %ecx
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: smax_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    cmpl $55, %edi
+; X64-NEXT:    movl $54, %eax
+; X64-NEXT:    cmovgel %edi, %eax
+; X64-NEXT:    bsfl %eax, %ecx
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    cmovnel %ecx, %eax
+; X64-NEXT:    retq
   %z = call i32 @llvm.smax.i32(i32 %x, i32 54)
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @rotr_known_nonzero(i32 %xx, i32 %y) {
-; CHECK-LABEL: rotr_known_nonzero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %esi, %ecx
-; CHECK-NEXT:    orl $256, %edi # imm = 0x100
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    rorl %cl, %edi
-; CHECK-NEXT:    testl %edi, %edi
-; CHECK-NEXT:    je .LBB18_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %edi, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB18_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: rotr_known_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $256, %eax # imm = 0x100
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    rorl %cl, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB18_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB18_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: rotr_known_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %ecx
+; X64-NEXT:    orl $256, %edi # imm = 0x100
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    rorl %cl, %edi
+; X64-NEXT:    testl %edi, %edi
+; X64-NEXT:    je .LBB18_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB18_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %x = or i32 %xx, 256
   %shr = lshr i32 %x, %y
   %sub = sub i32 32, %y
@@ -345,19 +573,33 @@ define i32 @rotr_known_nonzero(i32 %xx, i32 %y) {
 }
 
 define i32 @rotr_maybe_zero(i32 %x, i32 %y) {
-; CHECK-LABEL: rotr_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %esi, %ecx
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    rorl %cl, %edi
-; CHECK-NEXT:    testl %edi, %edi
-; CHECK-NEXT:    je .LBB19_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %edi, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB19_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: rotr_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    rorl %cl, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB19_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB19_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: rotr_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %ecx
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    rorl %cl, %edi
+; X64-NEXT:    testl %edi, %edi
+; X64-NEXT:    je .LBB19_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB19_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %shr = lshr i32 %x, %y
   %sub = sub i32 32, %y
   %shl = shl i32 %x, %sub
@@ -367,14 +609,23 @@ define i32 @rotr_maybe_zero(i32 %x, i32 %y) {
 }
 
 define i32 @rotr_with_fshr_known_nonzero(i32 %xx, i32 %y) {
-; CHECK-LABEL: rotr_with_fshr_known_nonzero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %esi, %ecx
-; CHECK-NEXT:    orl $256, %edi # imm = 0x100
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    rorl %cl, %edi
-; CHECK-NEXT:    rep bsfl %edi, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: rotr_with_fshr_known_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $256, %eax # imm = 0x100
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    rorl %cl, %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: rotr_with_fshr_known_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %ecx
+; X64-NEXT:    orl $256, %edi # imm = 0x100
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    rorl %cl, %edi
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    retq
   %x = or i32 %xx, 256
   %z = call i32 @llvm.fshr.i32(i32 %x, i32 %x, i32 %y)
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -382,39 +633,68 @@ define i32 @rotr_with_fshr_known_nonzero(i32 %xx, i32 %y) {
 }
 
 define i32 @rotr_with_fshr_maybe_zero(i32 %x, i32 %y) {
-; CHECK-LABEL: rotr_with_fshr_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %esi, %ecx
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    rorl %cl, %edi
-; CHECK-NEXT:    testl %edi, %edi
-; CHECK-NEXT:    je .LBB21_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %edi, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB21_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: rotr_with_fshr_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    rorl %cl, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB21_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB21_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: rotr_with_fshr_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %ecx
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    rorl %cl, %edi
+; X64-NEXT:    testl %edi, %edi
+; X64-NEXT:    je .LBB21_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB21_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %z = call i32 @llvm.fshr.i32(i32 %x, i32 %x, i32 %y)
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @rotl_known_nonzero(i32 %xx, i32 %y) {
-; CHECK-LABEL: rotl_known_nonzero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %esi, %ecx
-; CHECK-NEXT:    orl $256, %edi # imm = 0x100
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    roll %cl, %edi
-; CHECK-NEXT:    testl %edi, %edi
-; CHECK-NEXT:    je .LBB22_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %edi, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB22_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: rotl_known_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $256, %eax # imm = 0x100
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    roll %cl, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB22_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB22_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: rotl_known_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %ecx
+; X64-NEXT:    orl $256, %edi # imm = 0x100
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    roll %cl, %edi
+; X64-NEXT:    testl %edi, %edi
+; X64-NEXT:    je .LBB22_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB22_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %x = or i32 %xx, 256
   %shl = shl i32 %x, %y
   %sub = sub i32 32, %y
@@ -425,19 +705,33 @@ define i32 @rotl_known_nonzero(i32 %xx, i32 %y) {
 }
 
 define i32 @rotl_maybe_zero(i32 %x, i32 %y) {
-; CHECK-LABEL: rotl_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %esi, %ecx
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    roll %cl, %edi
-; CHECK-NEXT:    testl %edi, %edi
-; CHECK-NEXT:    je .LBB23_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %edi, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB23_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: rotl_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    roll %cl, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB23_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB23_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: rotl_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %ecx
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    roll %cl, %edi
+; X64-NEXT:    testl %edi, %edi
+; X64-NEXT:    je .LBB23_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB23_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %shl = shl i32 %x, %y
   %sub = sub i32 32, %y
   %shr = lshr i32 %x, %sub
@@ -447,14 +741,23 @@ define i32 @rotl_maybe_zero(i32 %x, i32 %y) {
 }
 
 define i32 @rotl_with_fshl_known_nonzero(i32 %xx, i32 %y) {
-; CHECK-LABEL: rotl_with_fshl_known_nonzero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %esi, %ecx
-; CHECK-NEXT:    orl $256, %edi # imm = 0x100
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    roll %cl, %edi
-; CHECK-NEXT:    rep bsfl %edi, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: rotl_with_fshl_known_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $256, %eax # imm = 0x100
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    roll %cl, %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: rotl_with_fshl_known_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %ecx
+; X64-NEXT:    orl $256, %edi # imm = 0x100
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    roll %cl, %edi
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    retq
   %x = or i32 %xx, 256
   %z = call i32 @llvm.fshl.i32(i32 %x, i32 %x, i32 %y)
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -462,47 +765,78 @@ define i32 @rotl_with_fshl_known_nonzero(i32 %xx, i32 %y) {
 }
 
 define i32 @rotl_with_fshl_maybe_zero(i32 %x, i32 %y) {
-; CHECK-LABEL: rotl_with_fshl_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %esi, %ecx
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    roll %cl, %edi
-; CHECK-NEXT:    testl %edi, %edi
-; CHECK-NEXT:    je .LBB25_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %edi, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB25_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: rotl_with_fshl_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    roll %cl, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB25_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB25_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: rotl_with_fshl_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %ecx
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    roll %cl, %edi
+; X64-NEXT:    testl %edi, %edi
+; X64-NEXT:    je .LBB25_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB25_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %z = call i32 @llvm.fshl.i32(i32 %x, i32 %x, i32 %y)
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @sra_known_nonzero_sign_bit_set(i32 %x) {
-; CHECK-LABEL: sra_known_nonzero_sign_bit_set:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    movl $-2147360405, %eax # imm = 0x8001E16B
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    sarl %cl, %eax
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: sra_known_nonzero_sign_bit_set:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $-2147360405, %eax # imm = 0x8001E16B
+; X86-NEXT:    sarl %cl, %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: sra_known_nonzero_sign_bit_set:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    movl $-2147360405, %eax # imm = 0x8001E16B
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    sarl %cl, %eax
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
   %z = ashr i32 2147606891, %x
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @sra_known_nonzero_exact(i32 %x, i32 %yy) {
-; CHECK-LABEL: sra_known_nonzero_exact:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    orl $256, %esi # imm = 0x100
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    sarl %cl, %esi
-; CHECK-NEXT:    rep bsfl %esi, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: sra_known_nonzero_exact:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $256, %eax # imm = 0x100
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    sarl %cl, %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: sra_known_nonzero_exact:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    orl $256, %esi # imm = 0x100
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    sarl %cl, %esi
+; X64-NEXT:    rep bsfl %esi, %eax
+; X64-NEXT:    retq
   %y = or i32 %yy, 256
   %z = ashr exact i32 %y, %x
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -510,47 +844,78 @@ define i32 @sra_known_nonzero_exact(i32 %x, i32 %yy) {
 }
 
 define i32 @sra_maybe_zero(i32 %x, i32 %y) {
-; CHECK-LABEL: sra_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    sarl %cl, %esi
-; CHECK-NEXT:    testl %esi, %esi
-; CHECK-NEXT:    je .LBB28_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %esi, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB28_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: sra_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    sarl %cl, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB28_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB28_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: sra_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    sarl %cl, %esi
+; X64-NEXT:    testl %esi, %esi
+; X64-NEXT:    je .LBB28_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %esi, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB28_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %z = ashr exact i32 %y, %x
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @srl_known_nonzero_sign_bit_set(i32 %x) {
-; CHECK-LABEL: srl_known_nonzero_sign_bit_set:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    movl $-2147360405, %eax # imm = 0x8001E16B
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    shrl %cl, %eax
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: srl_known_nonzero_sign_bit_set:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $-2147360405, %eax # imm = 0x8001E16B
+; X86-NEXT:    shrl %cl, %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: srl_known_nonzero_sign_bit_set:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    movl $-2147360405, %eax # imm = 0x8001E16B
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shrl %cl, %eax
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
   %z = lshr i32 2147606891, %x
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @srl_known_nonzero_exact(i32 %x, i32 %yy) {
-; CHECK-LABEL: srl_known_nonzero_exact:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    orl $256, %esi # imm = 0x100
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    shrl %cl, %esi
-; CHECK-NEXT:    rep bsfl %esi, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: srl_known_nonzero_exact:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $256, %eax # imm = 0x100
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shrl %cl, %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: srl_known_nonzero_exact:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    orl $256, %esi # imm = 0x100
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shrl %cl, %esi
+; X64-NEXT:    rep bsfl %esi, %eax
+; X64-NEXT:    retq
   %y = or i32 %yy, 256
   %z = lshr exact i32 %y, %x
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -558,33 +923,56 @@ define i32 @srl_known_nonzero_exact(i32 %x, i32 %yy) {
 }
 
 define i32 @srl_maybe_zero(i32 %x, i32 %y) {
-; CHECK-LABEL: srl_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    shrl %cl, %esi
-; CHECK-NEXT:    testl %esi, %esi
-; CHECK-NEXT:    je .LBB31_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %esi, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB31_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: srl_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shrl %cl, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB31_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB31_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: srl_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shrl %cl, %esi
+; X64-NEXT:    testl %esi, %esi
+; X64-NEXT:    je .LBB31_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %esi, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB31_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %z = lshr exact i32 %y, %x
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @udiv_known_nonzero(i32 %xx, i32 %y) {
-; CHECK-LABEL: udiv_known_nonzero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    orl $64, %eax
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    divl %esi
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: udiv_known_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl $64, %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divl {{[0-9]+}}(%esp)
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: udiv_known_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    orl $64, %eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divl %esi
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
   %x = or i32 %xx, 64
   %z = udiv exact i32 %x, %y
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -592,33 +980,56 @@ define i32 @udiv_known_nonzero(i32 %xx, i32 %y) {
 }
 
 define i32 @udiv_maybe_zero(i32 %x, i32 %y) {
-; CHECK-LABEL: udiv_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    divl %esi
-; CHECK-NEXT:    testl %eax, %eax
-; CHECK-NEXT:    je .LBB33_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB33_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: udiv_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    divl {{[0-9]+}}(%esp)
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB33_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB33_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: udiv_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    divl %esi
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    je .LBB33_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB33_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %z = udiv exact i32 %x, %y
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @sdiv_known_nonzero(i32 %xx, i32 %y) {
-; CHECK-LABEL: sdiv_known_nonzero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    orl $64, %eax
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %esi
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: sdiv_known_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl $64, %eax
+; X86-NEXT:    cltd
+; X86-NEXT:    idivl {{[0-9]+}}(%esp)
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: sdiv_known_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    orl $64, %eax
+; X64-NEXT:    cltd
+; X64-NEXT:    idivl %esi
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
   %x = or i32 %xx, 64
   %z = sdiv exact i32 %x, %y
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -626,31 +1037,53 @@ define i32 @sdiv_known_nonzero(i32 %xx, i32 %y) {
 }
 
 define i32 @sdiv_maybe_zero(i32 %x, i32 %y) {
-; CHECK-LABEL: sdiv_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    cltd
-; CHECK-NEXT:    idivl %esi
-; CHECK-NEXT:    testl %eax, %eax
-; CHECK-NEXT:    je .LBB35_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB35_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: sdiv_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cltd
+; X86-NEXT:    idivl {{[0-9]+}}(%esp)
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB35_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB35_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: sdiv_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    cltd
+; X64-NEXT:    idivl %esi
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    je .LBB35_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB35_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %z = sdiv exact i32 %x, %y
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @add_known_nonzero(i32 %xx, i32 %y) {
-; CHECK-LABEL: add_known_nonzero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    orl $1, %edi
-; CHECK-NEXT:    addl %esi, %edi
-; CHECK-NEXT:    rep bsfl %edi, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: add_known_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl $1, %eax
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: add_known_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    orl $1, %edi
+; X64-NEXT:    addl %esi, %edi
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    retq
   %x = or i32 %xx, 1
   %z = add nuw i32 %x, %y
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -658,17 +1091,30 @@ define i32 @add_known_nonzero(i32 %xx, i32 %y) {
 }
 
 define i32 @add_maybe_zero(i32 %xx, i32 %y) {
-; CHECK-LABEL: add_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    orl $1, %edi
-; CHECK-NEXT:    addl %esi, %edi
-; CHECK-NEXT:    je .LBB37_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %edi, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB37_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: add_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl $1, %eax
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    je .LBB37_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB37_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: add_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    orl $1, %edi
+; X64-NEXT:    addl %esi, %edi
+; X64-NEXT:    je .LBB37_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB37_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %x = or i32 %xx, 1
   %z = add nsw i32 %x, %y
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -676,15 +1122,24 @@ define i32 @add_maybe_zero(i32 %xx, i32 %y) {
 }
 
 define i32 @sub_known_nonzero_neg_case(i32 %xx) {
-; CHECK-LABEL: sub_known_nonzero_neg_case:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    movl $256, %eax # imm = 0x100
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    shll %cl, %eax
-; CHECK-NEXT:    negl %eax
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: sub_known_nonzero_neg_case:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $256, %eax # imm = 0x100
+; X86-NEXT:    shll %cl, %eax
+; X86-NEXT:    negl %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: sub_known_nonzero_neg_case:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    movl $256, %eax # imm = 0x100
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shll %cl, %eax
+; X64-NEXT:    negl %eax
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
   %x = shl nuw nsw i32 256, %xx
   %z = sub i32 0, %x
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -692,14 +1147,24 @@ define i32 @sub_known_nonzero_neg_case(i32 %xx) {
 }
 
 define i32 @sub_known_nonzero_ne_case(i32 %xx, i32 %yy) {
-; CHECK-LABEL: sub_known_nonzero_ne_case:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    orl $64, %eax
-; CHECK-NEXT:    andl $-65, %edi
-; CHECK-NEXT:    subl %eax, %edi
-; CHECK-NEXT:    rep bsfl %edi, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: sub_known_nonzero_ne_case:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl %eax, %ecx
+; X86-NEXT:    orl $64, %ecx
+; X86-NEXT:    andl $-65, %eax
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: sub_known_nonzero_ne_case:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    orl $64, %eax
+; X64-NEXT:    andl $-65, %edi
+; X64-NEXT:    subl %eax, %edi
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    retq
   %x = or i32 %xx, 64
   %y = and i32 %xx, -65
   %z = sub i32 %y, %x
@@ -708,18 +1173,32 @@ define i32 @sub_known_nonzero_ne_case(i32 %xx, i32 %yy) {
 }
 
 define i32 @sub_maybe_zero(i32 %x) {
-; CHECK-LABEL: sub_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    orl $64, %eax
-; CHECK-NEXT:    subl %edi, %eax
-; CHECK-NEXT:    je .LBB40_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB40_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: sub_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    orl $64, %eax
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    je .LBB40_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB40_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: sub_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    orl $64, %eax
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    je .LBB40_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB40_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %y = or i32 %x, 64
   %z = sub i32 %y, %x
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -727,34 +1206,60 @@ define i32 @sub_maybe_zero(i32 %x) {
 }
 
 define i32 @sub_maybe_zero2(i32 %x) {
-; CHECK-LABEL: sub_maybe_zero2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    negl %edi
-; CHECK-NEXT:    je .LBB41_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %edi, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB41_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: sub_maybe_zero2:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    negl %eax
+; X86-NEXT:    je .LBB41_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB41_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: sub_maybe_zero2:
+; X64:       # %bb.0:
+; X64-NEXT:    negl %edi
+; X64-NEXT:    je .LBB41_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB41_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %z = sub i32 0, %x
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @mul_known_nonzero_nsw(i32 %x, i32 %yy) {
-; CHECK-LABEL: mul_known_nonzero_nsw:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    orl $256, %esi # imm = 0x100
-; CHECK-NEXT:    imull %edi, %esi
-; CHECK-NEXT:    testl %esi, %esi
-; CHECK-NEXT:    je .LBB42_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %esi, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB42_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: mul_known_nonzero_nsw:
+; X86:       # %bb.0:
+; X86-NEXT:    movl $256, %eax # imm = 0x100
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    imull {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB42_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB42_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: mul_known_nonzero_nsw:
+; X64:       # %bb.0:
+; X64-NEXT:    orl $256, %esi # imm = 0x100
+; X64-NEXT:    imull %edi, %esi
+; X64-NEXT:    testl %esi, %esi
+; X64-NEXT:    je .LBB42_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %esi, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB42_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %y = or i32 %yy, 256
   %z = mul nsw i32 %y, %x
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -762,18 +1267,32 @@ define i32 @mul_known_nonzero_nsw(i32 %x, i32 %yy) {
 }
 
 define i32 @mul_known_nonzero_nuw(i32 %x, i32 %yy) {
-; CHECK-LABEL: mul_known_nonzero_nuw:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    orl $256, %esi # imm = 0x100
-; CHECK-NEXT:    imull %edi, %esi
-; CHECK-NEXT:    testl %esi, %esi
-; CHECK-NEXT:    je .LBB43_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %esi, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB43_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: mul_known_nonzero_nuw:
+; X86:       # %bb.0:
+; X86-NEXT:    movl $256, %eax # imm = 0x100
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    imull {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB43_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB43_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: mul_known_nonzero_nuw:
+; X64:       # %bb.0:
+; X64-NEXT:    orl $256, %esi # imm = 0x100
+; X64-NEXT:    imull %edi, %esi
+; X64-NEXT:    testl %esi, %esi
+; X64-NEXT:    je .LBB43_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %esi, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB43_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %y = or i32 %yy, 256
   %z = mul nuw i32 %y, %x
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -781,36 +1300,63 @@ define i32 @mul_known_nonzero_nuw(i32 %x, i32 %yy) {
 }
 
 define i32 @mul_maybe_zero(i32 %x, i32 %y) {
-; CHECK-LABEL: mul_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    imull %esi, %edi
-; CHECK-NEXT:    testl %edi, %edi
-; CHECK-NEXT:    je .LBB44_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %edi, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB44_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: mul_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    imull {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB44_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB44_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: mul_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    imull %esi, %edi
+; X64-NEXT:    testl %edi, %edi
+; X64-NEXT:    je .LBB44_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB44_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %z = mul nuw nsw i32 %y, %x
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @bitcast_known_nonzero(<2 x i16> %xx) {
-; CHECK-LABEL: bitcast_known_nonzero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; CHECK-NEXT:    pslld $23, %xmm0
-; CHECK-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT:    cvttps2dq %xmm0, %xmm0
-; CHECK-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
-; CHECK-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    bsfl %eax, %ecx
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    cmovnel %ecx, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: bitcast_known_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; X86-NEXT:    pslld $23, %xmm0
+; X86-NEXT:    paddd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-NEXT:    cvttps2dq %xmm0, %xmm0
+; X86-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; X86-NEXT:    pmullw {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-NEXT:    movd %xmm0, %eax
+; X86-NEXT:    bsfl %eax, %ecx
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: bitcast_known_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; X64-NEXT:    vpslld $23, %xmm0, %xmm0
+; X64-NEXT:    vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-NEXT:    vcvttps2dq %xmm0, %xmm0
+; X64-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; X64-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-NEXT:    vmovd %xmm0, %eax
+; X64-NEXT:    bsfl %eax, %ecx
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    cmovnel %ecx, %eax
+; X64-NEXT:    retq
   %x = shl nuw nsw <2 x i16> <i16 256, i16 256>, %xx
   %z = bitcast <2 x i16> %x to i32
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -818,49 +1364,83 @@ define i32 @bitcast_known_nonzero(<2 x i16> %xx) {
 }
 
 define i32 @bitcast_maybe_zero(<2 x i16> %x) {
-; CHECK-LABEL: bitcast_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    testl %eax, %eax
-; CHECK-NEXT:    je .LBB46_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB46_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: bitcast_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movd %xmm0, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB46_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB46_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: bitcast_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovd %xmm0, %eax
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    je .LBB46_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB46_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %z = bitcast <2 x i16> %x to i32
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @bitcast_from_float(float %x) {
-; CHECK-LABEL: bitcast_from_float:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    testl %eax, %eax
-; CHECK-NEXT:    je .LBB47_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB47_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: bitcast_from_float:
+; X86:       # %bb.0:
+; X86-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    movd %xmm0, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB47_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB47_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: bitcast_from_float:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovd %xmm0, %eax
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    je .LBB47_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB47_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %z = bitcast float %x to i32
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @zext_known_nonzero(i16 %xx) {
-; CHECK-LABEL: zext_known_nonzero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    movl $256, %eax # imm = 0x100
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    shll %cl, %eax
-; CHECK-NEXT:    movzwl %ax, %eax
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: zext_known_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $256, %eax # imm = 0x100
+; X86-NEXT:    shll %cl, %eax
+; X86-NEXT:    movzwl %ax, %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: zext_known_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    movl $256, %eax # imm = 0x100
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shll %cl, %eax
+; X64-NEXT:    movzwl %ax, %eax
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
   %x = shl nuw nsw i16 256, %xx
   %z = zext i16 %x to i32
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -868,32 +1448,54 @@ define i32 @zext_known_nonzero(i16 %xx) {
 }
 
 define i32 @zext_maybe_zero(i16 %x) {
-; CHECK-LABEL: zext_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    testw %di, %di
-; CHECK-NEXT:    je .LBB49_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    movzwl %di, %eax
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB49_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: zext_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    testw %ax, %ax
+; X86-NEXT:    je .LBB49_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    movzwl %ax, %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB49_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: zext_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    testw %di, %di
+; X64-NEXT:    je .LBB49_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    movzwl %di, %eax
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB49_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %z = zext i16 %x to i32
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
 define i32 @sext_known_nonzero(i16 %xx) {
-; CHECK-LABEL: sext_known_nonzero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    movl $256, %eax # imm = 0x100
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
-; CHECK-NEXT:    shll %cl, %eax
-; CHECK-NEXT:    cwtl
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: sext_known_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $256, %eax # imm = 0x100
+; X86-NEXT:    shll %cl, %eax
+; X86-NEXT:    cwtl
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: sext_known_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    movl $256, %eax # imm = 0x100
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shll %cl, %eax
+; X64-NEXT:    cwtl
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
   %x = shl nuw nsw i16 256, %xx
   %z = sext i16 %x to i32
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
@@ -901,17 +1503,29 @@ define i32 @sext_known_nonzero(i16 %xx) {
 }
 
 define i32 @sext_maybe_zero(i16 %x) {
-; CHECK-LABEL: sext_maybe_zero:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    testw %di, %di
-; CHECK-NEXT:    je .LBB51_1
-; CHECK-NEXT:  # %bb.2: # %cond.false
-; CHECK-NEXT:    movswl %di, %eax
-; CHECK-NEXT:    rep bsfl %eax, %eax
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB51_1:
-; CHECK-NEXT:    movl $32, %eax
-; CHECK-NEXT:    retq
+; X86-LABEL: sext_maybe_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB51_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB51_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: sext_maybe_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    testw %di, %di
+; X64-NEXT:    je .LBB51_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    movswl %di, %eax
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB51_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
   %z = sext i16 %x to i32
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
-- 
GitLab


From e8d5223ce4b2214e052b5b52b2e6453ffea0fe33 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Sun, 24 Mar 2024 12:32:47 +0000
Subject: [PATCH 068/404] [AArch64] Additional GISel test coverage. NFC

---
 llvm/test/CodeGen/AArch64/setcc_knownbits.ll | 95 +++++++++++++-------
 1 file changed, 65 insertions(+), 30 deletions(-)

diff --git a/llvm/test/CodeGen/AArch64/setcc_knownbits.ll b/llvm/test/CodeGen/AArch64/setcc_knownbits.ll
index bb9546af8bb7..6a337bf02e09 100644
--- a/llvm/test/CodeGen/AArch64/setcc_knownbits.ll
+++ b/llvm/test/CodeGen/AArch64/setcc_knownbits.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc < %s -mtriple=aarch64 | FileCheck %s
+; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc < %s -mtriple=aarch64 -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 define i1 @load_bv_v4i8(i1 zeroext %a) {
 ; CHECK-LABEL: load_bv_v4i8:
@@ -11,18 +12,31 @@ define i1 @load_bv_v4i8(i1 zeroext %a) {
 }
 
 define noundef i1 @logger(i32 noundef %logLevel, ptr %ea, ptr %pll) {
-; CHECK-LABEL: logger:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr w8, [x2]
-; CHECK-NEXT:    cmp w8, w0
-; CHECK-NEXT:    b.ls .LBB1_2
-; CHECK-NEXT:  // %bb.1:
-; CHECK-NEXT:    mov w0, wzr
-; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB1_2: // %land.rhs
-; CHECK-NEXT:    ldr x8, [x1]
-; CHECK-NEXT:    ldrb w0, [x8]
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: logger:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr w8, [x2]
+; CHECK-SD-NEXT:    cmp w8, w0
+; CHECK-SD-NEXT:    b.ls .LBB1_2
+; CHECK-SD-NEXT:  // %bb.1:
+; CHECK-SD-NEXT:    mov w0, wzr
+; CHECK-SD-NEXT:    ret
+; CHECK-SD-NEXT:  .LBB1_2: // %land.rhs
+; CHECK-SD-NEXT:    ldr x8, [x1]
+; CHECK-SD-NEXT:    ldrb w0, [x8]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: logger:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x2]
+; CHECK-GI-NEXT:    cmp w8, w0
+; CHECK-GI-NEXT:    mov w0, wzr
+; CHECK-GI-NEXT:    b.hi .LBB1_2
+; CHECK-GI-NEXT:  // %bb.1: // %land.rhs
+; CHECK-GI-NEXT:    ldr x8, [x1]
+; CHECK-GI-NEXT:    ldrb w8, [x8]
+; CHECK-GI-NEXT:    and w0, w8, #0x1
+; CHECK-GI-NEXT:  .LBB1_2: // %land.end
+; CHECK-GI-NEXT:    ret
 entry:
   %0 = load i32, ptr %pll, align 4
   %cmp.not = icmp ugt i32 %0, %logLevel
@@ -44,12 +58,20 @@ land.end:                                         ; preds = %land.rhs, %entry
 
 declare i64 @llvm.ctlz.i64(i64 %in, i1)
 define i1 @lshr_ctlz_undef_cmpeq_one_i64(i64 %in) {
-; CHECK-LABEL: lshr_ctlz_undef_cmpeq_one_i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    clz x8, x0
-; CHECK-NEXT:    lsr x0, x8, #6
-; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: lshr_ctlz_undef_cmpeq_one_i64:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    clz x8, x0
+; CHECK-SD-NEXT:    lsr x0, x8, #6
+; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: lshr_ctlz_undef_cmpeq_one_i64:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    clz x8, x0
+; CHECK-GI-NEXT:    lsr x8, x8, #6
+; CHECK-GI-NEXT:    cmp x8, #1
+; CHECK-GI-NEXT:    cset w0, eq
+; CHECK-GI-NEXT:    ret
   %ctlz = call i64 @llvm.ctlz.i64(i64 %in, i1 -1)
   %lshr = lshr i64 %ctlz, 6
   %icmp = icmp eq i64 %lshr, 1
@@ -57,17 +79,30 @@ define i1 @lshr_ctlz_undef_cmpeq_one_i64(i64 %in) {
 }
 
 define i32 @PR17487(i1 %tobool) {
-; CHECK-LABEL: PR17487:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    dup v0.2s, w0
-; CHECK-NEXT:    mov w8, #1 // =0x1
-; CHECK-NEXT:    dup v1.2d, x8
-; CHECK-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-NEXT:    bic v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    mov x8, v0.d[1]
-; CHECK-NEXT:    cmp x8, #1
-; CHECK-NEXT:    cset w0, ne
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: PR17487:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    dup v0.2s, w0
+; CHECK-SD-NEXT:    mov w8, #1 // =0x1
+; CHECK-SD-NEXT:    dup v1.2d, x8
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    bic v0.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT:    mov x8, v0.d[1]
+; CHECK-SD-NEXT:    cmp x8, #1
+; CHECK-SD-NEXT:    cset w0, ne
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: PR17487:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-GI-NEXT:    mov v0.d[1], x0
+; CHECK-GI-NEXT:    adrp x8, .LCPI3_0
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI3_0]
+; CHECK-GI-NEXT:    bic v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT:    mov d0, v0.d[1]
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    cmp x8, #1
+; CHECK-GI-NEXT:    cset w0, ne
+; CHECK-GI-NEXT:    ret
   %tmp = insertelement <2 x i1> undef, i1 %tobool, i32 1
   %tmp1 = zext <2 x i1> %tmp to <2 x i64>
   %tmp2 = xor <2 x i64> %tmp1, <i64 1, i64 1>
-- 
GitLab


From b3fe27f2be0585d5d2ad46f96956ccfd76ca003e Mon Sep 17 00:00:00 2001
From: Marc Auberer <marc.auberer@chillibits.com>
Date: Sun, 24 Mar 2024 16:14:56 +0100
Subject: [PATCH 069/404] [InstCombine] Copy flags of extractelement for extelt
 -> icmp combine (#86366)

Fixes #86164
---
 llvm/include/llvm/IR/InstrTypes.h                     | 11 +++++++++++
 llvm/lib/IR/Instructions.cpp                          | 10 ++++++++++
 .../Transforms/InstCombine/InstCombineVectorOps.cpp   |  4 +++-
 llvm/test/Transforms/InstCombine/scalarization.ll     | 11 +++++++++++
 4 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h
index e8c2cba8418d..d2b33fe9c651 100644
--- a/llvm/include/llvm/IR/InstrTypes.h
+++ b/llvm/include/llvm/IR/InstrTypes.h
@@ -1058,6 +1058,17 @@ public:
   static CmpInst *Create(OtherOps Op, Predicate predicate, Value *S1,
                          Value *S2, const Twine &Name, BasicBlock *InsertAtEnd);
 
+  /// Construct a compare instruction, given the opcode, the predicate,
+  /// the two operands and the instruction to copy the flags from. Optionally
+  /// (if InstBefore is specified) insert the instruction into a BasicBlock
+  /// right before the specified instruction. The specified Instruction is
+  /// allowed to be a dereferenced end iterator. Create a CmpInst
+  static CmpInst *CreateWithCopiedFlags(OtherOps Op, Predicate Pred, Value *S1,
+                                        Value *S2,
+                                        const Instruction *FlagsSource,
+                                        const Twine &Name = "",
+                                        Instruction *InsertBefore = nullptr);
+
   /// Get the opcode casted to the right type
   OtherOps getOpcode() const {
     return static_cast<OtherOps>(Instruction::getOpcode());
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 494d50f89e37..8ca211e6e2e7 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -4623,6 +4623,16 @@ CmpInst::Create(OtherOps Op, Predicate predicate, Value *S1, Value *S2,
                       S1, S2, Name);
 }
 
+CmpInst *CmpInst::CreateWithCopiedFlags(OtherOps Op, Predicate Pred, Value *S1,
+                                        Value *S2,
+                                        const Instruction *FlagsSource,
+                                        const Twine &Name,
+                                        Instruction *InsertBefore) {
+  CmpInst *Inst = Create(Op, Pred, S1, S2, Name, InsertBefore);
+  Inst->copyIRFlags(FlagsSource);
+  return Inst;
+}
+
 void CmpInst::swapOperands() {
   if (ICmpInst *IC = dyn_cast<ICmpInst>(this))
     IC->swapOperands();
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index c7f4fb17648c..99f1f8eb34bb 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -487,7 +487,9 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
     // extelt (cmp X, Y), Index --> cmp (extelt X, Index), (extelt Y, Index)
     Value *E0 = Builder.CreateExtractElement(X, Index);
     Value *E1 = Builder.CreateExtractElement(Y, Index);
-    return CmpInst::Create(cast<CmpInst>(SrcVec)->getOpcode(), Pred, E0, E1);
+    CmpInst *SrcCmpInst = cast<CmpInst>(SrcVec);
+    return CmpInst::CreateWithCopiedFlags(SrcCmpInst->getOpcode(), Pred, E0, E1,
+                                          SrcCmpInst);
   }
 
   if (auto *I = dyn_cast<Instruction>(SrcVec)) {
diff --git a/llvm/test/Transforms/InstCombine/scalarization.ll b/llvm/test/Transforms/InstCombine/scalarization.ll
index fe6dc526bd50..7e645ef7e883 100644
--- a/llvm/test/Transforms/InstCombine/scalarization.ll
+++ b/llvm/test/Transforms/InstCombine/scalarization.ll
@@ -341,6 +341,17 @@ define i1 @extractelt_vector_fcmp_constrhs_dynidx(<2 x float> %arg, i32 %idx) {
   ret i1 %ext
 }
 
+define i1 @extractelt_vector_fcmp_copy_flags(<4 x float> %x) {
+; CHECK-LABEL: @extractelt_vector_fcmp_copy_flags(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[X:%.*]], i64 2
+; CHECK-NEXT:    [[TMP2:%.*]] = fcmp nsz arcp oeq float [[TMP1]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp = fcmp nsz arcp oeq <4 x float> %x, zeroinitializer
+  %r = extractelement <4 x i1> %cmp, i32 2
+  ret i1 %r
+}
+
 define i1 @extractelt_vector_fcmp_not_cheap_to_scalarize_multi_use(<2 x float> %arg0, <2 x float> %arg1, <2 x float> %arg2, i32 %idx) {
 ;
 ; CHECK-LABEL: @extractelt_vector_fcmp_not_cheap_to_scalarize_multi_use(
-- 
GitLab


From 9632e1515c93453efc39752b1c9f32aedd358fbc Mon Sep 17 00:00:00 2001
From: houndlord <45179481+houndlord@users.noreply.github.com>
Date: Sun, 24 Mar 2024 16:33:16 +0100
Subject: [PATCH 070/404] Match fixed width ISD::AVGFLOORS + ISD::AVGCEILS
 patterns (#86222)

---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 37 ++++++++++++++-----
 llvm/test/CodeGen/AArch64/hadd-combine.ll     | 24 ++++++++++++
 2 files changed, 51 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index e27a8bb8fdac..05b4ce3aaa2c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -2529,20 +2529,28 @@ static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG) {
   return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit);
 }
 
-// Attempt to form avgceilu(A, B) from (A | B) - ((A ^ B) >> 1)
-static SDValue combineFixedwidthToAVGCEILU(SDNode *N, SelectionDAG &DAG) {
+// Attempt to form avgceil(A, B) from (A | B) - ((A ^ B) >> 1)
+static SDValue combineFixedwidthToAVGCEIL(SDNode *N, SelectionDAG &DAG) {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SDValue N0 = N->getOperand(0);
   EVT VT = N0.getValueType();
   SDLoc DL(N);
+  SDValue A, B;
+
   if (TLI.isOperationLegal(ISD::AVGCEILU, VT)) {
-    SDValue A, B;
     if (sd_match(N, m_Sub(m_Or(m_Value(A), m_Value(B)),
                           m_Srl(m_Xor(m_Deferred(A), m_Deferred(B)),
                                 m_SpecificInt(1))))) {
       return DAG.getNode(ISD::AVGCEILU, DL, VT, A, B);
     }
   }
+  if (TLI.isOperationLegal(ISD::AVGCEILS, VT)) {
+    if (sd_match(N, m_Sub(m_Or(m_Value(A), m_Value(B)),
+                          m_Sra(m_Xor(m_Deferred(A), m_Deferred(B)),
+                                m_SpecificInt(1))))) {
+      return DAG.getNode(ISD::AVGCEILS, DL, VT, A, B);
+    }
+  }
   return SDValue();
 }
 
@@ -2837,20 +2845,29 @@ SDValue DAGCombiner::visitADDLike(SDNode *N) {
   return SDValue();
 }
 
-// Attempt to form avgflooru(A, B) from (A & B) + ((A ^ B) >> 1)
-static SDValue combineFixedwidthToAVGFLOORU(SDNode *N, SelectionDAG &DAG) {
+// Attempt to form avgfloor(A, B) from (A & B) + ((A ^ B) >> 1)
+static SDValue combineFixedwidthToAVGFLOOR(SDNode *N, SelectionDAG &DAG) {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SDValue N0 = N->getOperand(0);
   EVT VT = N0.getValueType();
   SDLoc DL(N);
+  SDValue A, B;
+
   if (TLI.isOperationLegal(ISD::AVGFLOORU, VT)) {
-    SDValue A, B;
     if (sd_match(N, m_Add(m_And(m_Value(A), m_Value(B)),
                           m_Srl(m_Xor(m_Deferred(A), m_Deferred(B)),
                                 m_SpecificInt(1))))) {
       return DAG.getNode(ISD::AVGFLOORU, DL, VT, A, B);
     }
   }
+  if (TLI.isOperationLegal(ISD::AVGFLOORS, VT)) {
+    if (sd_match(N, m_Add(m_And(m_Value(A), m_Value(B)),
+                          m_Sra(m_Xor(m_Deferred(A), m_Deferred(B)),
+                                m_SpecificInt(1))))) {
+      return DAG.getNode(ISD::AVGFLOORS, DL, VT, A, B);
+    }
+  }
+
   return SDValue();
 }
 
@@ -2869,8 +2886,8 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
   if (SDValue V = foldAddSubOfSignBit(N, DAG))
     return V;
 
-  // Try to match AVGFLOORU fixedwidth pattern
-  if (SDValue V = combineFixedwidthToAVGFLOORU(N, DAG))
+  // Try to match AVGFLOOR fixedwidth pattern
+  if (SDValue V = combineFixedwidthToAVGFLOOR(N, DAG))
     return V;
 
   // fold (a+b) -> (a|b) iff a and b share no bits.
@@ -3868,8 +3885,8 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
   if (SDValue V = foldAddSubOfSignBit(N, DAG))
     return V;
 
-  // Try to match AVGCEILU fixedwidth pattern
-  if (SDValue V = combineFixedwidthToAVGCEILU(N, DAG))
+  // Try to match AVGCEIL fixedwidth pattern
+  if (SDValue V = combineFixedwidthToAVGCEIL(N, DAG))
     return V;
 
   if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, SDLoc(N)))
diff --git a/llvm/test/CodeGen/AArch64/hadd-combine.ll b/llvm/test/CodeGen/AArch64/hadd-combine.ll
index e12502980790..491bf40ea4aa 100644
--- a/llvm/test/CodeGen/AArch64/hadd-combine.ll
+++ b/llvm/test/CodeGen/AArch64/hadd-combine.ll
@@ -341,6 +341,18 @@ define <8 x i16> @sub_fixedwidth_v4i32(<8 x i16> %a0, <8 x i16> %a1)  {
   ret <8 x i16> %res
 }
 
+define <8 x i16> @srhadd_fixedwidth_v8i16(<8 x i16> %a0, <8 x i16> %a1)  {
+; CHECK-LABEL: srhadd_fixedwidth_v8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    srhadd v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    ret
+  %or = or <8 x i16> %a0, %a1
+  %xor = xor <8 x i16> %a0, %a1
+  %srl = ashr <8 x i16> %xor, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %res = sub <8 x i16> %or, %srl
+  ret <8 x i16> %res
+}
+
 define <8 x i16> @rhaddu_base(<8 x i16> %src1, <8 x i16> %src2) {
 ; CHECK-LABEL: rhaddu_base:
 ; CHECK:       // %bb.0:
@@ -879,6 +891,18 @@ define <8 x i16> @uhadd_fixedwidth_v4i32(<8 x i16> %a0, <8 x i16> %a1)  {
   ret <8 x i16> %res
 }
 
+define <8 x i16> @shadd_fixedwidth_v8i16(<8 x i16> %a0, <8 x i16> %a1)  {
+; CHECK-LABEL: shadd_fixedwidth_v8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shadd v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    ret
+  %and = and <8 x i16> %a0, %a1
+  %xor = xor <8 x i16> %a0, %a1
+  %srl = ashr <8 x i16> %xor, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %res = add <8 x i16> %and, %srl
+  ret <8 x i16> %res
+}
+
 declare <8 x i8> @llvm.aarch64.neon.shadd.v8i8(<8 x i8>, <8 x i8>)
 declare <4 x i16> @llvm.aarch64.neon.shadd.v4i16(<4 x i16>, <4 x i16>)
 declare <2 x i32> @llvm.aarch64.neon.shadd.v2i32(<2 x i32>, <2 x i32>)
-- 
GitLab


From 48048051323d5dd74057dc5f32df8c3c323afcd5 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Sun, 24 Mar 2024 11:43:15 -0600
Subject: [PATCH 071/404] [lld/ELF][X86] Respect outSecOff when checking if
 GOTPCREL can be relaxed (#86334)

The existing implementation didn't handle when the input text section
was some offset from the output section.

This resulted in an assert in relaxGot() with an lld built with asserts
for some large binaries, or even worse, a silently broken binary with an
lld without asserts.
---
 lld/ELF/Arch/X86_64.cpp                   |  7 ++++---
 lld/test/ELF/x86-64-gotpc-relax-too-far.s | 12 +++++++++++-
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/lld/ELF/Arch/X86_64.cpp b/lld/ELF/Arch/X86_64.cpp
index de459013595f..a85bf3aa0c09 100644
--- a/lld/ELF/Arch/X86_64.cpp
+++ b/lld/ELF/Arch/X86_64.cpp
@@ -328,9 +328,10 @@ bool X86_64::relaxOnce(int pass) const {
         if (rel.expr != R_RELAX_GOT_PC)
           continue;
 
-        uint64_t v = sec->getRelocTargetVA(
-            sec->file, rel.type, rel.addend,
-            sec->getOutputSection()->addr + rel.offset, *rel.sym, rel.expr);
+        uint64_t v = sec->getRelocTargetVA(sec->file, rel.type, rel.addend,
+                                           sec->getOutputSection()->addr +
+                                               sec->outSecOff + rel.offset,
+                                           *rel.sym, rel.expr);
         if (isInt<32>(v))
           continue;
         if (rel.sym->auxIdx == 0) {
diff --git a/lld/test/ELF/x86-64-gotpc-relax-too-far.s b/lld/test/ELF/x86-64-gotpc-relax-too-far.s
index 74aa6d8f65a0..ba41faab67de 100644
--- a/lld/test/ELF/x86-64-gotpc-relax-too-far.s
+++ b/lld/test/ELF/x86-64-gotpc-relax-too-far.s
@@ -5,7 +5,10 @@
 # RUN: llvm-objdump --no-print-imm-hex -d %t/bin | FileCheck --check-prefix=DISASM %s
 # RUN: llvm-readelf -S %t/bin | FileCheck --check-prefixes=GOT %s
 # RUN: ld.lld -T %t/lds2 %t/a.o -o %t/bin2
-# RUN: llvm-readelf -S %t/bin2 | FileCheck --check-prefixes=UNNECESSARY-GOT %s
+# RUN: llvm-objdump --no-print-imm-hex -d %t/bin2 | FileCheck --check-prefix=DISASM %s
+# RUN: llvm-readelf -S %t/bin2 | FileCheck --check-prefixes=GOT %s
+# RUN: ld.lld -T %t/lds3 %t/a.o -o %t/bin3
+# RUN: llvm-readelf -S %t/bin3 | FileCheck --check-prefixes=UNNECESSARY-GOT %s
 
 # DISASM:      <_foo>:
 # DISASM-NEXT: movl    2097146(%rip), %eax
@@ -47,6 +50,13 @@ SECTIONS {
   data 0x80200000 : { *(data) }
 }
 #--- lds2
+SECTIONS {
+  .text.foo 0x100000 : { *(.text.foo) }
+  .text 0x1ff000 : { . = . + 0x1000 ; *(.text) }
+  .got 0x300000 : { *(.got) }
+  data 0x80200000 : { *(data) }
+}
+#--- lds3
 SECTIONS {
   .text.foo 0x100000 : { *(.text.foo) }
   .text 0x200000 : { *(.text) }
-- 
GitLab


From d4a4585165f5c6ca8c42920b70e1b47696ff1172 Mon Sep 17 00:00:00 2001
From: Piotr Zegar <piotr.zegar@nokia.com>
Date: Sun, 24 Mar 2024 19:52:19 +0100
Subject: [PATCH 072/404] [clang-tidy] Ignore expresions in unevaluated context
 in bugprone-inc-dec-in-conditions (#85849)

Skip checking for references to variable in unevaluated context, like
decltype, static_assert and so on.

Closes #85838
---
 .../clang-tidy/bugprone/IncDecInConditionsCheck.cpp    |  8 +++++++-
 clang-tools-extra/docs/ReleaseNotes.rst                |  4 ++++
 .../checkers/bugprone/inc-dec-in-conditions.cpp        | 10 ++++++++++
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp
index 16f43128d55e..9b3b01eb0268 100644
--- a/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/IncDecInConditionsCheck.cpp
@@ -31,6 +31,10 @@ void IncDecInConditionsCheck::registerMatchers(MatchFinder *Finder) {
       anyOf(binaryOperator(anyOf(isComparisonOperator(), isLogicalOperator())),
             cxxOperatorCallExpr(isComparisonOperator())));
 
+  auto IsInUnevaluatedContext =
+      expr(anyOf(hasAncestor(expr(matchers::hasUnevaluatedContext())),
+                 hasAncestor(typeLoc())));
+
   Finder->addMatcher(
       expr(
           OperatorMatcher, unless(isExpansionInSystemHeader()),
@@ -42,12 +46,14 @@ void IncDecInConditionsCheck::registerMatchers(MatchFinder *Finder) {
                          cxxOperatorCallExpr(
                              isPrePostOperator(),
                              hasUnaryOperand(expr().bind("operand")))),
+                   unless(IsInUnevaluatedContext),
                    hasAncestor(
                        expr(equalsBoundNode("parent"),
                             hasDescendant(
                                 expr(unless(equalsBoundNode("operand")),
                                      matchers::isStatementIdenticalToBoundNode(
-                                         "operand"))
+                                         "operand"),
+                                     unless(IsInUnevaluatedContext))
                                     .bind("second")))))
                   .bind("operator"))),
       this);
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index a604e9276668..2392ccaf6575 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -139,6 +139,10 @@ Changes in existing checks
   <clang-tidy/checks/bugprone/assert-side-effect>` check by detecting side
   effect from calling a method with non-const reference parameters.
 
+- Improved :doc:`bugprone-inc-dec-in-conditions
+  <clang-tidy/checks/bugprone/inc-dec-in-conditions>` check to ignore code
+  within unevaluated contexts, such as ``decltype``.
+
 - Improved :doc:`bugprone-non-zero-enum-to-bool-conversion
   <clang-tidy/checks/bugprone/non-zero-enum-to-bool-conversion>` check by
   eliminating false positives resulting from direct usage of bitwise operators
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp
index 82af039973c3..91de013138f0 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/inc-dec-in-conditions.cpp
@@ -68,3 +68,13 @@ bool doubleCheck(Container<int> x) {
   // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: decrementing and referencing a variable in a complex condition can cause unintended side-effects due to C++'s order of evaluation, consider moving the modification outside of the condition to avoid misunderstandings [bugprone-inc-dec-in-conditions]
   // CHECK-MESSAGES: :[[@LINE-2]]:31: warning: incrementing and referencing a variable in a complex condition can cause unintended side-effects due to C++'s order of evaluation, consider moving the modification outside of the condition to avoid misunderstandings [bugprone-inc-dec-in-conditions]
 }
+
+namespace PR85838 {
+  void test()
+  {
+    auto foo = 0;
+    auto bar = 0;
+    if (++foo < static_cast<decltype(foo)>(bar)) {}
+    if (static_cast<decltype(++foo)>(bar) < foo) {}
+  }
+}
-- 
GitLab


From 909ea28ac60760a0e9b39369b2f54a4e8f1daec9 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Sun, 24 Mar 2024 14:29:23 -0500
Subject: [PATCH 073/404] [Libomptarget] Specificall add LLVM include dirs in
 plugins

---
 openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt
index 0420d0e6f1f8..b84f3d7b137c 100644
--- a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt
+++ b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt
@@ -60,6 +60,7 @@ target_link_options(PluginCommon PUBLIC ${offload_link_flags})
 
 target_include_directories(PluginCommon PUBLIC 
   ${CMAKE_CURRENT_SOURCE_DIR}/include
+  ${LIBOMPTARGET_LLVM_INCLUDE_DIRS}
   ${LIBOMPTARGET_INCLUDE_DIR}
 )
 
-- 
GitLab


From 488a18738f5c275093bcc5459da69e9b4c9de074 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Sun, 24 Mar 2024 10:41:21 -0500
Subject: [PATCH 074/404] [JITLink][ELF] Improve ELF section start/end symbol
 handling.

This commit adds section start and stop symbol handling to ELF/aarch64, and
fixes the section symbol prefixes (using `__start_` and `__stop_`, rather than
`__start` and `__end`). It also adds a testcase for handling of these symbols.
---
 .../llvm/ExecutionEngine/JITLink/JITLink.h    |  2 +-
 .../DefineExternalSectionStartAndEndSymbols.h | 19 ++++++++
 .../ExecutionEngine/JITLink/ELF_aarch64.cpp   |  7 +++
 .../ExecutionEngine/JITLink/ELF_x86_64.cpp    | 18 --------
 .../ELF_section_start_and_stop_symbols.s      | 43 +++++++++++++++++++
 5 files changed, 70 insertions(+), 19 deletions(-)
 create mode 100644 llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s

diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
index 30a9383dc0bc..4ce0c8d9fd17 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
@@ -567,7 +567,7 @@ public:
   orc::ExecutorAddrDiff getOffset() const { return Offset; }
 
   void setOffset(orc::ExecutorAddrDiff NewOffset) {
-    assert(NewOffset < getBlock().getSize() && "Offset out of range");
+    assert(NewOffset <= getBlock().getSize() && "Offset out of range");
     Offset = NewOffset;
   }
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h
index 159880e4b152..aef441d093a2 100644
--- a/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h
+++ b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h
@@ -108,6 +108,25 @@ createDefineExternalSectionStartAndEndSymbolsPass(
       std::forward<SymbolIdentifierFunction>(F));
 }
 
+/// ELF section start/end symbol detection.
+inline SectionRangeSymbolDesc
+identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
+  constexpr StringRef StartSymbolPrefix = "__start_";
+  constexpr StringRef EndSymbolPrefix = "__stop_";
+
+  auto SymName = Sym.getName();
+  if (SymName.starts_with(StartSymbolPrefix)) {
+    if (auto *Sec =
+            G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size())))
+      return {*Sec, true};
+  } else if (SymName.starts_with(EndSymbolPrefix)) {
+    if (auto *Sec =
+            G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size())))
+      return {*Sec, false};
+  }
+  return {};
+}
+
 } // end namespace jitlink
 } // end namespace llvm
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
index f17b2c626ac2..4f64d83162b2 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
@@ -20,6 +20,8 @@
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/Support/Endian.h"
 
+#include "DefineExternalSectionStartAndEndSymbols.h"
+
 #define DEBUG_TYPE "jitlink"
 
 using namespace llvm;
@@ -611,6 +613,11 @@ void link_ELF_aarch64(std::unique_ptr<LinkGraph> G,
     else
       Config.PrePrunePasses.push_back(markAllSymbolsLive);
 
+    // Resolve any external section start / end symbols.
+    Config.PostAllocationPasses.push_back(
+        createDefineExternalSectionStartAndEndSymbolsPass(
+            identifyELFSectionStartAndEndSymbols));
+
     // Add an in-place GOT/TLS/Stubs build pass.
     Config.PostPrunePasses.push_back(buildTables_ELF_aarch64);
   }
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
index a1fe9c5fcd73..52dd83d9040f 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
@@ -343,24 +343,6 @@ createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) {
       .buildGraph();
 }
 
-static SectionRangeSymbolDesc
-identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
-  constexpr StringRef StartSymbolPrefix = "__start";
-  constexpr StringRef EndSymbolPrefix = "__end";
-
-  auto SymName = Sym.getName();
-  if (SymName.starts_with(StartSymbolPrefix)) {
-    if (auto *Sec =
-            G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size())))
-      return {*Sec, true};
-  } else if (SymName.starts_with(EndSymbolPrefix)) {
-    if (auto *Sec =
-            G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size())))
-      return {*Sec, false};
-  }
-  return {};
-}
-
 void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
                      std::unique_ptr<JITLinkContext> Ctx) {
   PassConfiguration Config;
diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s
new file mode 100644
index 000000000000..f8e7ba96f006
--- /dev/null
+++ b/llvm/test/ExecutionEngine/JITLink/AArch64/ELF_section_start_and_stop_symbols.s
@@ -0,0 +1,43 @@
+# RUN: llvm-mc -triple=aarch64-unknown-linux-gnu -position-independent \
+# RUN:   -filetype=obj -o %t.o %s
+# RUN: llvm-jitlink -noexec -check %s %t.o
+
+	.text
+	.file	"elf_section_start_stop.c"
+	.globl	main
+	.p2align	2
+	.type	main,@function
+main:
+	adrp	x8, z
+	adrp	x9, y
+	ldr	w8, [x8, :lo12:z]
+	ldr	w9, [x9, :lo12:y]
+	sub	w0, w8, w9
+	ret
+.Lfunc_end0:
+	.size	main, .Lfunc_end0-main
+
+	.type	x,@object
+	.section	custom_section,"aw",@progbits
+	.globl	x
+	.p2align	2
+x:
+	.word	42
+	.size	x, 4
+
+# jitlink-check: *{8}z = (*{8}y) + 4
+
+	.type	y,@object
+	.data
+	.globl	y
+	.p2align	3, 0x0
+y:
+	.xword	__start_custom_section
+	.size	y, 8
+
+	.type	z,@object
+	.globl	z
+	.p2align	3, 0x0
+z:
+	.xword	__stop_custom_section
+	.size	z, 8
-- 
GitLab


From 9f0321ccf118b37e5cb93cabd2acbf600c36b6ee Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Sun, 24 Mar 2024 15:23:50 -0500
Subject: [PATCH 075/404] [Libomptarget] Make plugins depend explicitly on
 `intrinsics_gen`

Summary:
It's possible for the OpenMP offloading plugins to be build before
tablegen is run despite the fact that we rely on it. Simply make it
depend on it currently.
---
 openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt
index b84f3d7b137c..a7350e662a7c 100644
--- a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt
+++ b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt
@@ -19,6 +19,7 @@ add_library(PluginCommon OBJECT
   src/RPC.cpp
   src/Utils/ELF.cpp
 )
+add_dependencies(PluginCommon intrinsics_gen)
 
 # Only enable JIT for those targets that LLVM can support.
 string(TOUPPER "${LLVM_TARGETS_TO_BUILD}" TargetsSupported)
-- 
GitLab


From 81e2693c1202d6c4e48dbf2d985153b03cfccb79 Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990@gmail.com>
Date: Sun, 24 Mar 2024 20:52:47 +0000
Subject: [PATCH 076/404] [libc++][test] Fix race condition in
 condition_variable_any tests (#84788)

Some tests in `condition_variable_any` use two `shared_lock` to guard,
which does not work.
The fix is to make the writer to use `unique_lock`
---
 .../thread.condition.condvarany/wait_for_token_pred.pass.cpp    | 2 +-
 .../thread.condition.condvarany/wait_token_pred.pass.cpp        | 2 +-
 .../thread.condition.condvarany/wait_until_token_pred.pass.cpp  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp
index 4ea60557d9f8..7a39d1253a33 100644
--- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp
+++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_for_token_pred.pass.cpp
@@ -119,7 +119,7 @@ void test() {
     bool flag   = false;
     auto thread = support::make_test_thread([&]() {
       std::this_thread::sleep_for(2ms);
-      Lock lock2{mutex};
+      std::unique_lock<Mutex> lock2{mutex};
       flag = true;
       cv.notify_all();
     });
diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp
index e96a3e8bd1bc..f322d8cfdc68 100644
--- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp
+++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_token_pred.pass.cpp
@@ -63,7 +63,7 @@ void test() {
     bool flag   = false;
     auto thread = support::make_test_thread([&]() {
       std::this_thread::sleep_for(std::chrono::milliseconds(2));
-      Lock lock2{mutex};
+      std::unique_lock<Mutex> lock2{mutex};
       flag = true;
       cv.notify_all();
     });
diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp
index d649db025d75..e7388b9ce0e1 100644
--- a/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp
+++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvarany/wait_until_token_pred.pass.cpp
@@ -119,7 +119,7 @@ void test() {
     bool flag   = false;
     auto thread = support::make_test_thread([&]() {
       std::this_thread::sleep_for(std::chrono::milliseconds(2));
-      Lock lock2{mutex};
+      std::unique_lock<Mutex> lock2{mutex};
       flag = true;
       cv.notify_all();
     });
-- 
GitLab


From f0a8738401137c32f3e1b9799244ed643feb7bf0 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sun, 24 Mar 2024 20:59:53 +0000
Subject: [PATCH 077/404] [VPlan] Generate CalculateTripCountMinusVF for Part 0
 only. (NFCI).

The value produced by CalculateTripCountMinusVF VPInstructions is
independent of the part. Only compute it for part 0 and use that for
other parts.
---
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp |  3 +
 .../AArch64/scalable-strict-fadd.ll           | 63 +++----------------
 .../AArch64/sve-tail-folding-unroll.ll        | 42 ++-----------
 .../AArch64/uniform-args-call-variants.ll     | 13 +---
 4 files changed, 21 insertions(+), 100 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index d75e322a74cf..f91997a37993 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -345,6 +345,9 @@ Value *VPInstruction::generateInstruction(VPTransformState &State,
     return Builder.CreateVectorSplice(PartMinus1, V2, -1, Name);
   }
   case VPInstruction::CalculateTripCountMinusVF: {
+    if (Part != 0)
+      return State.get(this, 0, /*IsScalar*/ true);
+
     Value *ScalarTC = State.get(getOperand(0), {0, 0});
     Value *Step =
         createStepForVF(Builder, ScalarTC->getType(), State.VF, State.UF);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
index fc67fb5aded6..ad6e8534f318 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
@@ -403,21 +403,6 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 32
-; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = sub i64 [[N]], [[TMP11]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = icmp ugt i64 [[N]], [[TMP11]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 32
-; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = sub i64 [[N]], [[TMP16]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = icmp ugt i64 [[N]], [[TMP16]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 32
-; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = sub i64 [[N]], [[TMP21]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = icmp ugt i64 [[N]], [[TMP21]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0
 ; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-ORDERED-TF-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 8
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]]
@@ -492,9 +477,9 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[TMP78:%.*]] = mul i64 [[TMP77]], 24
 ; CHECK-ORDERED-TF-NEXT:    [[TMP79:%.*]] = add i64 [[INDEX]], [[TMP78]]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT12]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP73]], i64 [[TMP14]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT13]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP76]], i64 [[TMP19]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT14]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP79]], i64 [[TMP24]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT12]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP73]], i64 [[TMP9]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT13]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP76]], i64 [[TMP9]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT14]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP79]], i64 [[TMP9]])
 ; CHECK-ORDERED-TF-NEXT:    [[TMP80:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
 ; CHECK-ORDERED-TF-NEXT:    [[TMP81:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT12]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
 ; CHECK-ORDERED-TF-NEXT:    [[TMP82:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT13]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
@@ -1715,21 +1700,6 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 32
-; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = sub i64 [[N]], [[TMP11]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = icmp ugt i64 [[N]], [[TMP11]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 32
-; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = sub i64 [[N]], [[TMP16]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = icmp ugt i64 [[N]], [[TMP16]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 32
-; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = sub i64 [[N]], [[TMP21]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = icmp ugt i64 [[N]], [[TMP21]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0
 ; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-ORDERED-TF-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 8
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]]
@@ -1826,9 +1796,9 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[TMP96:%.*]] = mul i64 [[TMP95]], 24
 ; CHECK-ORDERED-TF-NEXT:    [[TMP97:%.*]] = add i64 [[INDEX]], [[TMP96]]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP14]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT17]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP19]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT18]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP24]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP9]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT17]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP9]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT18]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP9]])
 ; CHECK-ORDERED-TF-NEXT:    [[TMP98:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
 ; CHECK-ORDERED-TF-NEXT:    [[TMP99:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT16]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
 ; CHECK-ORDERED-TF-NEXT:    [[TMP100:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT17]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
@@ -2129,21 +2099,6 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 32
-; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = sub i64 [[N]], [[TMP11]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = icmp ugt i64 [[N]], [[TMP11]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 32
-; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = sub i64 [[N]], [[TMP16]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = icmp ugt i64 [[N]], [[TMP16]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 32
-; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = sub i64 [[N]], [[TMP21]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = icmp ugt i64 [[N]], [[TMP21]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0
 ; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-ORDERED-TF-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 8
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]]
@@ -2240,9 +2195,9 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[TMP96:%.*]] = mul i64 [[TMP95]], 24
 ; CHECK-ORDERED-TF-NEXT:    [[TMP97:%.*]] = add i64 [[INDEX]], [[TMP96]]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP14]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT17]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP19]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT18]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP24]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP91]], i64 [[TMP9]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT17]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP94]], i64 [[TMP9]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT18]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP97]], i64 [[TMP9]])
 ; CHECK-ORDERED-TF-NEXT:    [[TMP98:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
 ; CHECK-ORDERED-TF-NEXT:    [[TMP99:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT16]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
 ; CHECK-ORDERED-TF-NEXT:    [[TMP100:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT17]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll
index 1a6e83a61ce7..2acc1ddbffea 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll
@@ -25,21 +25,6 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
-; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 16
-; CHECK-NEXT:    [[TMP12:%.*]] = sub i64 [[UMAX]], [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp ugt i64 [[UMAX]], [[TMP11]]
-; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
-; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 16
-; CHECK-NEXT:    [[TMP17:%.*]] = sub i64 [[UMAX]], [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = icmp ugt i64 [[UMAX]], [[TMP16]]
-; CHECK-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0
-; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 16
-; CHECK-NEXT:    [[TMP22:%.*]] = sub i64 [[UMAX]], [[TMP21]]
-; CHECK-NEXT:    [[TMP23:%.*]] = icmp ugt i64 [[UMAX]], [[TMP21]]
-; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0
 ; CHECK-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 4
 ; CHECK-NEXT:    [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]]
@@ -107,9 +92,9 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    [[TMP70:%.*]] = mul i64 [[TMP69]], 12
 ; CHECK-NEXT:    [[TMP71:%.*]] = add i64 [[INDEX6]], [[TMP70]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX6]], i64 [[TMP9]])
-; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT11]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP65]], i64 [[TMP14]])
-; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT12]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP68]], i64 [[TMP19]])
-; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT13]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP71]], i64 [[TMP24]])
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT11]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP65]], i64 [[TMP9]])
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT12]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP68]], i64 [[TMP9]])
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT13]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP71]], i64 [[TMP9]])
 ; CHECK-NEXT:    [[TMP72:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[TMP73:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT11]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[TMP74:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT12]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)
@@ -167,21 +152,6 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias %
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
-; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 16
-; CHECK-NEXT:    [[TMP12:%.*]] = sub i64 [[UMAX]], [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp ugt i64 [[UMAX]], [[TMP11]]
-; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
-; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 16
-; CHECK-NEXT:    [[TMP17:%.*]] = sub i64 [[UMAX]], [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = icmp ugt i64 [[UMAX]], [[TMP16]]
-; CHECK-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 0
-; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 16
-; CHECK-NEXT:    [[TMP22:%.*]] = sub i64 [[UMAX]], [[TMP21]]
-; CHECK-NEXT:    [[TMP23:%.*]] = icmp ugt i64 [[UMAX]], [[TMP21]]
-; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i64 [[TMP22]], i64 0
 ; CHECK-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 4
 ; CHECK-NEXT:    [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]]
@@ -275,9 +245,9 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias %
 ; CHECK-NEXT:    [[TMP92:%.*]] = mul i64 [[TMP91]], 12
 ; CHECK-NEXT:    [[TMP93:%.*]] = add i64 [[INDEX6]], [[TMP92]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX6]], i64 [[TMP9]])
-; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT14]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP87]], i64 [[TMP14]])
-; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT15]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP90]], i64 [[TMP19]])
-; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP93]], i64 [[TMP24]])
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT14]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP87]], i64 [[TMP9]])
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT15]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP90]], i64 [[TMP9]])
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP93]], i64 [[TMP9]])
 ; CHECK-NEXT:    [[TMP94:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[TMP95:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT14]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[TMP96:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT15]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
index 4957bbeda671..d8f14f30295b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
@@ -40,9 +40,6 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6
 ; INTERLEAVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; INTERLEAVE-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 2
 ; INTERLEAVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP3]])
-; INTERLEAVE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP5]], 2
-; INTERLEAVE-NEXT:    [[TMP7:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP6]])
 ; INTERLEAVE-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
 ; INTERLEAVE-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP8]], 1
 ; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
@@ -71,7 +68,7 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6
 ; INTERLEAVE-NEXT:    [[TMP21:%.*]] = shl i64 [[TMP20]], 1
 ; INTERLEAVE-NEXT:    [[TMP22:%.*]] = add i64 [[INDEX]], [[TMP21]]
 ; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP4]])
-; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP7]])
+; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP4]])
 ; INTERLEAVE-NEXT:    [[TMP23:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
 ; INTERLEAVE-NEXT:    br i1 [[TMP23]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
 ; INTERLEAVE:       for.cond.cleanup:
@@ -129,9 +126,6 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3
 ; INTERLEAVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; INTERLEAVE-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 2
 ; INTERLEAVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP3]])
-; INTERLEAVE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP5]], 2
-; INTERLEAVE-NEXT:    [[TMP7:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP6]])
 ; INTERLEAVE-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
 ; INTERLEAVE-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP8]], 1
 ; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
@@ -160,7 +154,7 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3
 ; INTERLEAVE-NEXT:    [[TMP21:%.*]] = shl i64 [[TMP20]], 1
 ; INTERLEAVE-NEXT:    [[TMP22:%.*]] = add i64 [[INDEX]], [[TMP21]]
 ; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP4]])
-; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP7]])
+; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP22]], i64 [[TMP4]])
 ; INTERLEAVE-NEXT:    [[TMP23:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
 ; INTERLEAVE-NEXT:    br i1 [[TMP23]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP3:![0-9]+]]
 ; INTERLEAVE:       for.cond.cleanup:
@@ -207,7 +201,6 @@ define void @test_uniform_not_invariant(ptr noalias %dst, ptr readonly %src, i64
 ; INTERLEAVE-SAME: (ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; INTERLEAVE-NEXT:  entry:
 ; INTERLEAVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 2)
-; INTERLEAVE-NEXT:    [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 2)
 ; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ne i64 [[N]], 0
 ; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ugt i64 [[N]], 1
 ; INTERLEAVE-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -237,7 +230,7 @@ define void @test_uniform_not_invariant(ptr noalias %dst, ptr readonly %src, i64
 ; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
 ; INTERLEAVE-NEXT:    [[TMP11:%.*]] = or disjoint i64 [[INDEX]], 1
 ; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX]], [[TMP0]]
-; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP11]], [[TMP1]]
+; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT5]] = icmp ult i64 [[TMP11]], [[TMP0]]
 ; INTERLEAVE-NEXT:    br i1 [[ACTIVE_LANE_MASK_NEXT]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP4:![0-9]+]]
 ; INTERLEAVE:       for.cond.cleanup:
 ; INTERLEAVE-NEXT:    ret void
-- 
GitLab


From 18a49f03aa2b6bfeb073648b9eb75277a2386fc4 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 24 Mar 2024 14:07:09 -0700
Subject: [PATCH 078/404] [ELF] Merge relaIplt into relaDyn

`relaIplt` was added so that IRELATIVE relocations are placed at the end
of .rela.dyn (since https://reviews.llvm.org/D65651) or .rela.plt
(--pack-dyn-relocs=android[+relr]). Unfortunately, handling `relaIplt`
requires special cases all over the code base. We can extend
partitionRels/computeRels to partition both RELATIVE and IRELATIVE
relocations, rendering `relaIplt` unneeded.

The change allows IRELATIVE relocations in the DT_ANDROID_REL[A] table
(untested?!), which may be processed before other types of relocations.
This seems acceptable for Bionic's DEFINE_IFUNC_FOR use cases.

In addition, this change simplies changing .rel[a].dyn to a compact
relocation format (CREL).

SHF_INFO_LINK is removed from .rel[a].dyn with IRELATIVE relocations.
(See https://reviews.llvm.org/D89828).
---
 lld/ELF/Relocations.cpp                     | 10 ++---
 lld/ELF/SyntheticSections.cpp               | 19 +++-----
 lld/ELF/SyntheticSections.h                 |  1 -
 lld/ELF/Writer.cpp                          | 50 ++++++---------------
 lld/test/ELF/aarch64-gnu-ifunc.s            |  3 +-
 lld/test/ELF/arm-gnu-ifunc.s                |  3 +-
 lld/test/ELF/gnu-ifunc-i386.s               |  3 +-
 lld/test/ELF/systemz-ifunc-nonpreemptible.s |  2 +-
 8 files changed, 28 insertions(+), 63 deletions(-)

diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index 92a1b9baaca3..33c50133bec4 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -1620,12 +1620,8 @@ static bool handleNonPreemptibleIfunc(Symbol &sym, uint16_t flags) {
   //   relatively straightforward. We create a PLT entry in Iplt, which is
   //   usually at the end of .plt, which makes an indirect call using a
   //   matching GOT entry in igotPlt, which is usually at the end of .got.plt.
-  //   The GOT entry is relocated using an IRELATIVE relocation in relaIplt,
-  //   which is usually at the end of .rela.plt. Unlike most relocations in
-  //   .rela.plt, which may be evaluated lazily without -z now, dynamic
-  //   loaders evaluate IRELATIVE relocs eagerly, which means that for
-  //   IRELATIVE relocs only, GOT-generating relocations can point directly to
-  //   .got.plt without requiring a separate GOT entry.
+  //   The GOT entry is relocated using an IRELATIVE relocation in relaDyn,
+  //   which is usually at the end of .rela.dyn.
   //
   // - Despite the fact that an ifunc does not have a fixed value, compilers
   //   that are not passed -fPIC will assume that they do, and will emit
@@ -1665,7 +1661,7 @@ static bool handleNonPreemptibleIfunc(Symbol &sym, uint16_t flags) {
   // section/value fixed.
   auto *directSym = makeDefined(cast<Defined>(sym));
   directSym->allocateAux();
-  addPltEntry(*in.iplt, *in.igotPlt, *in.relaIplt, target->iRelativeRel,
+  addPltEntry(*in.iplt, *in.igotPlt, *mainPart->relaDyn, target->iRelativeRel,
               *directSym);
   sym.allocateAux();
   symAux.back().pltIdx = symAux[directSym->auxIdx].pltIdx;
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index 10eda17f0cb3..f924756ddddf 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -1267,15 +1267,12 @@ DynamicSection<ELFT>::DynamicSection()
 // The output section .rela.dyn may include these synthetic sections:
 //
 // - part.relaDyn
-// - in.relaIplt: this is included if in.relaIplt is named .rela.dyn
 // - in.relaPlt: this is included if a linker script places .rela.plt inside
 //   .rela.dyn
 //
 // DT_RELASZ is the total size of the included sections.
 static uint64_t addRelaSz(const RelocationBaseSection &relaDyn) {
   size_t size = relaDyn.getSize();
-  if (in.relaIplt->getParent() == relaDyn.getParent())
-    size += in.relaIplt->getSize();
   if (in.relaPlt->getParent() == relaDyn.getParent())
     size += in.relaPlt->getSize();
   return size;
@@ -1372,9 +1369,7 @@ DynamicSection<ELFT>::computeContents() {
   if (!config->shared && !config->relocatable && !config->zRodynamic)
     addInt(DT_DEBUG, 0);
 
-  if (part.relaDyn->isNeeded() ||
-      (in.relaIplt->isNeeded() &&
-       part.relaDyn->getParent() == in.relaIplt->getParent())) {
+  if (part.relaDyn->isNeeded()) {
     addInSec(part.relaDyn->dynamicTag, *part.relaDyn);
     entries.emplace_back(part.relaDyn->sizeDynamicTag,
                          addRelaSz(*part.relaDyn));
@@ -1657,10 +1652,6 @@ void RelocationBaseSection::finalizeContents() {
     getParent()->flags |= ELF::SHF_INFO_LINK;
     getParent()->info = in.gotPlt->getParent()->sectionIndex;
   }
-  if (in.relaIplt.get() == this && in.igotPlt->getParent()) {
-    getParent()->flags |= ELF::SHF_INFO_LINK;
-    getParent()->info = in.igotPlt->getParent()->sectionIndex;
-  }
 }
 
 void DynamicReloc::computeRaw(SymbolTableBaseSection *symtab) {
@@ -1674,6 +1665,11 @@ void RelocationBaseSection::computeRels() {
   SymbolTableBaseSection *symTab = getPartition().dynSymTab.get();
   parallelForEach(relocs,
                   [symTab](DynamicReloc &rel) { rel.computeRaw(symTab); });
+
+  auto irelative = std::partition(
+      relocs.begin() + numRelativeRelocs, relocs.end(),
+      [t = target->iRelativeRel](auto &r) { return r.type != t; });
+
   // Sort by (!IsRelative,SymIndex,r_offset). DT_REL[A]COUNT requires us to
   // place R_*_RELATIVE first. SymIndex is to improve locality, while r_offset
   // is to make results easier to read.
@@ -1682,7 +1678,7 @@ void RelocationBaseSection::computeRels() {
     parallelSort(relocs.begin(), nonRelative,
                  [&](auto &a, auto &b) { return a.r_offset < b.r_offset; });
     // Non-relative relocations are few, so don't bother with parallelSort.
-    llvm::sort(nonRelative, relocs.end(), [&](auto &a, auto &b) {
+    llvm::sort(nonRelative, irelative, [&](auto &a, auto &b) {
       return std::tie(a.r_sym, a.r_offset) < std::tie(b.r_sym, b.r_offset);
     });
   }
@@ -3843,7 +3839,6 @@ void InStruct::reset() {
   ppc32Got2.reset();
   ibtPlt.reset();
   relaPlt.reset();
-  relaIplt.reset();
   shStrTab.reset();
   strTab.reset();
   symTab.reset();
diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h
index b41e69410054..fa21b80a5a5e 100644
--- a/lld/ELF/SyntheticSections.h
+++ b/lld/ELF/SyntheticSections.h
@@ -1358,7 +1358,6 @@ struct InStruct {
   std::unique_ptr<PPC32Got2Section> ppc32Got2;
   std::unique_ptr<IBTPltSection> ibtPlt;
   std::unique_ptr<RelocationBaseSection> relaPlt;
-  std::unique_ptr<RelocationBaseSection> relaIplt;
   std::unique_ptr<StringTableSection> shStrTab;
   std::unique_ptr<StringTableSection> strTab;
   std::unique_ptr<SymbolTableBaseSection> symTab;
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index 4eca7b22e90b..40d617b7fdf3 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -449,8 +449,8 @@ template <class ELFT> void elf::createSyntheticSections() {
 
       add(*part.dynamic);
       add(*part.dynStrTab);
-      add(*part.relaDyn);
     }
+    add(*part.relaDyn);
 
     if (config->relrPackDynRelocs) {
       part.relrDyn = std::make_unique<RelrSection<ELFT>>(threadCount);
@@ -550,17 +550,6 @@ template <class ELFT> void elf::createSyntheticSections() {
       /*threadCount=*/1);
   add(*in.relaPlt);
 
-  // The relaIplt immediately follows .rel[a].dyn to ensure that the IRelative
-  // relocations are processed last by the dynamic loader. We cannot place the
-  // iplt section in .rel.dyn when Android relocation packing is enabled because
-  // that would cause a section type mismatch. However, because the Android
-  // dynamic loader reads .rel.plt after .rel.dyn, we can get the desired
-  // behaviour by placing the iplt section in .rel.plt.
-  in.relaIplt = std::make_unique<RelocationSection<ELFT>>(
-      config->androidPackDynRelocs ? in.relaPlt->name : relaDynName,
-      /*sort=*/false, /*threadCount=*/1);
-  add(*in.relaIplt);
-
   if ((config->emachine == EM_386 || config->emachine == EM_X86_64) &&
       (config->andFeatures & GNU_PROPERTY_X86_FEATURE_1_IBT)) {
     in.ibtPlt = std::make_unique<IBTPltSection>();
@@ -1071,20 +1060,18 @@ void PhdrEntry::add(OutputSection *sec) {
     sec->ptLoad = this;
 }
 
-// The beginning and the ending of .rel[a].plt section are marked
-// with __rel[a]_iplt_{start,end} symbols if it is a statically linked
-// executable. The runtime needs these symbols in order to resolve
-// all IRELATIVE relocs on startup. For dynamic executables, we don't
-// need these symbols, since IRELATIVE relocs are resolved through GOT
-// and PLT. For details, see http://www.airs.com/blog/archives/403.
+// A statically linked position-dependent executable should only contain
+// IRELATIVE relocations and no other dynamic relocations. Encapsulation symbols
+// __rel[a]_iplt_{start,end} will be defined for .rel[a].dyn, to be
+// processed by the libc runtime. Other executables or DSOs use dynamic tags
+// instead.
 template <class ELFT> void Writer<ELFT>::addRelIpltSymbols() {
   if (config->isPic)
     return;
 
-  // By default, __rela_iplt_{start,end} belong to a dummy section 0
-  // because .rela.plt might be empty and thus removed from output.
-  // We'll override Out::elfHeader with In.relaIplt later when we are
-  // sure that .rela.plt exists in output.
+  // __rela_iplt_{start,end} are initially defined relative to dummy section 0.
+  // We'll override Out::elfHeader with relaDyn later when we are sure that
+  // .rela.dyn will be present in the output.
   ElfSym::relaIpltStart = addOptionalRegular(
       config->isRela ? "__rela_iplt_start" : "__rel_iplt_start",
       Out::elfHeader, 0, STV_HIDDEN);
@@ -1110,11 +1097,11 @@ template <class ELFT> void Writer<ELFT>::setReservedSymbolSections() {
     ElfSym::globalOffsetTable->section = sec;
   }
 
-  // .rela_iplt_{start,end} mark the start and the end of in.relaIplt.
-  if (ElfSym::relaIpltStart && in.relaIplt->isNeeded()) {
-    ElfSym::relaIpltStart->section = in.relaIplt.get();
-    ElfSym::relaIpltEnd->section = in.relaIplt.get();
-    ElfSym::relaIpltEnd->value = in.relaIplt->getSize();
+  // .rela_iplt_{start,end} mark the start and the end of .rel[a].dyn.
+  if (ElfSym::relaIpltStart && mainPart->relaDyn->isNeeded()) {
+    ElfSym::relaIpltStart->section = mainPart->relaDyn.get();
+    ElfSym::relaIpltEnd->section = mainPart->relaDyn.get();
+    ElfSym::relaIpltEnd->value = mainPart->relaDyn->getSize();
   }
 
   PhdrEntry *last = nullptr;
@@ -1470,14 +1457,6 @@ static void sortSection(OutputSection &osec,
   if (name == ".init" || name == ".fini")
     return;
 
-  // IRelative relocations that usually live in the .rel[a].dyn section should
-  // be processed last by the dynamic loader. To achieve that we add synthetic
-  // sections in the required order from the beginning so that the in.relaIplt
-  // section is placed last in an output section. Here we just do not apply
-  // sorting for an output section which holds the in.relaIplt section.
-  if (in.relaIplt->getParent() == &osec)
-    return;
-
   // Sort input sections by priority using the list provided by
   // --symbol-ordering-file or --shuffle-sections=. This is a least significant
   // digit radix sort. The sections may be sorted stably again by a more
@@ -2196,7 +2175,6 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() {
     finalizeSynthetic(in.mipsGot.get());
     finalizeSynthetic(in.igotPlt.get());
     finalizeSynthetic(in.gotPlt.get());
-    finalizeSynthetic(in.relaIplt.get());
     finalizeSynthetic(in.relaPlt.get());
     finalizeSynthetic(in.plt.get());
     finalizeSynthetic(in.iplt.get());
diff --git a/lld/test/ELF/aarch64-gnu-ifunc.s b/lld/test/ELF/aarch64-gnu-ifunc.s
index dee24779d913..d76b54eabf8a 100644
--- a/lld/test/ELF/aarch64-gnu-ifunc.s
+++ b/lld/test/ELF/aarch64-gnu-ifunc.s
@@ -11,13 +11,12 @@
 // CHECK-NEXT:  Type: SHT_RELA
 // CHECK-NEXT:  Flags [
 // CHECK-NEXT:    SHF_ALLOC
-// CHECK-NEXT:    SHF_INFO_LINK
 // CHECK-NEXT:  ]
 // CHECK-NEXT:  Address: [[RELA:.*]]
 // CHECK-NEXT:  Offset: 0x158
 // CHECK-NEXT:  Size: 48
 // CHECK-NEXT:  Link: 0
-// CHECK-NEXT:  Info: 4
+// CHECK-NEXT:  Info: 0
 // CHECK-NEXT:  AddressAlignment: 8
 // CHECK-NEXT:  EntrySize: 24
 // CHECK-NEXT: }
diff --git a/lld/test/ELF/arm-gnu-ifunc.s b/lld/test/ELF/arm-gnu-ifunc.s
index 562478256fd8..d49ca18e991e 100644
--- a/lld/test/ELF/arm-gnu-ifunc.s
+++ b/lld/test/ELF/arm-gnu-ifunc.s
@@ -30,13 +30,12 @@ _start:
 // CHECK-NEXT:     Type: SHT_REL
 // CHECK-NEXT:     Flags [
 // CHECK-NEXT:       SHF_ALLOC
-// CHECK-NEXT:       SHF_INFO_LINK
 // CHECK-NEXT:     ]
 // CHECK-NEXT:     Address: 0x100F4
 // CHECK-NEXT:     Offset: 0xF4
 // CHECK-NEXT:     Size: 16
 // CHECK-NEXT:     Link:
-// CHECK-NEXT:     Info: 4
+// CHECK-NEXT:     Info: 0
 // CHECK:          Name: .iplt
 // CHECK-NEXT:     Type: SHT_PROGBITS
 // CHECK-NEXT:     Flags [
diff --git a/lld/test/ELF/gnu-ifunc-i386.s b/lld/test/ELF/gnu-ifunc-i386.s
index b502fd6e9ae2..43b19b27ea4e 100644
--- a/lld/test/ELF/gnu-ifunc-i386.s
+++ b/lld/test/ELF/gnu-ifunc-i386.s
@@ -11,13 +11,12 @@
 // CHECK-NEXT:  Type: SHT_REL
 // CHECK-NEXT:  Flags [
 // CHECK-NEXT:    SHF_ALLOC
-// CHECK-NEXT:    SHF_INFO_LINK
 // CHECK-NEXT:  ]
 // CHECK-NEXT:  Address: [[RELA:.*]]
 // CHECK-NEXT:  Offset: 0xD4
 // CHECK-NEXT:  Size: 16
 // CHECK-NEXT:  Link: 0
-// CHECK-NEXT:  Info: 4
+// CHECK-NEXT:  Info: 0
 // CHECK-NEXT:  AddressAlignment: 4
 // CHECK-NEXT:  EntrySize: 8
 // CHECK-NEXT: }
diff --git a/lld/test/ELF/systemz-ifunc-nonpreemptible.s b/lld/test/ELF/systemz-ifunc-nonpreemptible.s
index 5056db302ca1..892bbde8d9c7 100644
--- a/lld/test/ELF/systemz-ifunc-nonpreemptible.s
+++ b/lld/test/ELF/systemz-ifunc-nonpreemptible.s
@@ -10,7 +10,7 @@
 # CHECK:      Section Headers:
 # CHECK-NEXT:  [Nr] Name              Type            Address          Off    Size   ES Flg Lk Inf Al
 # CHECK-NEXT:  [ 0]                   NULL            0000000000000000 000000 000000 00      0   0  0
-# CHECK-NEXT:  [ 1] .rela.dyn         RELA            0000000001000158 000158 000030 18  AI  0   4  8
+# CHECK-NEXT:  [ 1] .rela.dyn         RELA            0000000001000158 000158 000030 18   A  0   0  8
 # CHECK-NEXT:  [ 2] .text             PROGBITS        0000000001001188 000188 00001c 00  AX  0   0  4
 # CHECK-NEXT:  [ 3] .iplt             PROGBITS        00000000010011b0 0001b0 000040 00  AX  0   0 16
 # CHECK-NEXT:  [ 4] .got.plt          PROGBITS        00000000010021f0 0001f0 000010 00  WA  0   0  8
-- 
GitLab


From 67f2267ae006f35edfdf36567646403e61527d1b Mon Sep 17 00:00:00 2001
From: Min Hsu <min@myhsu.dev>
Date: Sun, 24 Mar 2024 14:24:44 -0700
Subject: [PATCH 079/404] [M68k][NFC] Suppress warning on an unused variable
 when assertion is disabled

NFC.
---
 llvm/lib/Target/M68k/M68kISelLowering.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/lib/Target/M68k/M68kISelLowering.cpp b/llvm/lib/Target/M68k/M68kISelLowering.cpp
index 158393f02a24..786aa7bcb64e 100644
--- a/llvm/lib/Target/M68k/M68kISelLowering.cpp
+++ b/llvm/lib/Target/M68k/M68kISelLowering.cpp
@@ -939,6 +939,7 @@ SDValue M68kTargetLowering::LowerFormalArguments(
   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
     CCValAssign &VA = ArgLocs[i];
     assert(VA.getValNo() != LastVal && "Same value in different locations");
+    (void)LastVal;
 
     LastVal = VA.getValNo();
 
-- 
GitLab


From cceedc939a43c7c732a5888364251775bffc2dba Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Sun, 24 Mar 2024 15:22:40 -0700
Subject: [PATCH 080/404] [clang-format] Fix a crash with
 AlignArrayOfStructures option (#86420)

Fixes #86109.
---
 clang/lib/Format/WhitespaceManager.cpp |  2 +-
 clang/unittests/Format/FormatTest.cpp  | 18 ++++++++++++++++--
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp
index fef85abf79a3..710bf8d8a8ec 100644
--- a/clang/lib/Format/WhitespaceManager.cpp
+++ b/clang/lib/Format/WhitespaceManager.cpp
@@ -1491,7 +1491,7 @@ WhitespaceManager::CellDescriptions WhitespaceManager::getCells(unsigned Start,
                                                                 : Cell);
         // Go to the next non-comment and ensure there is a break in front
         const auto *NextNonComment = C.Tok->getNextNonComment();
-        while (NextNonComment->is(tok::comma))
+        while (NextNonComment && NextNonComment->is(tok::comma))
           NextNonComment = NextNonComment->getNextNonComment();
         auto j = i;
         while (j < End && Changes[j].Tok != NextNonComment)
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index cf8d6ab691d9..03005384a6f6 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -21100,7 +21100,14 @@ TEST_F(FormatTest, CatchAlignArrayOfStructuresRightAlignment) {
                 "    [0] = {1, 1},\n"
                 "    [1] { 1, 1, },\n"
                 "    [2] { 1, 1, },\n"
-                "};");
+                "};",
+                Style);
+  verifyNoCrash("test arr[] = {\n"
+                "#define FOO(i) {i, i},\n"
+                "SOME_GENERATOR(FOO)\n"
+                "{2, 2}\n"
+                "};",
+                Style);
 
   verifyFormat("return GradForUnaryCwise(g, {\n"
                "                                {{\"sign\"}, \"Sign\",  "
@@ -21353,7 +21360,14 @@ TEST_F(FormatTest, CatchAlignArrayOfStructuresLeftAlignment) {
                 "    [0] = {1, 1},\n"
                 "    [1] { 1, 1, },\n"
                 "    [2] { 1, 1, },\n"
-                "};");
+                "};",
+                Style);
+  verifyNoCrash("test arr[] = {\n"
+                "#define FOO(i) {i, i},\n"
+                "SOME_GENERATOR(FOO)\n"
+                "{2, 2}\n"
+                "};",
+                Style);
 
   verifyFormat("return GradForUnaryCwise(g, {\n"
                "                                {{\"sign\"}, \"Sign\", {\"x\", "
-- 
GitLab


From 230b1895c493c511c11541af3b5bc819887c82a8 Mon Sep 17 00:00:00 2001
From: Finn Plummer <50529406+inbelic@users.noreply.github.com>
Date: Sun, 24 Mar 2024 18:56:47 -0700
Subject: [PATCH 081/404] [mlir][spirv] Add folding for [S|U|LessThan[Equal]
 (#85435)

Add missing constant propogation folder for [S|U]LessThan[Equal].

Implement additional folding when the operands are equal for all ops.

Allows for constant folding in the IndexToSPIRV pass.

Part of work #70704
---
 .../mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td  |   8 +
 .../SPIRV/IR/SPIRVCanonicalization.cpp        |  82 ++++++++
 .../SPIRV/Transforms/canonicalize.mlir        | 176 ++++++++++++++++++
 3 files changed, 266 insertions(+)

diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td
index 3ee239d6e1e3..14d639bc26f2 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td
@@ -716,6 +716,8 @@ def SPIRV_SLessThanOp : SPIRV_LogicalBinaryOp<"SLessThan",
 
     ```
   }];
+
+  let hasFolder = 1;
 }
 
 // -----
@@ -745,6 +747,8 @@ def SPIRV_SLessThanEqualOp : SPIRV_LogicalBinaryOp<"SLessThanEqual",
     %5 = spirv.SLessThanEqual %2, %3 : vector<4xi32>
     ```
   }];
+
+  let hasFolder = 1;
 }
 
 // -----
@@ -886,6 +890,8 @@ def SPIRV_ULessThanOp : SPIRV_LogicalBinaryOp<"ULessThan",
     %5 = spirv.ULessThan %2, %3 : vector<4xi32>
     ```
   }];
+
+  let hasFolder = 1;
 }
 
 // -----
@@ -949,6 +955,8 @@ def SPIRV_ULessThanEqualOp : SPIRV_LogicalBinaryOp<"ULessThanEqual",
     %5 = spirv.ULessThanEqual %2, %3 : vector<4xi32>
     ```
   }];
+
+  let hasFolder = 1;
 }
 
 #endif // MLIR_DIALECT_SPIRV_IR_LOGICAL_OPS
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
index ff4bace9a4d8..3f1a7826f8b6 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
@@ -880,6 +880,88 @@ OpFoldResult spirv::INotEqualOp::fold(spirv::INotEqualOp::FoldAdaptor adaptor) {
       });
 }
 
+//===----------------------------------------------------------------------===//
+// spirv.SLessThan
+//===----------------------------------------------------------------------===//
+
+OpFoldResult spirv::SLessThanOp::fold(spirv::SLessThanOp::FoldAdaptor adaptor) {
+  // x == x -> false
+  if (getOperand1() == getOperand2()) {
+    auto falseAttr = BoolAttr::get(getContext(), false);
+    if (isa<IntegerType>(getType()))
+      return falseAttr;
+    if (auto vecTy = dyn_cast<VectorType>(getType()))
+      return SplatElementsAttr::get(vecTy, falseAttr);
+  }
+
+  return constFoldBinaryOp<IntegerAttr>(
+      adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) {
+        return a.slt(b) ? APInt::getAllOnes(1) : APInt::getZero(1);
+      });
+}
+
+//===----------------------------------------------------------------------===//
+// spirv.SLessThanEqual
+//===----------------------------------------------------------------------===//
+
+OpFoldResult
+spirv::SLessThanEqualOp::fold(spirv::SLessThanEqualOp::FoldAdaptor adaptor) {
+  // x == x -> true
+  if (getOperand1() == getOperand2()) {
+    auto trueAttr = BoolAttr::get(getContext(), true);
+    if (isa<IntegerType>(getType()))
+      return trueAttr;
+    if (auto vecTy = dyn_cast<VectorType>(getType()))
+      return SplatElementsAttr::get(vecTy, trueAttr);
+  }
+
+  return constFoldBinaryOp<IntegerAttr>(
+      adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) {
+        return a.sle(b) ? APInt::getAllOnes(1) : APInt::getZero(1);
+      });
+}
+
+//===----------------------------------------------------------------------===//
+// spirv.ULessThan
+//===----------------------------------------------------------------------===//
+
+OpFoldResult spirv::ULessThanOp::fold(spirv::ULessThanOp::FoldAdaptor adaptor) {
+  // x == x -> false
+  if (getOperand1() == getOperand2()) {
+    auto falseAttr = BoolAttr::get(getContext(), false);
+    if (isa<IntegerType>(getType()))
+      return falseAttr;
+    if (auto vecTy = dyn_cast<VectorType>(getType()))
+      return SplatElementsAttr::get(vecTy, falseAttr);
+  }
+
+  return constFoldBinaryOp<IntegerAttr>(
+      adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) {
+        return a.ult(b) ? APInt::getAllOnes(1) : APInt::getZero(1);
+      });
+}
+
+//===----------------------------------------------------------------------===//
+// spirv.ULessThanEqual
+//===----------------------------------------------------------------------===//
+
+OpFoldResult
+spirv::ULessThanEqualOp::fold(spirv::ULessThanEqualOp::FoldAdaptor adaptor) {
+  // x == x -> true
+  if (getOperand1() == getOperand2()) {
+    auto trueAttr = BoolAttr::get(getContext(), true);
+    if (isa<IntegerType>(getType()))
+      return trueAttr;
+    if (auto vecTy = dyn_cast<VectorType>(getType()))
+      return SplatElementsAttr::get(vecTy, trueAttr);
+  }
+
+  return constFoldBinaryOp<IntegerAttr>(
+      adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) {
+        return a.ule(b) ? APInt::getAllOnes(1) : APInt::getZero(1);
+      });
+}
+
 //===----------------------------------------------------------------------===//
 // spirv.ShiftLeftLogical
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir
index de21d114e9fc..ed0bd070c43c 100644
--- a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir
+++ b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir
@@ -1478,6 +1478,182 @@ func.func @const_fold_vector_inotequal() -> vector<3xi1> {
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// spirv.SLessThan
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @slt_same
+func.func @slt_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) {
+  // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false
+  // CHECK-DAG: %[[CVFALSE:.*]] = spirv.Constant dense<false>
+  %0 = spirv.SLessThan %arg0, %arg0 : i32
+  %1 = spirv.SLessThan %arg1, %arg1 : vector<3xi32>
+
+  // CHECK: return %[[CFALSE]], %[[CVFALSE]]
+  return %0, %1 : i1, vector<3xi1>
+}
+
+// CHECK-LABEL: @const_fold_scalar_slt
+func.func @const_fold_scalar_slt() -> (i1, i1) {
+  %c4 = spirv.Constant 4 : i32
+  %c5 = spirv.Constant 5 : i32
+  %c6 = spirv.Constant 6 : i32
+
+  // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true
+  // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false
+  %0 = spirv.SLessThan %c5, %c6 : i32
+  %1 = spirv.SLessThan %c5, %c4 : i32
+
+  // CHECK: return %[[CTRUE]], %[[CFALSE]]
+  return %0, %1 : i1, i1
+}
+
+// CHECK-LABEL: @const_fold_vector_slt
+func.func @const_fold_vector_slt() -> vector<3xi1> {
+  %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32>
+  %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32>
+
+  // CHECK: %[[RET:.*]] = spirv.Constant dense<[false, true, false]>
+  %0 = spirv.SLessThan %cv0, %cv1 : vector<3xi32>
+
+  // CHECK: return %[[RET]]
+  return %0 : vector<3xi1>
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.SLessThanEqual
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @sle_same
+func.func @sle_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) {
+  // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true
+  // CHECK-DAG: %[[CVTRUE:.*]] = spirv.Constant dense<true>
+  %0 = spirv.SLessThanEqual %arg0, %arg0 : i32
+  %1 = spirv.SLessThanEqual %arg1, %arg1 : vector<3xi32>
+
+  // CHECK: return %[[CTRUE]], %[[CVTRUE]]
+  return %0, %1 : i1, vector<3xi1>
+}
+
+// CHECK-LABEL: @const_fold_scalar_sle
+func.func @const_fold_scalar_sle() -> (i1, i1) {
+  %c4 = spirv.Constant 4 : i32
+  %c5 = spirv.Constant 5 : i32
+  %c6 = spirv.Constant 6 : i32
+
+  // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true
+  // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false
+  %0 = spirv.SLessThanEqual %c5, %c6 : i32
+  %1 = spirv.SLessThanEqual %c5, %c4 : i32
+
+  // CHECK: return %[[CTRUE]], %[[CFALSE]]
+  return %0, %1 : i1, i1
+}
+
+// CHECK-LABEL: @const_fold_vector_sle
+func.func @const_fold_vector_sle() -> vector<3xi1> {
+  %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32>
+  %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32>
+
+  // CHECK: %[[RET:.*]] = spirv.Constant dense<[true, true, false]>
+  %0 = spirv.SLessThanEqual %cv0, %cv1 : vector<3xi32>
+
+  // CHECK: return %[[RET]]
+  return %0 : vector<3xi1>
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.ULessThan
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @ult_same
+func.func @ult_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) {
+  // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false
+  // CHECK-DAG: %[[CVFALSE:.*]] = spirv.Constant dense<false>
+  %0 = spirv.ULessThan %arg0, %arg0 : i32
+  %1 = spirv.ULessThan %arg1, %arg1 : vector<3xi32>
+
+  // CHECK: return %[[CFALSE]], %[[CVFALSE]]
+  return %0, %1 : i1, vector<3xi1>
+}
+
+// CHECK-LABEL: @const_fold_scalar_ult
+func.func @const_fold_scalar_ult() -> (i1, i1) {
+  %c4 = spirv.Constant 4 : i32
+  %c5 = spirv.Constant 5 : i32
+  %cn6 = spirv.Constant -6 : i32
+
+  // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true
+  // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false
+  %0 = spirv.ULessThan %c5, %cn6 : i32
+  %1 = spirv.ULessThan %c5, %c4 : i32
+
+  // CHECK: return %[[CTRUE]], %[[CFALSE]]
+  return %0, %1 : i1, i1
+}
+
+// CHECK-LABEL: @const_fold_vector_ult
+func.func @const_fold_vector_ult() -> vector<3xi1> {
+  %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32>
+  %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32>
+
+  // CHECK: %[[RET:.*]] = spirv.Constant dense<[false, true, false]>
+  %0 = spirv.ULessThan %cv0, %cv1 : vector<3xi32>
+
+  // CHECK: return %[[RET]]
+  return %0 : vector<3xi1>
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.ULessThanEqual
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @ule_same
+func.func @ule_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) {
+  // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true
+  // CHECK-DAG: %[[CVTRUE:.*]] = spirv.Constant dense<true>
+  %0 = spirv.ULessThanEqual %arg0, %arg0 : i32
+  %1 = spirv.ULessThanEqual %arg1, %arg1 : vector<3xi32>
+
+  // CHECK: return %[[CTRUE]], %[[CVTRUE]]
+  return %0, %1 : i1, vector<3xi1>
+}
+
+// CHECK-LABEL: @const_fold_scalar_ule
+func.func @const_fold_scalar_ule() -> (i1, i1) {
+  %c4 = spirv.Constant 4 : i32
+  %c5 = spirv.Constant 5 : i32
+  %cn6 = spirv.Constant -6 : i32
+
+  // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true
+  // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false
+  %0 = spirv.ULessThanEqual %c5, %cn6 : i32
+  %1 = spirv.ULessThanEqual %c5, %c4 : i32
+
+  // CHECK: return %[[CTRUE]], %[[CFALSE]]
+  return %0, %1 : i1, i1
+}
+
+// CHECK-LABEL: @const_fold_vector_ule
+func.func @const_fold_vector_ule() -> vector<3xi1> {
+  %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32>
+  %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32>
+
+  // CHECK: %[[RET:.*]] = spirv.Constant dense<[true, true, false]>
+  %0 = spirv.ULessThanEqual %cv0, %cv1 : vector<3xi32>
+
+  // CHECK: return %[[RET]]
+  return %0 : vector<3xi1>
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.LeftShiftLogical
 //===----------------------------------------------------------------------===//
-- 
GitLab


From 7d2d8e2a7245e4e64da22cb3c422ea3be5a0bf0a Mon Sep 17 00:00:00 2001
From: Kai Sasaki <lewuathe@gmail.com>
Date: Mon, 25 Mar 2024 10:59:42 +0900
Subject: [PATCH 082/404] [mlir][complex] Fastmath flag for the trigonometric
 ops in complex (#85563)

Support Fastmath flag to convert trigonometric ops in the complex
dialect.

See:
https://discourse.llvm.org/t/rfc-fastmath-flags-support-in-complex-dialect/71981
---
 .../ComplexToStandard/ComplexToStandard.cpp   | 50 +++++++++++--------
 .../convert-to-standard.mlir                  | 46 +++++++++++++++++
 2 files changed, 75 insertions(+), 21 deletions(-)

diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
index 76729278ec1b..17f64f1b65b7 100644
--- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
+++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
@@ -196,6 +196,7 @@ struct TrigonometricOpConversion : public OpConversionPattern<TrigonometricOp> {
     auto loc = op.getLoc();
     auto type = cast<ComplexType>(adaptor.getComplex().getType());
     auto elementType = cast<FloatType>(type.getElementType());
+    arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
     Value real =
         rewriter.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
@@ -207,14 +208,14 @@ struct TrigonometricOpConversion : public OpConversionPattern<TrigonometricOp> {
     // implementation in the subclass to combine them.
     Value half = rewriter.create<arith::ConstantOp>(
         loc, elementType, rewriter.getFloatAttr(elementType, 0.5));
-    Value exp = rewriter.create<math::ExpOp>(loc, imag);
-    Value scaledExp = rewriter.create<arith::MulFOp>(loc, half, exp);
-    Value reciprocalExp = rewriter.create<arith::DivFOp>(loc, half, exp);
-    Value sin = rewriter.create<math::SinOp>(loc, real);
-    Value cos = rewriter.create<math::CosOp>(loc, real);
+    Value exp = rewriter.create<math::ExpOp>(loc, imag, fmf);
+    Value scaledExp = rewriter.create<arith::MulFOp>(loc, half, exp, fmf);
+    Value reciprocalExp = rewriter.create<arith::DivFOp>(loc, half, exp, fmf);
+    Value sin = rewriter.create<math::SinOp>(loc, real, fmf);
+    Value cos = rewriter.create<math::CosOp>(loc, real, fmf);
 
     auto resultPair =
-        combine(loc, scaledExp, reciprocalExp, sin, cos, rewriter);
+        combine(loc, scaledExp, reciprocalExp, sin, cos, rewriter, fmf);
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultPair.first,
                                                    resultPair.second);
@@ -223,15 +224,17 @@ struct TrigonometricOpConversion : public OpConversionPattern<TrigonometricOp> {
 
   virtual std::pair<Value, Value>
   combine(Location loc, Value scaledExp, Value reciprocalExp, Value sin,
-          Value cos, ConversionPatternRewriter &rewriter) const = 0;
+          Value cos, ConversionPatternRewriter &rewriter,
+          arith::FastMathFlagsAttr fmf) const = 0;
 };
 
 struct CosOpConversion : public TrigonometricOpConversion<complex::CosOp> {
   using TrigonometricOpConversion<complex::CosOp>::TrigonometricOpConversion;
 
-  std::pair<Value, Value>
-  combine(Location loc, Value scaledExp, Value reciprocalExp, Value sin,
-          Value cos, ConversionPatternRewriter &rewriter) const override {
+  std::pair<Value, Value> combine(Location loc, Value scaledExp,
+                                  Value reciprocalExp, Value sin, Value cos,
+                                  ConversionPatternRewriter &rewriter,
+                                  arith::FastMathFlagsAttr fmf) const override {
     // Complex cosine is defined as;
     //   cos(x + iy) = 0.5 * (exp(i(x + iy)) + exp(-i(x + iy)))
     // Plugging in:
@@ -241,10 +244,12 @@ struct CosOpConversion : public TrigonometricOpConversion<complex::CosOp> {
     // We get:
     //   Re(cos(x + iy)) = (0.5/t + 0.5*t) * cos x
     //   Im(cos(x + iy)) = (0.5/t - 0.5*t) * sin x
-    Value sum = rewriter.create<arith::AddFOp>(loc, reciprocalExp, scaledExp);
-    Value resultReal = rewriter.create<arith::MulFOp>(loc, sum, cos);
-    Value diff = rewriter.create<arith::SubFOp>(loc, reciprocalExp, scaledExp);
-    Value resultImag = rewriter.create<arith::MulFOp>(loc, diff, sin);
+    Value sum =
+        rewriter.create<arith::AddFOp>(loc, reciprocalExp, scaledExp, fmf);
+    Value resultReal = rewriter.create<arith::MulFOp>(loc, sum, cos, fmf);
+    Value diff =
+        rewriter.create<arith::SubFOp>(loc, reciprocalExp, scaledExp, fmf);
+    Value resultImag = rewriter.create<arith::MulFOp>(loc, diff, sin, fmf);
     return {resultReal, resultImag};
   }
 };
@@ -813,9 +818,10 @@ struct NegOpConversion : public OpConversionPattern<complex::NegOp> {
 struct SinOpConversion : public TrigonometricOpConversion<complex::SinOp> {
   using TrigonometricOpConversion<complex::SinOp>::TrigonometricOpConversion;
 
-  std::pair<Value, Value>
-  combine(Location loc, Value scaledExp, Value reciprocalExp, Value sin,
-          Value cos, ConversionPatternRewriter &rewriter) const override {
+  std::pair<Value, Value> combine(Location loc, Value scaledExp,
+                                  Value reciprocalExp, Value sin, Value cos,
+                                  ConversionPatternRewriter &rewriter,
+                                  arith::FastMathFlagsAttr fmf) const override {
     // Complex sine is defined as;
     //   sin(x + iy) = -0.5i * (exp(i(x + iy)) - exp(-i(x + iy)))
     // Plugging in:
@@ -825,10 +831,12 @@ struct SinOpConversion : public TrigonometricOpConversion<complex::SinOp> {
     // We get:
     //   Re(sin(x + iy)) = (0.5*t + 0.5/t) * sin x
     //   Im(cos(x + iy)) = (0.5*t - 0.5/t) * cos x
-    Value sum = rewriter.create<arith::AddFOp>(loc, scaledExp, reciprocalExp);
-    Value resultReal = rewriter.create<arith::MulFOp>(loc, sum, sin);
-    Value diff = rewriter.create<arith::SubFOp>(loc, scaledExp, reciprocalExp);
-    Value resultImag = rewriter.create<arith::MulFOp>(loc, diff, cos);
+    Value sum =
+        rewriter.create<arith::AddFOp>(loc, scaledExp, reciprocalExp, fmf);
+    Value resultReal = rewriter.create<arith::MulFOp>(loc, sum, sin, fmf);
+    Value diff =
+        rewriter.create<arith::SubFOp>(loc, scaledExp, reciprocalExp, fmf);
+    Value resultImag = rewriter.create<arith::MulFOp>(loc, diff, cos, fmf);
     return {resultReal, resultImag};
   }
 };
diff --git a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
index 5918ff2e0f36..bac94aae6b74 100644
--- a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
+++ b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
@@ -1834,3 +1834,49 @@ func.func @complex_sqrt_with_fmf(%arg: complex<f32>) -> complex<f32> {
 // CHECK: %[[VAR40:.*]] = arith.select %[[VAR38]], %cst, %[[VAR32]] : f32
 // CHECK: %[[VAR41:.*]] = complex.create %[[VAR39]], %[[VAR40]] : complex<f32>
 // CHECK: return %[[VAR41]] : complex<f32>
+
+// -----
+
+// CHECK-LABEL: func @complex_cos_with_fmf
+// CHECK-SAME: %[[ARG:.*]]: complex<f32>
+func.func @complex_cos_with_fmf(%arg: complex<f32>) -> complex<f32> {
+  %cos = complex.cos %arg fastmath<nnan,contract> : complex<f32>
+  return %cos : complex<f32>
+}
+// CHECK-DAG: %[[REAL:.*]] = complex.re %[[ARG]]
+// CHECK-DAG: %[[IMAG:.*]] = complex.im %[[ARG]]
+// CHECK-DAG: %[[HALF:.*]] = arith.constant 5.000000e-01 : f32
+// CHECK-DAG: %[[EXP:.*]] = math.exp %[[IMAG]] fastmath<nnan,contract> : f32
+// CHECK-DAG: %[[HALF_EXP:.*]] = arith.mulf %[[HALF]], %[[EXP]] fastmath<nnan,contract>
+// CHECK-DAG: %[[HALF_REXP:.*]] = arith.divf %[[HALF]], %[[EXP]] fastmath<nnan,contract>
+// CHECK-DAG: %[[SIN:.*]] = math.sin %[[REAL]] fastmath<nnan,contract> : f32
+// CHECK-DAG: %[[COS:.*]] = math.cos %[[REAL]] fastmath<nnan,contract> : f32
+// CHECK-DAG: %[[EXP_SUM:.*]] = arith.addf %[[HALF_REXP]], %[[HALF_EXP]] fastmath<nnan,contract>
+// CHECK-DAG: %[[RESULT_REAL:.*]] = arith.mulf %[[EXP_SUM]], %[[COS]] fastmath<nnan,contract>
+// CHECK-DAG: %[[EXP_DIFF:.*]] = arith.subf %[[HALF_REXP]], %[[HALF_EXP]] fastmath<nnan,contract>
+// CHECK-DAG: %[[RESULT_IMAG:.*]] = arith.mulf %[[EXP_DIFF]], %[[SIN]] fastmath<nnan,contract>
+// CHECK-DAG: %[[RESULT:.*]] = complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : complex<f32>
+// CHECK:     return %[[RESULT]]
+
+// -----
+
+// CHECK-LABEL: func @complex_sin_with_fmf
+// CHECK-SAME: %[[ARG:.*]]: complex<f32>
+func.func @complex_sin_with_fmf(%arg: complex<f32>) -> complex<f32> {
+  %cos = complex.sin %arg fastmath<nnan,contract> : complex<f32>
+  return %cos : complex<f32>
+}
+// CHECK-DAG: %[[REAL:.*]] = complex.re %[[ARG]]
+// CHECK-DAG: %[[IMAG:.*]] = complex.im %[[ARG]]
+// CHECK-DAG: %[[HALF:.*]] = arith.constant 5.000000e-01 : f32
+// CHECK-DAG: %[[EXP:.*]] = math.exp %[[IMAG]] fastmath<nnan,contract> : f32
+// CHECK-DAG: %[[HALF_EXP:.*]] = arith.mulf %[[HALF]], %[[EXP]] fastmath<nnan,contract>
+// CHECK-DAG: %[[HALF_REXP:.*]] = arith.divf %[[HALF]], %[[EXP]] fastmath<nnan,contract>
+// CHECK-DAG: %[[SIN:.*]] = math.sin %[[REAL]] fastmath<nnan,contract> : f32
+// CHECK-DAG: %[[COS:.*]] = math.cos %[[REAL]] fastmath<nnan,contract> : f32
+// CHECK-DAG: %[[EXP_SUM:.*]] = arith.addf %[[HALF_EXP]], %[[HALF_REXP]] fastmath<nnan,contract>
+// CHECK-DAG: %[[RESULT_REAL:.*]] = arith.mulf %[[EXP_SUM]], %[[SIN]] fastmath<nnan,contract>
+// CHECK-DAG: %[[EXP_DIFF:.*]] = arith.subf %[[HALF_EXP]], %[[HALF_REXP]] fastmath<nnan,contract>
+// CHECK-DAG: %[[RESULT_IMAG:.*]] = arith.mulf %[[EXP_DIFF]], %[[COS]] fastmath<nnan,contract>
+// CHECK-DAG: %[[RESULT:.*]] = complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : complex<f32>
+// CHECK:     return %[[RESULT]]
-- 
GitLab


From 2e4e04c59043a645572cf548de5d9c333a5d6641 Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang@intel.com>
Date: Mon, 25 Mar 2024 10:06:12 +0800
Subject: [PATCH 083/404] [X86][BF16] Do not lower to VCVTNEPS2BF16 without
 AVX512VL (#86395)

Fixes: #86305
---
 llvm/lib/Target/X86/X86ISelLowering.cpp |  7 ++-
 llvm/test/CodeGen/X86/pr86305.ll        | 74 +++++++++++++++++++++++++
 2 files changed, 79 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/X86/pr86305.ll

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 35f756ea5e1d..9acbe17d0bca 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -21512,7 +21512,9 @@ SDValue X86TargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
   }
 
   if (VT.getScalarType() == MVT::bf16) {
-    if (SVT.getScalarType() == MVT::f32 && isTypeLegal(VT))
+    if (SVT.getScalarType() == MVT::f32 &&
+        ((Subtarget.hasBF16() && Subtarget.hasVLX()) ||
+         Subtarget.hasAVXNECONVERT()))
       return Op;
     return SDValue();
   }
@@ -21619,7 +21621,8 @@ SDValue X86TargetLowering::LowerFP_TO_BF16(SDValue Op,
   SDLoc DL(Op);
 
   MVT SVT = Op.getOperand(0).getSimpleValueType();
-  if (SVT == MVT::f32 && (Subtarget.hasBF16() || Subtarget.hasAVXNECONVERT())) {
+  if (SVT == MVT::f32 && ((Subtarget.hasBF16() && Subtarget.hasVLX()) ||
+                          Subtarget.hasAVXNECONVERT())) {
     SDValue Res;
     Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, Op.getOperand(0));
     Res = DAG.getNode(X86ISD::CVTNEPS2BF16, DL, MVT::v8bf16, Res);
diff --git a/llvm/test/CodeGen/X86/pr86305.ll b/llvm/test/CodeGen/X86/pr86305.ll
new file mode 100644
index 000000000000..79b42bb2532c
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr86305.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=avx512bf16 | FileCheck %s
+
+define void @add(ptr %pa, ptr %pb, ptr %pc) nounwind {
+; CHECK-LABEL: add:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movq %rdx, %rbx
+; CHECK-NEXT:    movzwl (%rsi), %eax
+; CHECK-NEXT:    shll $16, %eax
+; CHECK-NEXT:    vmovd %eax, %xmm0
+; CHECK-NEXT:    movzwl (%rdi), %eax
+; CHECK-NEXT:    shll $16, %eax
+; CHECK-NEXT:    vmovd %eax, %xmm1
+; CHECK-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    callq __truncsfbf2@PLT
+; CHECK-NEXT:    vpextrw $0, %xmm0, (%rbx)
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    retq
+  %a = load bfloat, ptr %pa
+  %b = load bfloat, ptr %pb
+  %add = fadd bfloat %a, %b
+  store bfloat %add, ptr %pc
+  ret void
+}
+
+define <4 x bfloat> @fptrunc_v4f32(<4 x float> %a) nounwind {
+; CHECK-LABEL: fptrunc_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    subq $72, %rsp
+; CHECK-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    callq __truncsfbf2@PLT
+; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vpermilpd $1, (%rsp), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[1,0]
+; CHECK-NEXT:    callq __truncsfbf2@PLT
+; CHECK-NEXT:    vmovapd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vpshufd $255, (%rsp), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[3,3,3,3]
+; CHECK-NEXT:    callq __truncsfbf2@PLT
+; CHECK-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    callq __truncsfbf2@PLT
+; CHECK-NEXT:    vpextrw $0, %xmm0, %ebx
+; CHECK-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vpextrw $0, %xmm0, %ebp
+; CHECK-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vpextrw $0, %xmm0, %r14d
+; CHECK-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    vpextrw $0, %xmm0, %r15d
+; CHECK-NEXT:    vmovshdup (%rsp), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    # xmm0 = mem[1,1,3,3]
+; CHECK-NEXT:    callq __truncsfbf2@PLT
+; CHECK-NEXT:    vpextrw $0, %xmm0, %eax
+; CHECK-NEXT:    vmovd %r15d, %xmm0
+; CHECK-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0
+; CHECK-NEXT:    vpinsrw $2, %r14d, %xmm0, %xmm0
+; CHECK-NEXT:    vpinsrw $3, %ebp, %xmm0, %xmm0
+; CHECK-NEXT:    vpinsrw $4, %ebx, %xmm0, %xmm0
+; CHECK-NEXT:    vpinsrw $5, %ebx, %xmm0, %xmm0
+; CHECK-NEXT:    vpinsrw $6, %ebx, %xmm0, %xmm0
+; CHECK-NEXT:    vpinsrw $7, %ebx, %xmm0, %xmm0
+; CHECK-NEXT:    addq $72, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+  %b = fptrunc <4 x float> %a to <4 x bfloat>
+  ret <4 x bfloat> %b
+}
-- 
GitLab


From 5e5b6561029665e69e033cff4216fecb78302259 Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88@gmail.com>
Date: Mon, 25 Mar 2024 05:13:48 +0300
Subject: [PATCH 084/404] [MC] Make `MCParsedAsmOperand::getReg()` return
 `MCRegister` (#86444)

---
 .../llvm/MC/MCParser/MCParsedAsmOperand.h     |  3 ++-
 .../llvm/MC/MCParser/MCTargetAsmParser.h      |  4 +--
 llvm/lib/MC/MCParser/MCTargetAsmParser.cpp    |  6 +++++
 .../AArch64/AsmParser/AArch64AsmParser.cpp    |  2 +-
 .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp      |  2 +-
 .../lib/Target/ARM/AsmParser/ARMAsmParser.cpp |  2 +-
 .../lib/Target/AVR/AsmParser/AVRAsmParser.cpp |  2 +-
 .../lib/Target/BPF/AsmParser/BPFAsmParser.cpp |  2 +-
 .../Target/CSKY/AsmParser/CSKYAsmParser.cpp   |  2 +-
 .../Hexagon/AsmParser/HexagonAsmParser.cpp    |  2 +-
 .../Target/Lanai/AsmParser/LanaiAsmParser.cpp |  2 +-
 .../AsmParser/LoongArchAsmParser.cpp          |  4 +--
 .../Target/M68k/AsmParser/M68kAsmParser.cpp   |  4 +--
 .../MSP430/AsmParser/MSP430AsmParser.cpp      |  2 +-
 .../Target/Mips/AsmParser/MipsAsmParser.cpp   |  2 +-
 .../Target/PowerPC/AsmParser/PPCAsmParser.cpp | 26 ++++++++++---------
 .../Target/RISCV/AsmParser/RISCVAsmParser.cpp |  4 +--
 .../Target/Sparc/AsmParser/SparcAsmParser.cpp |  2 +-
 .../SystemZ/AsmParser/SystemZAsmParser.cpp    |  2 +-
 llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp  |  2 +-
 .../AsmParser/WebAssemblyAsmParser.cpp        |  2 +-
 llvm/lib/Target/X86/AsmParser/X86Operand.h    |  2 +-
 .../Xtensa/AsmParser/XtensaAsmParser.cpp      |  2 +-
 llvm/utils/TableGen/AsmMatcherEmitter.cpp     |  2 +-
 24 files changed, 46 insertions(+), 39 deletions(-)

diff --git a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h
index 0c9668904e82..27ecb7b85d22 100644
--- a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h
+++ b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h
@@ -15,6 +15,7 @@
 
 namespace llvm {
 
+class MCRegister;
 class raw_ostream;
 
 /// MCParsedAsmOperand - This abstract class represents a source-level assembly
@@ -57,7 +58,7 @@ public:
   virtual bool isImm() const = 0;
   /// isReg - Is this a register operand?
   virtual bool isReg() const = 0;
-  virtual unsigned getReg() const = 0;
+  virtual MCRegister getReg() const = 0;
 
   /// isMem - Is this a memory operand?
   virtual bool isMem() const = 0;
diff --git a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h
index 7edd3f8ce490..49ce417e6fbb 100644
--- a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h
+++ b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h
@@ -514,9 +514,7 @@ public:
   /// by the tied-operands checks in the AsmMatcher. This method can be
   /// overridden to allow e.g. a sub- or super-register as the tied operand.
   virtual bool areEqualRegs(const MCParsedAsmOperand &Op1,
-                            const MCParsedAsmOperand &Op2) const {
-    return Op1.isReg() && Op2.isReg() && Op1.getReg() == Op2.getReg();
-  }
+                            const MCParsedAsmOperand &Op2) const;
 
   // Return whether this parser uses assignment statements with equals tokens
   virtual bool equalIsAsmAssignment() { return true; };
diff --git a/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp b/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp
index 0db5fb36f795..665d92eb9a21 100644
--- a/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/MCTargetAsmParser.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCRegister.h"
 
 using namespace llvm;
 
@@ -48,3 +49,8 @@ ParseStatus MCTargetAsmParser::parseDirective(AsmToken DirectiveID) {
     return ParseStatus::Failure;
   return ParseStatus::NoMatch;
 }
+
+bool MCTargetAsmParser::areEqualRegs(const MCParsedAsmOperand &Op1,
+                                     const MCParsedAsmOperand &Op2) const {
+  return Op1.isReg() && Op2.isReg() && Op1.getReg() == Op2.getReg();
+}
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index b807aaf76fdb..21643ebb4138 100644
--- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -654,7 +654,7 @@ public:
     return Barrier.HasnXSModifier;
   }
 
-  unsigned getReg() const override {
+  MCRegister getReg() const override {
     assert(Kind == k_Register && "Invalid access!");
     return Reg.RegNum;
   }
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 529705479646..4648df199c74 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -981,7 +981,7 @@ public:
     return Imm.Type;
   }
 
-  unsigned getReg() const override {
+  MCRegister getReg() const override {
     assert(isRegKind());
     return Reg.RegNo;
   }
diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index 9cfdb15a0f43..2ad576ab1a9c 100644
--- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -1002,7 +1002,7 @@ public:
     return StringRef(Tok.Data, Tok.Length);
   }
 
-  unsigned getReg() const override {
+  MCRegister getReg() const override {
     assert((Kind == k_Register || Kind == k_CCOut) && "Invalid access!");
     return Reg.RegNum;
   }
diff --git a/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp b/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp
index db4aa03437c6..383dfcc31117 100644
--- a/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp
+++ b/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp
@@ -195,7 +195,7 @@ public:
     return Tok;
   }
 
-  unsigned getReg() const override {
+  MCRegister getReg() const override {
     assert((Kind == k_Register || Kind == k_Memri) && "Invalid access!");
 
     return RegImm.Reg;
diff --git a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
index 1688355f427c..9672ed009e9b 100644
--- a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
+++ b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
@@ -148,7 +148,7 @@ public:
   /// getEndLoc - Gets location of the last token of this operand
   SMLoc getEndLoc() const override { return EndLoc; }
 
-  unsigned getReg() const override {
+  MCRegister getReg() const override {
     assert(Kind == Register && "Invalid type access!");
     return Reg.RegNum;
   }
diff --git a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp
index 4711e58bbed6..30bd3dcefa60 100644
--- a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp
+++ b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp
@@ -400,7 +400,7 @@ public:
   /// Gets location of the last token of this operand.
   SMLoc getEndLoc() const override { return EndLoc; }
 
-  unsigned getReg() const override {
+  MCRegister getReg() const override {
     assert(Kind == Register && "Invalid type access!");
     return Reg.RegNum;
   }
diff --git a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
index 864591d4eb95..092cccbcca9c 100644
--- a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
+++ b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
@@ -245,7 +245,7 @@ public:
   /// getEndLoc - Get the location of the last token of this operand.
   SMLoc getEndLoc() const override { return EndLoc; }
 
-  unsigned getReg() const override {
+  MCRegister getReg() const override {
     assert(Kind == Register && "Invalid access!");
     return Reg.RegNum;
   }
diff --git a/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp b/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
index ff3649b77e35..6ab1375b974e 100644
--- a/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
+++ b/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
@@ -151,7 +151,7 @@ public:
   // getEndLoc - Gets location of the last token of this operand
   SMLoc getEndLoc() const override { return EndLoc; }
 
-  unsigned getReg() const override {
+  MCRegister getReg() const override {
     assert(isReg() && "Invalid type access!");
     return Reg.RegNum;
   }
diff --git a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
index cf163e4e1200..20284b18428b 100644
--- a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
+++ b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
@@ -467,9 +467,9 @@ public:
   /// Gets location of the last token of this operand.
   SMLoc getEndLoc() const override { return EndLoc; }
 
-  unsigned getReg() const override {
+  MCRegister getReg() const override {
     assert(Kind == KindTy::Register && "Invalid type access!");
-    return Reg.RegNum.id();
+    return Reg.RegNum;
   }
 
   const MCExpr *getImm() const {
diff --git a/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp b/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp
index b2c0fda1ccc2..126176133dc0 100644
--- a/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp
+++ b/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp
@@ -157,7 +157,7 @@ public:
   bool isDReg() const;
   bool isFPDReg() const;
   bool isFPCReg() const;
-  unsigned getReg() const override;
+  MCRegister getReg() const override;
   void addRegOperands(MCInst &Inst, unsigned N) const;
 
   static std::unique_ptr<M68kOperand> createMemOp(M68kMemOp MemOp, SMLoc Start,
@@ -312,7 +312,7 @@ bool M68kOperand::isReg() const {
   return Kind == KindTy::MemOp && MemOp.Op == M68kMemOp::Kind::Reg;
 }
 
-unsigned M68kOperand::getReg() const {
+MCRegister M68kOperand::getReg() const {
   assert(isReg());
   return MemOp.OuterReg;
 }
diff --git a/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp b/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp
index 818a468612a5..2bc1a89ef59c 100644
--- a/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp
+++ b/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp
@@ -183,7 +183,7 @@ public:
     return Tok;
   }
 
-  unsigned getReg() const override {
+  MCRegister getReg() const override {
     assert(Kind == k_Reg && "Invalid access!");
     return Reg;
   }
diff --git a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
index 9d6e8dc573a8..076e0a20cb97 100644
--- a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
+++ b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
@@ -1458,7 +1458,7 @@ public:
     return StringRef(Tok.Data, Tok.Length);
   }
 
-  unsigned getReg() const override {
+  MCRegister getReg() const override {
     // As a special case until we sort out the definition of div/divu, accept
     // $0/$zero here so that MCK_ZERO works correctly.
     if (Kind == k_RegisterIndex && RegIdx.Index == 0 &&
diff --git a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
index 8108cfa521c8..55978af38000 100644
--- a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
+++ b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
@@ -276,9 +276,11 @@ public:
     return TLSReg.Sym;
   }
 
-  unsigned getReg() const override {
+  MCRegister getReg() const override { llvm_unreachable("Not implemented"); }
+
+  unsigned getRegNum() const {
     assert(isRegNumber() && "Invalid access!");
-    return (unsigned) Imm.Val;
+    return (unsigned)Imm.Val;
   }
 
   unsigned getFpReg() const {
@@ -459,22 +461,22 @@ public:
 
   void addRegGPRCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createReg(RRegs[getReg()]));
+    Inst.addOperand(MCOperand::createReg(RRegs[getRegNum()]));
   }
 
   void addRegGPRCNoR0Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createReg(RRegsNoR0[getReg()]));
+    Inst.addOperand(MCOperand::createReg(RRegsNoR0[getRegNum()]));
   }
 
   void addRegG8RCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createReg(XRegs[getReg()]));
+    Inst.addOperand(MCOperand::createReg(XRegs[getRegNum()]));
   }
 
   void addRegG8RCNoX0Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createReg(XRegsNoX0[getReg()]));
+    Inst.addOperand(MCOperand::createReg(XRegsNoX0[getRegNum()]));
   }
 
   void addRegG8pRCOperands(MCInst &Inst, unsigned N) const {
@@ -498,12 +500,12 @@ public:
 
   void addRegF4RCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createReg(FRegs[getReg()]));
+    Inst.addOperand(MCOperand::createReg(FRegs[getRegNum()]));
   }
 
   void addRegF8RCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createReg(FRegs[getReg()]));
+    Inst.addOperand(MCOperand::createReg(FRegs[getRegNum()]));
   }
 
   void addRegFpRCOperands(MCInst &Inst, unsigned N) const {
@@ -513,12 +515,12 @@ public:
 
   void addRegVFRCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createReg(VFRegs[getReg()]));
+    Inst.addOperand(MCOperand::createReg(VFRegs[getRegNum()]));
   }
 
   void addRegVRRCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createReg(VRegs[getReg()]));
+    Inst.addOperand(MCOperand::createReg(VRegs[getRegNum()]));
   }
 
   void addRegVSRCOperands(MCInst &Inst, unsigned N) const {
@@ -538,12 +540,12 @@ public:
 
   void addRegSPE4RCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createReg(RRegs[getReg()]));
+    Inst.addOperand(MCOperand::createReg(RRegs[getRegNum()]));
   }
 
   void addRegSPERCOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createReg(SPERegs[getReg()]));
+    Inst.addOperand(MCOperand::createReg(SPERegs[getRegNum()]));
   }
 
   void addRegACCRCOperands(MCInst &Inst, unsigned N) const {
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index cb2ba52390e2..5e594d6cad6b 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -977,9 +977,9 @@ public:
     return Imm.IsRV64;
   }
 
-  unsigned getReg() const override {
+  MCRegister getReg() const override {
     assert(Kind == KindTy::Register && "Invalid type access!");
-    return Reg.RegNum.id();
+    return Reg.RegNum;
   }
 
   StringRef getSysReg() const {
diff --git a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
index be4ec1e9dce2..67e2b9d7c997 100644
--- a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
+++ b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
@@ -307,7 +307,7 @@ public:
     return StringRef(Tok.Data, Tok.Length);
   }
 
-  unsigned getReg() const override {
+  MCRegister getReg() const override {
     assert((Kind == k_Register) && "Invalid access!");
     return Reg.RegNum;
   }
diff --git a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
index a58e8e0dfedf..f2c04215d12d 100644
--- a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
+++ b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
@@ -227,7 +227,7 @@ public:
   bool isReg(RegisterKind RegKind) const {
     return Kind == KindReg && Reg.Kind == RegKind;
   }
-  unsigned getReg() const override {
+  MCRegister getReg() const override {
     assert(Kind == KindReg && "Not a register");
     return Reg.Num;
   }
diff --git a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp
index f9e30a3a9378..691fe8fe3aa4 100644
--- a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp
+++ b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp
@@ -344,7 +344,7 @@ public:
     return StringRef(Tok.Data, Tok.Length);
   }
 
-  unsigned getReg() const override {
+  MCRegister getReg() const override {
     assert((Kind == k_Register) && "Invalid access!");
     return Reg.RegNum;
   }
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
index 3cc4d50271eb..020c0d6229d2 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
@@ -100,7 +100,7 @@ struct WebAssemblyOperand : public MCParsedAsmOperand {
   bool isReg() const override { return false; }
   bool isBrList() const { return Kind == BrList; }
 
-  unsigned getReg() const override {
+  MCRegister getReg() const override {
     llvm_unreachable("Assembly inspects a register operand");
     return 0;
   }
diff --git a/llvm/lib/Target/X86/AsmParser/X86Operand.h b/llvm/lib/Target/X86/AsmParser/X86Operand.h
index 641158cb351f..78669784dd03 100644
--- a/llvm/lib/Target/X86/AsmParser/X86Operand.h
+++ b/llvm/lib/Target/X86/AsmParser/X86Operand.h
@@ -167,7 +167,7 @@ struct X86Operand final : public MCParsedAsmOperand {
     Tok.Length = Value.size();
   }
 
-  unsigned getReg() const override {
+  MCRegister getReg() const override {
     assert(Kind == Register && "Invalid access!");
     return Reg.RegNo;
   }
diff --git a/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp b/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp
index 3f808298527f..1fa00af2111e 100644
--- a/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp
+++ b/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp
@@ -244,7 +244,7 @@ public:
   /// getEndLoc - Gets location of the last token of this operand
   SMLoc getEndLoc() const override { return EndLoc; }
 
-  unsigned getReg() const override {
+  MCRegister getReg() const override {
     assert(Kind == Register && "Invalid type access!");
     return Reg.RegNum;
   }
diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
index 5df7990d8fc2..0a2b1cf0f9a6 100644
--- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
@@ -2519,7 +2519,7 @@ static void emitValidateOperandClass(AsmMatcherInfo &Info, raw_ostream &OS) {
   // Check for register operands, including sub-classes.
   OS << "  if (Operand.isReg()) {\n";
   OS << "    MatchClassKind OpKind;\n";
-  OS << "    switch (Operand.getReg()) {\n";
+  OS << "    switch (Operand.getReg().id()) {\n";
   OS << "    default: OpKind = InvalidMatchClass; break;\n";
   for (const auto &RC : Info.RegisterClasses)
     OS << "    case " << RC.first->getValueAsString("Namespace")
-- 
GitLab


From 6af6416e89de1f4656a145c4843226b468718434 Mon Sep 17 00:00:00 2001
From: Wang Pengcheng <wangpengcheng.pp@bytedance.com>
Date: Mon, 25 Mar 2024 11:44:16 +0800
Subject: [PATCH 085/404] [RISCV] Add a tune feature to disable stripping W
 suffix (#86255)

We have a hidden option to disable it, but I'd like to make it a
tune feature.

For some implementations, instructions with W suffix would be less
costly as they only perform on 32 bits data. Though we may lose some
chances to compress.
---
 llvm/lib/Target/RISCV/RISCVFeatures.td    |  4 ++
 llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp |  2 +-
 llvm/test/CodeGen/RISCV/strip-w-suffix.ll | 74 +++++++++++++++++++++++
 3 files changed, 79 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/RISCV/strip-w-suffix.ll

diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index f3e641e25018..6ef2289bb4be 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -1226,6 +1226,10 @@ def TuneNoSinkSplatOperands
                        "false", "Disable sink splat operands to enable .vx, .vf,"
                        ".wx, and .wf instructions">;
 
+def TuneNoStripWSuffix
+    : SubtargetFeature<"no-strip-w-suffix", "EnableStripWSuffix", "false",
+                       "Disable strip W suffix">;
+
 def TuneConditionalCompressedMoveFusion
     : SubtargetFeature<"conditional-cmv-fusion", "HasConditionalCompressedMoveFusion",
                        "true", "Enable branch+c.mv fusion">;
diff --git a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp
index dcf70e8cad64..39d420c2fbf0 100644
--- a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp
+++ b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp
@@ -672,7 +672,7 @@ bool RISCVOptWInstrs::stripWSuffixes(MachineFunction &MF,
                                      const RISCVInstrInfo &TII,
                                      const RISCVSubtarget &ST,
                                      MachineRegisterInfo &MRI) {
-  if (DisableStripWSuffix)
+  if (DisableStripWSuffix || !ST.enableStripWSuffix())
     return false;
 
   bool MadeChange = false;
diff --git a/llvm/test/CodeGen/RISCV/strip-w-suffix.ll b/llvm/test/CodeGen/RISCV/strip-w-suffix.ll
new file mode 100644
index 000000000000..4124b3d0d360
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/strip-w-suffix.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=STRIP %s
+; RUN: llc -mtriple=riscv64 -mattr=+m,+no-strip-w-suffix -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=NO-STRIP %s
+
+define i32 @addiw(i32 %a) {
+; STRIP-LABEL: addiw:
+; STRIP:       # %bb.0:
+; STRIP-NEXT:    lui a1, 1
+; STRIP-NEXT:    addi a1, a1, -1
+; STRIP-NEXT:    addw a0, a0, a1
+; STRIP-NEXT:    ret
+;
+; NO-STRIP-LABEL: addiw:
+; NO-STRIP:       # %bb.0:
+; NO-STRIP-NEXT:    lui a1, 1
+; NO-STRIP-NEXT:    addiw a1, a1, -1
+; NO-STRIP-NEXT:    addw a0, a0, a1
+; NO-STRIP-NEXT:    ret
+  %ret = add i32 %a, 4095
+  ret i32 %ret
+}
+
+define i32 @addw(i32 %a, i32 %b) {
+; STRIP-LABEL: addw:
+; STRIP:       # %bb.0:
+; STRIP-NEXT:    add a0, a0, a1
+; STRIP-NEXT:    addiw a0, a0, 1024
+; STRIP-NEXT:    ret
+;
+; NO-STRIP-LABEL: addw:
+; NO-STRIP:       # %bb.0:
+; NO-STRIP-NEXT:    addw a0, a0, a1
+; NO-STRIP-NEXT:    addiw a0, a0, 1024
+; NO-STRIP-NEXT:    ret
+  %add = add i32 %a, %b
+  %ret = add i32 %add, 1024
+  ret i32 %ret
+}
+
+define i32 @mulw(i32 %a, i32 %b) {
+; STRIP-LABEL: mulw:
+; STRIP:       # %bb.0:
+; STRIP-NEXT:    mul a0, a0, a1
+; STRIP-NEXT:    addiw a0, a0, 1024
+; STRIP-NEXT:    ret
+;
+; NO-STRIP-LABEL: mulw:
+; NO-STRIP:       # %bb.0:
+; NO-STRIP-NEXT:    mulw a0, a0, a1
+; NO-STRIP-NEXT:    addiw a0, a0, 1024
+; NO-STRIP-NEXT:    ret
+  %mul = mul i32 %a, %b
+  %ret = add i32 %mul, 1024
+  ret i32 %ret
+}
+
+define i32 @slliw(i32 %a) {
+; STRIP-LABEL: slliw:
+; STRIP:       # %bb.0:
+; STRIP-NEXT:    slli a0, a0, 1
+; STRIP-NEXT:    addiw a0, a0, 1024
+; STRIP-NEXT:    ret
+;
+; NO-STRIP-LABEL: slliw:
+; NO-STRIP:       # %bb.0:
+; NO-STRIP-NEXT:    slliw a0, a0, 1
+; NO-STRIP-NEXT:    addiw a0, a0, 1024
+; NO-STRIP-NEXT:    ret
+  %shl = shl i32 %a, 1
+  %ret = add i32 %shl, 1024
+  ret i32 %ret
+}
-- 
GitLab


From d9746a6a5d523e21eee2c1b50c2f08aa19396965 Mon Sep 17 00:00:00 2001
From: Wang Pengcheng <wangpengcheng.pp@bytedance.com>
Date: Mon, 25 Mar 2024 12:42:59 +0800
Subject: [PATCH 086/404] [RISCV][NFC] Pass LMUL to copyPhysRegVector

The opcode will be determined by LMUL.

Reviewers: preames, lukel97, topperc

Reviewed By: lukel97, topperc

Pull Request: https://github.com/llvm/llvm-project/pull/84448
---
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 52 ++++++++++++------------
 llvm/lib/Target/RISCV/RISCVInstrInfo.h   |  2 +-
 2 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 4ca300f9151e..14c2d41e80f1 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -299,36 +299,36 @@ void RISCVInstrInfo::copyPhysRegVector(MachineBasicBlock &MBB,
                                        MachineBasicBlock::iterator MBBI,
                                        const DebugLoc &DL, MCRegister DstReg,
                                        MCRegister SrcReg, bool KillSrc,
-                                       unsigned Opc, unsigned NF) const {
+                                       RISCVII::VLMUL LMul, unsigned NF) const {
   const TargetRegisterInfo *TRI = STI.getRegisterInfo();
 
-  RISCVII::VLMUL LMul;
+  unsigned Opc;
   unsigned SubRegIdx;
   unsigned VVOpc, VIOpc;
-  switch (Opc) {
+  switch (LMul) {
   default:
     llvm_unreachable("Impossible LMUL for vector register copy.");
-  case RISCV::VMV1R_V:
-    LMul = RISCVII::LMUL_1;
+  case RISCVII::LMUL_1:
+    Opc = RISCV::VMV1R_V;
     SubRegIdx = RISCV::sub_vrm1_0;
     VVOpc = RISCV::PseudoVMV_V_V_M1;
     VIOpc = RISCV::PseudoVMV_V_I_M1;
     break;
-  case RISCV::VMV2R_V:
-    LMul = RISCVII::LMUL_2;
+  case RISCVII::LMUL_2:
+    Opc = RISCV::VMV2R_V;
     SubRegIdx = RISCV::sub_vrm2_0;
     VVOpc = RISCV::PseudoVMV_V_V_M2;
     VIOpc = RISCV::PseudoVMV_V_I_M2;
     break;
-  case RISCV::VMV4R_V:
-    LMul = RISCVII::LMUL_4;
+  case RISCVII::LMUL_4:
+    Opc = RISCV::VMV4R_V;
     SubRegIdx = RISCV::sub_vrm4_0;
     VVOpc = RISCV::PseudoVMV_V_V_M4;
     VIOpc = RISCV::PseudoVMV_V_I_M4;
     break;
-  case RISCV::VMV8R_V:
+  case RISCVII::LMUL_8:
     assert(NF == 1);
-    LMul = RISCVII::LMUL_8;
+    Opc = RISCV::VMV8R_V;
     SubRegIdx = RISCV::sub_vrm1_0; // There is no sub_vrm8_0.
     VVOpc = RISCV::PseudoVMV_V_V_M8;
     VIOpc = RISCV::PseudoVMV_V_I_M8;
@@ -505,87 +505,87 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
 
   // VR->VR copies.
   if (RISCV::VRRegClass.contains(DstReg, SrcReg)) {
-    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V);
+    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1);
     return;
   }
 
   if (RISCV::VRM2RegClass.contains(DstReg, SrcReg)) {
-    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V);
+    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2);
     return;
   }
 
   if (RISCV::VRM4RegClass.contains(DstReg, SrcReg)) {
-    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV4R_V);
+    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_4);
     return;
   }
 
   if (RISCV::VRM8RegClass.contains(DstReg, SrcReg)) {
-    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV8R_V);
+    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_8);
     return;
   }
 
   if (RISCV::VRN2M1RegClass.contains(DstReg, SrcReg)) {
-    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V,
+    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1,
                       /*NF=*/2);
     return;
   }
 
   if (RISCV::VRN2M2RegClass.contains(DstReg, SrcReg)) {
-    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V,
+    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2,
                       /*NF=*/2);
     return;
   }
 
   if (RISCV::VRN2M4RegClass.contains(DstReg, SrcReg)) {
-    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV4R_V,
+    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_4,
                       /*NF=*/2);
     return;
   }
 
   if (RISCV::VRN3M1RegClass.contains(DstReg, SrcReg)) {
-    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V,
+    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1,
                       /*NF=*/3);
     return;
   }
 
   if (RISCV::VRN3M2RegClass.contains(DstReg, SrcReg)) {
-    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V,
+    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2,
                       /*NF=*/3);
     return;
   }
 
   if (RISCV::VRN4M1RegClass.contains(DstReg, SrcReg)) {
-    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V,
+    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1,
                       /*NF=*/4);
     return;
   }
 
   if (RISCV::VRN4M2RegClass.contains(DstReg, SrcReg)) {
-    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV2R_V,
+    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_2,
                       /*NF=*/4);
     return;
   }
 
   if (RISCV::VRN5M1RegClass.contains(DstReg, SrcReg)) {
-    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V,
+    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1,
                       /*NF=*/5);
     return;
   }
 
   if (RISCV::VRN6M1RegClass.contains(DstReg, SrcReg)) {
-    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V,
+    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1,
                       /*NF=*/6);
     return;
   }
 
   if (RISCV::VRN7M1RegClass.contains(DstReg, SrcReg)) {
-    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V,
+    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1,
                       /*NF=*/7);
     return;
   }
 
   if (RISCV::VRN8M1RegClass.contains(DstReg, SrcReg)) {
-    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCV::VMV1R_V,
+    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RISCVII::LMUL_1,
                       /*NF=*/8);
     return;
   }
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
index 8a312ee5e779..dd049fca0597 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -69,7 +69,7 @@ public:
   void copyPhysRegVector(MachineBasicBlock &MBB,
                          MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
                          MCRegister DstReg, MCRegister SrcReg, bool KillSrc,
-                         unsigned Opc, unsigned NF = 1) const;
+                         RISCVII::VLMUL LMul, unsigned NF = 1) const;
   void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
                    const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg,
                    bool KillSrc) const override;
-- 
GitLab


From 373e77b4c0ad9b0bf370f0e5a32a4100a5459d82 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Mon, 25 Mar 2024 13:08:56 +0800
Subject: [PATCH 087/404] [RISCV] Generalize (sub zext, zext) -> (sext (sub
 zext, zext)) to add (#86248)

This generalizes the combine added in #82455 to other binary ops,
beginning with adds in this patch.

Because the two zext operands are always +ve when treated as signed, and
we don't get any overflow since the add is carried out in at least N * 2
bits of the narrow type, the result of the add will always be +ve. So we
can use a zext for the outer extend, unlike sub which may produce a -ve
result from two +ve operands.

Although we could still use sext for add, I plan to add support for
other binary ops like mul in a later patch, but mul requires zext to be
correct (because the maximum value will take up the full N * 2 bits). So
I've opted to use zext here too for consistency.

Alive2 proof: https://alive2.llvm.org/ce/z/PRNsUM
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |  75 +++++++---
 .../CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll |  32 ++---
 llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll   | 128 ++++++++----------
 3 files changed, 125 insertions(+), 110 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 5214595485ca..4ace50aa477a 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -12913,6 +12913,55 @@ static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG,
   return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT));
 }
 
+// add (zext, zext) -> zext (add (zext, zext))
+// sub (zext, zext) -> sext (sub (zext, zext))
+//
+// where the sum of the extend widths match, and the the range of the bin op
+// fits inside the width of the narrower bin op. (For profitability on rvv, we
+// use a power of two for both inner and outer extend.)
+//
+// TODO: Extend this to other binary ops
+static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG) {
+
+  EVT VT = N->getValueType(0);
+  if (!VT.isVector() || !DAG.getTargetLoweringInfo().isTypeLegal(VT))
+    return SDValue();
+
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+  if (N0.getOpcode() != ISD::ZERO_EXTEND || N1.getOpcode() != ISD::ZERO_EXTEND)
+    return SDValue();
+  if (!N0.hasOneUse() || !N1.hasOneUse())
+    return SDValue();
+
+  SDValue Src0 = N0.getOperand(0);
+  SDValue Src1 = N1.getOperand(0);
+  EVT SrcVT = Src0.getValueType();
+  if (!DAG.getTargetLoweringInfo().isTypeLegal(SrcVT) ||
+      SrcVT != Src1.getValueType() || SrcVT.getScalarSizeInBits() < 8 ||
+      SrcVT.getScalarSizeInBits() >= VT.getScalarSizeInBits() / 2)
+    return SDValue();
+
+  LLVMContext &C = *DAG.getContext();
+  EVT ElemVT = VT.getVectorElementType().getHalfSizedIntegerVT(C);
+  EVT NarrowVT = EVT::getVectorVT(C, ElemVT, VT.getVectorElementCount());
+
+  Src0 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src0), NarrowVT, Src0);
+  Src1 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src1), NarrowVT, Src1);
+
+  // Src0 and Src1 are zero extended, so they're always positive if signed.
+  //
+  // sub can produce a negative from two positive operands, so it needs sign
+  // extended. Other nodes produce a positive from two positive operands, so
+  // zero extend instead.
+  unsigned OuterExtend =
+      N->getOpcode() == ISD::SUB ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
+
+  return DAG.getNode(
+      OuterExtend, SDLoc(N), VT,
+      DAG.getNode(N->getOpcode(), SDLoc(N), NarrowVT, Src0, Src1));
+}
+
 // Try to turn (add (xor bool, 1) -1) into (neg bool).
 static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG) {
   SDValue N0 = N->getOperand(0);
@@ -12950,6 +12999,8 @@ static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG,
     return V;
   if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
     return V;
+  if (SDValue V = combineBinOpOfZExt(N, DAG))
+    return V;
 
   // fold (add (select lhs, rhs, cc, 0, y), x) ->
   //      (select lhs, rhs, cc, x, (add x, y))
@@ -13017,28 +13068,8 @@ static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG,
     }
   }
 
-  // sub (zext, zext) -> sext (sub (zext, zext))
-  //   where the sum of the extend widths match, and the inner zexts
-  //   add at least one bit.  (For profitability on rvv, we use a
-  //   power of two for both inner and outer extend.)
-  if (VT.isVector() && Subtarget.getTargetLowering()->isTypeLegal(VT) &&
-      N0.getOpcode() == N1.getOpcode() && N0.getOpcode() == ISD::ZERO_EXTEND &&
-      N0.hasOneUse() && N1.hasOneUse()) {
-    SDValue Src0 = N0.getOperand(0);
-    SDValue Src1 = N1.getOperand(0);
-    EVT SrcVT = Src0.getValueType();
-    if (Subtarget.getTargetLowering()->isTypeLegal(SrcVT) &&
-        SrcVT == Src1.getValueType() && SrcVT.getScalarSizeInBits() >= 8 &&
-        SrcVT.getScalarSizeInBits() < VT.getScalarSizeInBits() / 2) {
-      LLVMContext &C = *DAG.getContext();
-      EVT ElemVT = VT.getVectorElementType().getHalfSizedIntegerVT(C);
-      EVT NarrowVT = EVT::getVectorVT(C, ElemVT, VT.getVectorElementCount());
-      Src0 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src0), NarrowVT, Src0);
-      Src1 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src1), NarrowVT, Src1);
-      return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT,
-                         DAG.getNode(ISD::SUB, SDLoc(N), NarrowVT, Src0, Src1));
-    }
-  }
+  if (SDValue V = combineBinOpOfZExt(N, DAG))
+    return V;
 
   // fold (sub x, (select lhs, rhs, cc, 0, y)) ->
   //      (select lhs, rhs, cc, x, (sub x, y))
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll
index 57a72c639b33..bc0bf5dd76ad 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwaddu.ll
@@ -385,12 +385,12 @@ define <32 x i64> @vwaddu_v32i64(ptr %x, ptr %y) nounwind {
 define <2 x i32> @vwaddu_v2i32_v2i8(ptr %x, ptr %y) {
 ; CHECK-LABEL: vwaddu_v2i32_v2i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vle8.v v9, (a1)
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vzext.vf2 v11, v9
-; CHECK-NEXT:    vwaddu.vv v8, v10, v11
+; CHECK-NEXT:    vwaddu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v10
 ; CHECK-NEXT:    ret
   %a = load <2 x i8>, ptr %x
   %b = load <2 x i8>, ptr %y
@@ -912,12 +912,12 @@ define <4 x i64> @crash(<4 x i16> %x, <4 x i16> %y) {
 define <2 x i32> @vwaddu_v2i32_of_v2i8(ptr %x, ptr %y) {
 ; CHECK-LABEL: vwaddu_v2i32_of_v2i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vle8.v v9, (a1)
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vzext.vf2 v11, v9
-; CHECK-NEXT:    vwaddu.vv v8, v10, v11
+; CHECK-NEXT:    vwaddu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v10
 ; CHECK-NEXT:    ret
   %a = load <2 x i8>, ptr %x
   %b = load <2 x i8>, ptr %y
@@ -930,12 +930,12 @@ define <2 x i32> @vwaddu_v2i32_of_v2i8(ptr %x, ptr %y) {
 define <2 x i64> @vwaddu_v2i64_of_v2i8(ptr %x, ptr %y) {
 ; CHECK-LABEL: vwaddu_v2i64_of_v2i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vle8.v v9, (a1)
-; CHECK-NEXT:    vzext.vf4 v10, v8
-; CHECK-NEXT:    vzext.vf4 v11, v9
-; CHECK-NEXT:    vwaddu.vv v8, v10, v11
+; CHECK-NEXT:    vwaddu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v10
 ; CHECK-NEXT:    ret
   %a = load <2 x i8>, ptr %x
   %b = load <2 x i8>, ptr %y
@@ -948,12 +948,12 @@ define <2 x i64> @vwaddu_v2i64_of_v2i8(ptr %x, ptr %y) {
 define <2 x i64> @vwaddu_v2i64_of_v2i16(ptr %x, ptr %y) {
 ; CHECK-LABEL: vwaddu_v2i64_of_v2i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
 ; CHECK-NEXT:    vle16.v v9, (a1)
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vzext.vf2 v11, v9
-; CHECK-NEXT:    vwaddu.vv v8, v10, v11
+; CHECK-NEXT:    vwaddu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v10
 ; CHECK-NEXT:    ret
   %a = load <2 x i16>, ptr %x
   %b = load <2 x i16>, ptr %y
diff --git a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll
index 66a7eea18be5..0a7051633a19 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll
@@ -435,10 +435,10 @@ define <vscale x 1 x i64> @vwadd_vv_nxv1i64_nxv1i16(<vscale x 1 x i16> %va, <vsc
 define <vscale x 1 x i64> @vwaddu_vv_nxv1i64_nxv1i16(<vscale x 1 x i16> %va, <vscale x 1 x i16> %vb) {
 ; CHECK-LABEL: vwaddu_vv_nxv1i64_nxv1i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vzext.vf2 v11, v9
-; CHECK-NEXT:    vwaddu.vv v8, v10, v11
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vwaddu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v10
 ; CHECK-NEXT:    ret
   %vc = zext <vscale x 1 x i16> %va to <vscale x 1 x i64>
   %vd = zext <vscale x 1 x i16> %vb to <vscale x 1 x i64>
@@ -468,11 +468,9 @@ define <vscale x 1 x i64> @vwaddu_vx_nxv1i64_nxv1i16(<vscale x 1 x i16> %va, i16
 ; CHECK-LABEL: vwaddu_vx_nxv1i64_nxv1i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vzext.vf2 v11, v9
-; CHECK-NEXT:    vwaddu.vv v8, v10, v11
+; CHECK-NEXT:    vwaddu.vx v9, v8, a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v9
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i16> poison, i16 %b, i16 0
   %splat = shufflevector <vscale x 1 x i16> %head, <vscale x 1 x i16> poison, <vscale x 1 x i32> zeroinitializer
@@ -555,10 +553,10 @@ define <vscale x 2 x i64> @vwadd_vv_nxv2i64_nxv2i16(<vscale x 2 x i16> %va, <vsc
 define <vscale x 2 x i64> @vwaddu_vv_nxv2i64_nxv2i16(<vscale x 2 x i16> %va, <vscale x 2 x i16> %vb) {
 ; CHECK-LABEL: vwaddu_vv_nxv2i64_nxv2i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vzext.vf2 v11, v9
-; CHECK-NEXT:    vwaddu.vv v8, v10, v11
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vwaddu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v10
 ; CHECK-NEXT:    ret
   %vc = zext <vscale x 2 x i16> %va to <vscale x 2 x i64>
   %vd = zext <vscale x 2 x i16> %vb to <vscale x 2 x i64>
@@ -588,11 +586,9 @@ define <vscale x 2 x i64> @vwaddu_vx_nxv2i64_nxv2i16(<vscale x 2 x i16> %va, i16
 ; CHECK-LABEL: vwaddu_vx_nxv2i64_nxv2i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vzext.vf2 v11, v9
-; CHECK-NEXT:    vwaddu.vv v8, v10, v11
+; CHECK-NEXT:    vwaddu.vx v10, v8, a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v10
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i16> poison, i16 %b, i16 0
   %splat = shufflevector <vscale x 2 x i16> %head, <vscale x 2 x i16> poison, <vscale x 2 x i32> zeroinitializer
@@ -675,10 +671,10 @@ define <vscale x 4 x i64> @vwadd_vv_nxv4i64_nxv4i16(<vscale x 4 x i16> %va, <vsc
 define <vscale x 4 x i64> @vwaddu_vv_nxv4i64_nxv4i16(<vscale x 4 x i16> %va, <vscale x 4 x i16> %vb) {
 ; CHECK-LABEL: vwaddu_vv_nxv4i64_nxv4i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v12, v8
-; CHECK-NEXT:    vzext.vf2 v14, v9
-; CHECK-NEXT:    vwaddu.vv v8, v12, v14
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vwaddu.vv v12, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v12
 ; CHECK-NEXT:    ret
   %vc = zext <vscale x 4 x i16> %va to <vscale x 4 x i64>
   %vd = zext <vscale x 4 x i16> %vb to <vscale x 4 x i64>
@@ -708,11 +704,9 @@ define <vscale x 4 x i64> @vwaddu_vx_nxv4i64_nxv4i16(<vscale x 4 x i16> %va, i16
 ; CHECK-LABEL: vwaddu_vx_nxv4i64_nxv4i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v12, v8
-; CHECK-NEXT:    vzext.vf2 v14, v9
-; CHECK-NEXT:    vwaddu.vv v8, v12, v14
+; CHECK-NEXT:    vwaddu.vx v12, v8, a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v12
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i16> poison, i16 %b, i16 0
   %splat = shufflevector <vscale x 4 x i16> %head, <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
@@ -795,10 +789,10 @@ define <vscale x 8 x i64> @vwadd_vv_nxv8i64_nxv8i16(<vscale x 8 x i16> %va, <vsc
 define <vscale x 8 x i64> @vwaddu_vv_nxv8i64_nxv8i16(<vscale x 8 x i16> %va, <vscale x 8 x i16> %vb) {
 ; CHECK-LABEL: vwaddu_vv_nxv8i64_nxv8i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vzext.vf2 v16, v8
-; CHECK-NEXT:    vzext.vf2 v20, v10
-; CHECK-NEXT:    vwaddu.vv v8, v16, v20
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vwaddu.vv v16, v8, v10
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v16
 ; CHECK-NEXT:    ret
   %vc = zext <vscale x 8 x i16> %va to <vscale x 8 x i64>
   %vd = zext <vscale x 8 x i16> %vb to <vscale x 8 x i64>
@@ -828,11 +822,9 @@ define <vscale x 8 x i64> @vwaddu_vx_nxv8i64_nxv8i16(<vscale x 8 x i16> %va, i16
 ; CHECK-LABEL: vwaddu_vx_nxv8i64_nxv8i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vzext.vf2 v16, v8
-; CHECK-NEXT:    vzext.vf2 v20, v10
-; CHECK-NEXT:    vwaddu.vv v8, v16, v20
+; CHECK-NEXT:    vwaddu.vx v16, v8, a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i16> poison, i16 %b, i16 0
   %splat = shufflevector <vscale x 8 x i16> %head, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
@@ -915,10 +907,10 @@ define <vscale x 1 x i64> @vwadd_vv_nxv1i64_nxv1i8(<vscale x 1 x i8> %va, <vscal
 define <vscale x 1 x i64> @vwaddu_vv_nxv1i64_nxv1i8(<vscale x 1 x i8> %va, <vscale x 1 x i8> %vb) {
 ; CHECK-LABEL: vwaddu_vv_nxv1i64_nxv1i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vzext.vf4 v10, v8
-; CHECK-NEXT:    vzext.vf4 v11, v9
-; CHECK-NEXT:    vwaddu.vv v8, v10, v11
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
+; CHECK-NEXT:    vwaddu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v10
 ; CHECK-NEXT:    ret
   %vc = zext <vscale x 1 x i8> %va to <vscale x 1 x i64>
   %vd = zext <vscale x 1 x i8> %vb to <vscale x 1 x i64>
@@ -948,11 +940,9 @@ define <vscale x 1 x i64> @vwaddu_vx_nxv1i64_nxv1i8(<vscale x 1 x i8> %va, i8 %b
 ; CHECK-LABEL: vwaddu_vx_nxv1i64_nxv1i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vzext.vf4 v10, v8
-; CHECK-NEXT:    vzext.vf4 v11, v9
-; CHECK-NEXT:    vwaddu.vv v8, v10, v11
+; CHECK-NEXT:    vwaddu.vx v9, v8, a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v9
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i8> poison, i8 %b, i8 0
   %splat = shufflevector <vscale x 1 x i8> %head, <vscale x 1 x i8> poison, <vscale x 1 x i32> zeroinitializer
@@ -1035,10 +1025,10 @@ define <vscale x 2 x i64> @vwadd_vv_nxv2i64_nxv2i8(<vscale x 2 x i8> %va, <vscal
 define <vscale x 2 x i64> @vwaddu_vv_nxv2i64_nxv2i8(<vscale x 2 x i8> %va, <vscale x 2 x i8> %vb) {
 ; CHECK-LABEL: vwaddu_vv_nxv2i64_nxv2i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vzext.vf4 v10, v8
-; CHECK-NEXT:    vzext.vf4 v11, v9
-; CHECK-NEXT:    vwaddu.vv v8, v10, v11
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf4, ta, ma
+; CHECK-NEXT:    vwaddu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v10
 ; CHECK-NEXT:    ret
   %vc = zext <vscale x 2 x i8> %va to <vscale x 2 x i64>
   %vd = zext <vscale x 2 x i8> %vb to <vscale x 2 x i64>
@@ -1068,11 +1058,9 @@ define <vscale x 2 x i64> @vwaddu_vx_nxv2i64_nxv2i8(<vscale x 2 x i8> %va, i8 %b
 ; CHECK-LABEL: vwaddu_vx_nxv2i64_nxv2i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vzext.vf4 v10, v8
-; CHECK-NEXT:    vzext.vf4 v11, v9
-; CHECK-NEXT:    vwaddu.vv v8, v10, v11
+; CHECK-NEXT:    vwaddu.vx v10, v8, a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v10
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i8> poison, i8 %b, i8 0
   %splat = shufflevector <vscale x 2 x i8> %head, <vscale x 2 x i8> poison, <vscale x 2 x i32> zeroinitializer
@@ -1155,10 +1143,10 @@ define <vscale x 4 x i64> @vwadd_vv_nxv4i64_nxv4i8(<vscale x 4 x i8> %va, <vscal
 define <vscale x 4 x i64> @vwaddu_vv_nxv4i64_nxv4i8(<vscale x 4 x i8> %va, <vscale x 4 x i8> %vb) {
 ; CHECK-LABEL: vwaddu_vv_nxv4i64_nxv4i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vzext.vf4 v12, v8
-; CHECK-NEXT:    vzext.vf4 v14, v9
-; CHECK-NEXT:    vwaddu.vv v8, v12, v14
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vwaddu.vv v12, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v12
 ; CHECK-NEXT:    ret
   %vc = zext <vscale x 4 x i8> %va to <vscale x 4 x i64>
   %vd = zext <vscale x 4 x i8> %vb to <vscale x 4 x i64>
@@ -1188,11 +1176,9 @@ define <vscale x 4 x i64> @vwaddu_vx_nxv4i64_nxv4i8(<vscale x 4 x i8> %va, i8 %b
 ; CHECK-LABEL: vwaddu_vx_nxv4i64_nxv4i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vzext.vf4 v12, v8
-; CHECK-NEXT:    vzext.vf4 v14, v9
-; CHECK-NEXT:    vwaddu.vv v8, v12, v14
+; CHECK-NEXT:    vwaddu.vx v12, v8, a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v12
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i8> poison, i8 %b, i8 0
   %splat = shufflevector <vscale x 4 x i8> %head, <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
@@ -1275,10 +1261,10 @@ define <vscale x 8 x i64> @vwadd_vv_nxv8i64_nxv8i8(<vscale x 8 x i8> %va, <vscal
 define <vscale x 8 x i64> @vwaddu_vv_nxv8i64_nxv8i8(<vscale x 8 x i8> %va, <vscale x 8 x i8> %vb) {
 ; CHECK-LABEL: vwaddu_vv_nxv8i64_nxv8i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vzext.vf4 v16, v8
-; CHECK-NEXT:    vzext.vf4 v20, v9
-; CHECK-NEXT:    vwaddu.vv v8, v16, v20
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwaddu.vv v16, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v16
 ; CHECK-NEXT:    ret
   %vc = zext <vscale x 8 x i8> %va to <vscale x 8 x i64>
   %vd = zext <vscale x 8 x i8> %vb to <vscale x 8 x i64>
@@ -1308,11 +1294,9 @@ define <vscale x 8 x i64> @vwaddu_vx_nxv8i64_nxv8i8(<vscale x 8 x i8> %va, i8 %b
 ; CHECK-LABEL: vwaddu_vx_nxv8i64_nxv8i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vzext.vf4 v16, v8
-; CHECK-NEXT:    vzext.vf4 v20, v9
-; CHECK-NEXT:    vwaddu.vv v8, v16, v20
+; CHECK-NEXT:    vwaddu.vx v16, v8, a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i8> poison, i8 %b, i8 0
   %splat = shufflevector <vscale x 8 x i8> %head, <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
-- 
GitLab


From babbdad15b8049a6a78087d15a163d897f07d320 Mon Sep 17 00:00:00 2001
From: Pierre van Houtryve <pierre.vanhoutryve@amd.com>
Date: Mon, 25 Mar 2024 09:23:40 +0100
Subject: [PATCH 088/404] [AMDGPU] Handle non-register operands for
 S_SUB/ADD_U64_PSEUDO (#86104)

This pseudo uses SSrc_b64 so it allows both an immediate or a register,
but the lowering crashed on immediate operands.
---
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |  4 +-
 .../CodeGen/AMDGPU/add_sub_u64_pseudos.mir    | 68 +++++++++++++++++++
 2 files changed, 70 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir

diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 7f0cff72c186..d437f339a687 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -4859,8 +4859,8 @@ MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter(
     if (Subtarget->hasScalarAddSub64()) {
       unsigned Opc = IsAdd ? AMDGPU::S_ADD_U64 : AMDGPU::S_SUB_U64;
       BuildMI(*BB, MI, DL, TII->get(Opc), Dest.getReg())
-          .addReg(Src0.getReg())
-          .addReg(Src1.getReg());
+        .add(Src0)
+        .add(Src1);
     } else {
       const SIRegisterInfo *TRI = ST.getRegisterInfo();
       const TargetRegisterClass *BoolRC = TRI->getBoolRC();
diff --git a/llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir b/llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir
new file mode 100644
index 000000000000..cba114c3568a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/add_sub_u64_pseudos.mir
@@ -0,0 +1,68 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=finalize-isel -o - %s | FileCheck -check-prefix=GFX11 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=finalize-isel -o - %s | FileCheck -check-prefix=GFX12 %s
+
+---
+name: reg_ops
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GFX11-LABEL: name: reg_ops
+    ; GFX11: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+    ; GFX11-NEXT: [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub1
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[DEF1]].sub0
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[DEF1]].sub1
+    ; GFX11-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[COPY2]], implicit-def $scc
+    ; GFX11-NEXT: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[COPY1]], [[COPY3]], implicit-def $scc, implicit $scc
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1
+    ;
+    ; GFX12-LABEL: name: reg_ops
+    ; GFX12: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+    ; GFX12-NEXT: [[DEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+    ; GFX12-NEXT: [[S_ADD_U64_:%[0-9]+]]:sreg_64 = S_ADD_U64 [[DEF]], [[DEF1]]
+    %0:sreg_64 = IMPLICIT_DEF
+    %1:sreg_64 = IMPLICIT_DEF
+    %2:sreg_64 = S_ADD_U64_PSEUDO %0, %1, implicit-def $scc
+...
+
+---
+name: lhs_imm
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GFX11-LABEL: name: lhs_imm
+    ; GFX11: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub1
+    ; GFX11-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 6565, [[COPY]], implicit-def $scc
+    ; GFX11-NEXT: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 0, [[COPY1]], implicit-def $scc, implicit $scc
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1
+    ;
+    ; GFX12-LABEL: name: lhs_imm
+    ; GFX12: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+    ; GFX12-NEXT: [[S_ADD_U64_:%[0-9]+]]:sreg_64 = S_ADD_U64 6565, [[DEF]]
+    %0:sreg_64 = IMPLICIT_DEF
+    %1:sreg_64 = S_ADD_U64_PSEUDO 6565, %0, implicit-def $scc
+...
+
+---
+name: rhs_imm
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GFX11-LABEL: name: rhs_imm
+    ; GFX11: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[DEF]].sub1
+    ; GFX11-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], 6565, implicit-def $scc
+    ; GFX11-NEXT: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[COPY1]], 0, implicit-def $scc, implicit $scc
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1
+    ;
+    ; GFX12-LABEL: name: rhs_imm
+    ; GFX12: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+    ; GFX12-NEXT: [[S_ADD_U64_:%[0-9]+]]:sreg_64 = S_ADD_U64 [[DEF]], 6565
+    %0:sreg_64 = IMPLICIT_DEF
+    %1:sreg_64 = S_ADD_U64_PSEUDO %0, 6565, implicit-def $scc
+...
-- 
GitLab


From fa3d789df15bd1f58fb8ba4ea3be909218cf7f03 Mon Sep 17 00:00:00 2001
From: Pierre van Houtryve <pierre.vanhoutryve@amd.com>
Date: Mon, 25 Mar 2024 09:40:35 +0100
Subject: [PATCH 089/404] [RFC][TableGen] Restructure TableGen Source (#80847)

Refactor of the llvm-tblgen source into:
- a "Basic" library, which contains the bare minimum utilities to build
`llvm-min-tablegen`
- a "Common" library which contains all of the helpers for TableGen
backends. Such helpers can be shared by more than one backend, and even
unit tested (e.g. CodeExpander is, maybe we can add more over time)

Fixes #80647
---
 llvm/unittests/TableGen/CMakeLists.txt        |  2 +-
 llvm/unittests/TableGen/CodeExpanderTest.cpp  |  4 +-
 llvm/utils/TableGen/AsmMatcherEmitter.cpp     | 12 ++--
 llvm/utils/TableGen/AsmWriterEmitter.cpp      | 14 ++--
 llvm/utils/TableGen/Basic/CMakeLists.txt      | 21 ++++++
 .../{ => Basic}/CodeGenIntrinsics.cpp         |  0
 .../TableGen/{ => Basic}/CodeGenIntrinsics.h  |  0
 .../TableGen/{ => Basic}/SDNodeProperties.cpp |  0
 .../TableGen/{ => Basic}/SDNodeProperties.h   |  0
 .../{ => Basic}/SequenceToOffsetTable.h       |  0
 llvm/utils/TableGen/CMakeLists.txt            | 67 ++++++++-----------
 llvm/utils/TableGen/CallingConvEmitter.cpp    |  2 +-
 llvm/utils/TableGen/CodeEmitterGen.cpp        | 10 +--
 llvm/utils/TableGen/CodeGenMapTable.cpp       |  4 +-
 .../TableGen/{ => Common}/AsmWriterInst.cpp   |  0
 .../TableGen/{ => Common}/AsmWriterInst.h     |  0
 llvm/utils/TableGen/Common/CMakeLists.txt     | 48 +++++++++++++
 .../{ => Common}/CodeGenDAGPatterns.cpp       |  0
 .../{ => Common}/CodeGenDAGPatterns.h         |  4 +-
 .../TableGen/{ => Common}/CodeGenHwModes.cpp  |  0
 .../TableGen/{ => Common}/CodeGenHwModes.h    |  0
 .../{ => Common}/CodeGenInstAlias.cpp         |  0
 .../TableGen/{ => Common}/CodeGenInstAlias.h  |  0
 .../{ => Common}/CodeGenInstruction.cpp       |  0
 .../{ => Common}/CodeGenInstruction.h         |  0
 .../{ => Common}/CodeGenRegisters.cpp         |  0
 .../TableGen/{ => Common}/CodeGenRegisters.h  |  0
 .../TableGen/{ => Common}/CodeGenSchedule.cpp |  0
 .../TableGen/{ => Common}/CodeGenSchedule.h   |  0
 .../TableGen/{ => Common}/CodeGenTarget.cpp   |  0
 .../TableGen/{ => Common}/CodeGenTarget.h     |  2 +-
 .../TableGen/{ => Common}/DAGISelMatcher.cpp  |  0
 .../TableGen/{ => Common}/DAGISelMatcher.h    |  0
 .../{ => Common}/GlobalISel/CXXPredicates.cpp |  0
 .../{ => Common}/GlobalISel/CXXPredicates.h   |  0
 .../{ => Common}/GlobalISel/CodeExpander.cpp  |  0
 .../{ => Common}/GlobalISel/CodeExpander.h    |  0
 .../{ => Common}/GlobalISel/CodeExpansions.h  |  0
 .../{ => Common}/GlobalISel/CombinerUtils.h   |  0
 .../GlobalISel}/GlobalISelMatchTable.cpp      |  4 +-
 .../GlobalISel}/GlobalISelMatchTable.h        |  2 +-
 .../GlobalISelMatchTableExecutorEmitter.cpp   |  0
 .../GlobalISelMatchTableExecutorEmitter.h     |  2 +-
 .../{ => Common}/GlobalISel/MatchDataInfo.cpp |  0
 .../{ => Common}/GlobalISel/MatchDataInfo.h   |  0
 .../{ => Common}/GlobalISel/Patterns.cpp      |  4 +-
 .../{ => Common}/GlobalISel/Patterns.h        |  0
 .../TableGen/{ => Common}/InfoByHwMode.cpp    |  0
 .../TableGen/{ => Common}/InfoByHwMode.h      |  0
 .../TableGen/{ => Common}/OptEmitter.cpp      |  0
 llvm/utils/TableGen/{ => Common}/OptEmitter.h |  0
 .../{ => Common}/PredicateExpander.cpp        |  0
 .../TableGen/{ => Common}/PredicateExpander.h |  0
 .../{ => Common}/SubtargetFeatureInfo.cpp     |  0
 .../{ => Common}/SubtargetFeatureInfo.h       |  0
 llvm/utils/TableGen/{ => Common}/Types.cpp    |  0
 llvm/utils/TableGen/{ => Common}/Types.h      |  0
 .../{ => Common}/VarLenCodeEmitterGen.cpp     | 15 ++---
 .../{ => Common}/VarLenCodeEmitterGen.h       |  0
 llvm/utils/TableGen/CompressInstEmitter.cpp   |  6 +-
 llvm/utils/TableGen/DAGISelEmitter.cpp        |  8 +--
 llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 12 ++--
 llvm/utils/TableGen/DAGISelMatcherGen.cpp     | 14 ++--
 llvm/utils/TableGen/DAGISelMatcherOpt.cpp     |  6 +-
 llvm/utils/TableGen/DFAEmitter.cpp            |  2 +-
 llvm/utils/TableGen/DFAPacketizerEmitter.cpp  |  4 +-
 llvm/utils/TableGen/DXILEmitter.cpp           |  4 +-
 llvm/utils/TableGen/DecoderEmitter.cpp        | 10 +--
 llvm/utils/TableGen/DisassemblerEmitter.cpp   |  2 +-
 llvm/utils/TableGen/FastISelEmitter.cpp       | 10 +--
 llvm/utils/TableGen/GlobalISel/CMakeLists.txt | 20 ------
 .../TableGen/GlobalISelCombinerEmitter.cpp    | 24 +++----
 llvm/utils/TableGen/GlobalISelEmitter.cpp     | 18 ++---
 llvm/utils/TableGen/InstrDocsEmitter.cpp      |  6 +-
 llvm/utils/TableGen/InstrInfoEmitter.cpp      | 16 ++---
 llvm/utils/TableGen/IntrinsicEmitter.cpp      |  4 +-
 .../TableGen/MacroFusionPredicatorEmitter.cpp |  4 +-
 llvm/utils/TableGen/OptParserEmitter.cpp      |  2 +-
 llvm/utils/TableGen/OptRSTEmitter.cpp         |  2 +-
 llvm/utils/TableGen/PseudoLoweringEmitter.cpp |  4 +-
 llvm/utils/TableGen/RegisterBankEmitter.cpp   |  6 +-
 llvm/utils/TableGen/RegisterInfoEmitter.cpp   | 12 ++--
 .../utils/TableGen/SearchableTableEmitter.cpp |  4 +-
 llvm/utils/TableGen/SubtargetEmitter.cpp      |  8 +--
 .../WebAssemblyDisassemblerEmitter.cpp        |  2 +-
 .../TableGen/X86CompressEVEXTablesEmitter.cpp |  4 +-
 llvm/utils/TableGen/X86FoldTablesEmitter.cpp  |  4 +-
 llvm/utils/TableGen/X86MnemonicTables.cpp     |  4 +-
 llvm/utils/TableGen/X86RecognizableInstr.h    |  2 +-
 89 files changed, 238 insertions(+), 203 deletions(-)
 create mode 100644 llvm/utils/TableGen/Basic/CMakeLists.txt
 rename llvm/utils/TableGen/{ => Basic}/CodeGenIntrinsics.cpp (100%)
 rename llvm/utils/TableGen/{ => Basic}/CodeGenIntrinsics.h (100%)
 rename llvm/utils/TableGen/{ => Basic}/SDNodeProperties.cpp (100%)
 rename llvm/utils/TableGen/{ => Basic}/SDNodeProperties.h (100%)
 rename llvm/utils/TableGen/{ => Basic}/SequenceToOffsetTable.h (100%)
 rename llvm/utils/TableGen/{ => Common}/AsmWriterInst.cpp (100%)
 rename llvm/utils/TableGen/{ => Common}/AsmWriterInst.h (100%)
 create mode 100644 llvm/utils/TableGen/Common/CMakeLists.txt
 rename llvm/utils/TableGen/{ => Common}/CodeGenDAGPatterns.cpp (100%)
 rename llvm/utils/TableGen/{ => Common}/CodeGenDAGPatterns.h (99%)
 rename llvm/utils/TableGen/{ => Common}/CodeGenHwModes.cpp (100%)
 rename llvm/utils/TableGen/{ => Common}/CodeGenHwModes.h (100%)
 rename llvm/utils/TableGen/{ => Common}/CodeGenInstAlias.cpp (100%)
 rename llvm/utils/TableGen/{ => Common}/CodeGenInstAlias.h (100%)
 rename llvm/utils/TableGen/{ => Common}/CodeGenInstruction.cpp (100%)
 rename llvm/utils/TableGen/{ => Common}/CodeGenInstruction.h (100%)
 rename llvm/utils/TableGen/{ => Common}/CodeGenRegisters.cpp (100%)
 rename llvm/utils/TableGen/{ => Common}/CodeGenRegisters.h (100%)
 rename llvm/utils/TableGen/{ => Common}/CodeGenSchedule.cpp (100%)
 rename llvm/utils/TableGen/{ => Common}/CodeGenSchedule.h (100%)
 rename llvm/utils/TableGen/{ => Common}/CodeGenTarget.cpp (100%)
 rename llvm/utils/TableGen/{ => Common}/CodeGenTarget.h (99%)
 rename llvm/utils/TableGen/{ => Common}/DAGISelMatcher.cpp (100%)
 rename llvm/utils/TableGen/{ => Common}/DAGISelMatcher.h (100%)
 rename llvm/utils/TableGen/{ => Common}/GlobalISel/CXXPredicates.cpp (100%)
 rename llvm/utils/TableGen/{ => Common}/GlobalISel/CXXPredicates.h (100%)
 rename llvm/utils/TableGen/{ => Common}/GlobalISel/CodeExpander.cpp (100%)
 rename llvm/utils/TableGen/{ => Common}/GlobalISel/CodeExpander.h (100%)
 rename llvm/utils/TableGen/{ => Common}/GlobalISel/CodeExpansions.h (100%)
 rename llvm/utils/TableGen/{ => Common}/GlobalISel/CombinerUtils.h (100%)
 rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTable.cpp (99%)
 rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTable.h (99%)
 rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTableExecutorEmitter.cpp (100%)
 rename llvm/utils/TableGen/{ => Common/GlobalISel}/GlobalISelMatchTableExecutorEmitter.h (99%)
 rename llvm/utils/TableGen/{ => Common}/GlobalISel/MatchDataInfo.cpp (100%)
 rename llvm/utils/TableGen/{ => Common}/GlobalISel/MatchDataInfo.h (100%)
 rename llvm/utils/TableGen/{ => Common}/GlobalISel/Patterns.cpp (99%)
 rename llvm/utils/TableGen/{ => Common}/GlobalISel/Patterns.h (100%)
 rename llvm/utils/TableGen/{ => Common}/InfoByHwMode.cpp (100%)
 rename llvm/utils/TableGen/{ => Common}/InfoByHwMode.h (100%)
 rename llvm/utils/TableGen/{ => Common}/OptEmitter.cpp (100%)
 rename llvm/utils/TableGen/{ => Common}/OptEmitter.h (100%)
 rename llvm/utils/TableGen/{ => Common}/PredicateExpander.cpp (100%)
 rename llvm/utils/TableGen/{ => Common}/PredicateExpander.h (100%)
 rename llvm/utils/TableGen/{ => Common}/SubtargetFeatureInfo.cpp (100%)
 rename llvm/utils/TableGen/{ => Common}/SubtargetFeatureInfo.h (100%)
 rename llvm/utils/TableGen/{ => Common}/Types.cpp (100%)
 rename llvm/utils/TableGen/{ => Common}/Types.h (100%)
 rename llvm/utils/TableGen/{ => Common}/VarLenCodeEmitterGen.cpp (97%)
 rename llvm/utils/TableGen/{ => Common}/VarLenCodeEmitterGen.h (100%)
 delete mode 100644 llvm/utils/TableGen/GlobalISel/CMakeLists.txt

diff --git a/llvm/unittests/TableGen/CMakeLists.txt b/llvm/unittests/TableGen/CMakeLists.txt
index 7830e0218045..fae0eee06e4b 100644
--- a/llvm/unittests/TableGen/CMakeLists.txt
+++ b/llvm/unittests/TableGen/CMakeLists.txt
@@ -15,4 +15,4 @@ add_llvm_unittest(TableGenTests DISABLE_LLVM_LINK_LLVM_DYLIB
   ParserEntryPointTest.cpp
   )
 
-target_link_libraries(TableGenTests PRIVATE LLVMTableGenGlobalISel LLVMTableGen)
+target_link_libraries(TableGenTests PRIVATE LLVMTableGenCommon LLVMTableGen)
diff --git a/llvm/unittests/TableGen/CodeExpanderTest.cpp b/llvm/unittests/TableGen/CodeExpanderTest.cpp
index 4a9a0e8c114b..1528884ffdf6 100644
--- a/llvm/unittests/TableGen/CodeExpanderTest.cpp
+++ b/llvm/unittests/TableGen/CodeExpanderTest.cpp
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "GlobalISel/CodeExpander.h"
-#include "GlobalISel/CodeExpansions.h"
+#include "Common/GlobalISel/CodeExpander.h"
+#include "Common/GlobalISel/CodeExpansions.h"
 
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Error.h"
diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
index 0a2b1cf0f9a6..8b82ce899a48 100644
--- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
@@ -95,12 +95,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenInstAlias.h"
-#include "CodeGenInstruction.h"
-#include "CodeGenRegisters.h"
-#include "CodeGenTarget.h"
-#include "SubtargetFeatureInfo.h"
-#include "Types.h"
+#include "Common/CodeGenInstAlias.h"
+#include "Common/CodeGenInstruction.h"
+#include "Common/CodeGenRegisters.h"
+#include "Common/CodeGenTarget.h"
+#include "Common/SubtargetFeatureInfo.h"
+#include "Common/Types.h"
 #include "llvm/ADT/CachedHashString.h"
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/STLExtras.h"
diff --git a/llvm/utils/TableGen/AsmWriterEmitter.cpp b/llvm/utils/TableGen/AsmWriterEmitter.cpp
index a27061ee585a..16661cd29edc 100644
--- a/llvm/utils/TableGen/AsmWriterEmitter.cpp
+++ b/llvm/utils/TableGen/AsmWriterEmitter.cpp
@@ -11,13 +11,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "AsmWriterInst.h"
-#include "CodeGenInstAlias.h"
-#include "CodeGenInstruction.h"
-#include "CodeGenRegisters.h"
-#include "CodeGenTarget.h"
-#include "SequenceToOffsetTable.h"
-#include "Types.h"
+#include "Basic/SequenceToOffsetTable.h"
+#include "Common/AsmWriterInst.h"
+#include "Common/CodeGenInstAlias.h"
+#include "Common/CodeGenInstruction.h"
+#include "Common/CodeGenRegisters.h"
+#include "Common/CodeGenTarget.h"
+#include "Common/Types.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
diff --git a/llvm/utils/TableGen/Basic/CMakeLists.txt b/llvm/utils/TableGen/Basic/CMakeLists.txt
new file mode 100644
index 000000000000..f2927d05c175
--- /dev/null
+++ b/llvm/utils/TableGen/Basic/CMakeLists.txt
@@ -0,0 +1,21 @@
+# The basic TableGen library contains as little dependencies as possible.
+# In particular, it does not depend on vt_gen -> it does not use ValueTypes.
+#
+# This library is the only thing included in `llvm-min-tablegen`.
+
+set(LLVM_LINK_COMPONENTS
+  Support
+  TableGen
+  )
+
+add_llvm_library(LLVMTableGenBasic STATIC OBJECT EXCLUDE_FROM_ALL
+  CodeGenIntrinsics.cpp
+  SDNodeProperties.cpp
+)
+set_target_properties(LLVMTableGenBasic PROPERTIES FOLDER "Tablegenning")
+
+# Users may include its headers as "Basic/*.h"
+target_include_directories(LLVMTableGenBasic
+  INTERFACE
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>
+  )
diff --git a/llvm/utils/TableGen/CodeGenIntrinsics.cpp b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp
similarity index 100%
rename from llvm/utils/TableGen/CodeGenIntrinsics.cpp
rename to llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp
diff --git a/llvm/utils/TableGen/CodeGenIntrinsics.h b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h
similarity index 100%
rename from llvm/utils/TableGen/CodeGenIntrinsics.h
rename to llvm/utils/TableGen/Basic/CodeGenIntrinsics.h
diff --git a/llvm/utils/TableGen/SDNodeProperties.cpp b/llvm/utils/TableGen/Basic/SDNodeProperties.cpp
similarity index 100%
rename from llvm/utils/TableGen/SDNodeProperties.cpp
rename to llvm/utils/TableGen/Basic/SDNodeProperties.cpp
diff --git a/llvm/utils/TableGen/SDNodeProperties.h b/llvm/utils/TableGen/Basic/SDNodeProperties.h
similarity index 100%
rename from llvm/utils/TableGen/SDNodeProperties.h
rename to llvm/utils/TableGen/Basic/SDNodeProperties.h
diff --git a/llvm/utils/TableGen/SequenceToOffsetTable.h b/llvm/utils/TableGen/Basic/SequenceToOffsetTable.h
similarity index 100%
rename from llvm/utils/TableGen/SequenceToOffsetTable.h
rename to llvm/utils/TableGen/Basic/SequenceToOffsetTable.h
diff --git a/llvm/utils/TableGen/CMakeLists.txt b/llvm/utils/TableGen/CMakeLists.txt
index 0100bf345ec2..14690329cabf 100644
--- a/llvm/utils/TableGen/CMakeLists.txt
+++ b/llvm/utils/TableGen/CMakeLists.txt
@@ -1,26 +1,25 @@
-add_subdirectory(GlobalISel)
+# Basic utilities which is the strict minimum needed to build
+# llvm-min-tblgen.
+add_subdirectory(Basic)
+# Common utilities are all of the reusable components and helper
+# code needed by the backends.
+add_subdirectory(Common)
 
-add_llvm_library(LLVMTableGenCommon STATIC OBJECT EXCLUDE_FROM_ALL
+set(LLVM_LINK_COMPONENTS Support)
+
+# llvm-min-tablegen only contains a subset of backends necessary to
+# build llvm/include. It must not depend on TableGenCommon, as
+# TableGenCommon depends on this already to generate things such as
+# ValueType definitions.
+add_tablegen(llvm-min-tblgen LLVM_HEADERS
+  TableGen.cpp
   Attributes.cpp
-  CodeGenIntrinsics.cpp
   DirectiveEmitter.cpp
   IntrinsicEmitter.cpp
   RISCVTargetDefEmitter.cpp
-  SDNodeProperties.cpp
   VTEmitter.cpp
-  PARTIAL_SOURCES_INTENDED
-
-  LINK_COMPONENTS
-  Support
-  TableGen
-  )
-set_target_properties(LLVMTableGenCommon PROPERTIES FOLDER "Tablegenning")
+  $<TARGET_OBJECTS:obj.LLVMTableGenBasic>
 
-set(LLVM_LINK_COMPONENTS Support)
-
-add_tablegen(llvm-min-tblgen LLVM_HEADERS
-  TableGen.cpp
-  $<TARGET_OBJECTS:obj.LLVMTableGenCommon>
   PARTIAL_SOURCES_INTENDED
   )
 set_target_properties(llvm-min-tblgen PROPERTIES FOLDER "Tablegenning")
@@ -35,63 +34,51 @@ add_tablegen(llvm-tblgen LLVM
   EXPORT LLVM
   AsmMatcherEmitter.cpp
   AsmWriterEmitter.cpp
-  AsmWriterInst.cpp
-  CTagsEmitter.cpp
+  Attributes.cpp
   CallingConvEmitter.cpp
   CodeEmitterGen.cpp
-  CodeGenDAGPatterns.cpp
-  CodeGenHwModes.cpp
-  CodeGenInstAlias.cpp
-  CodeGenInstruction.cpp
   CodeGenMapTable.cpp
-  CodeGenRegisters.cpp
-  CodeGenSchedule.cpp
-  CodeGenTarget.cpp
+  CompressInstEmitter.cpp
+  CTagsEmitter.cpp
   DAGISelEmitter.cpp
   DAGISelMatcherEmitter.cpp
   DAGISelMatcherGen.cpp
   DAGISelMatcherOpt.cpp
-  DAGISelMatcher.cpp
   DecoderEmitter.cpp
   DFAEmitter.cpp
   DFAPacketizerEmitter.cpp
+  DirectiveEmitter.cpp
   DisassemblerEmitter.cpp
   DXILEmitter.cpp
   ExegesisEmitter.cpp
   FastISelEmitter.cpp
   GlobalISelCombinerEmitter.cpp
   GlobalISelEmitter.cpp
-  GlobalISelMatchTable.cpp
-  GlobalISelMatchTableExecutorEmitter.cpp
-  InfoByHwMode.cpp
-  InstrInfoEmitter.cpp
   InstrDocsEmitter.cpp
-  OptEmitter.cpp
+  InstrInfoEmitter.cpp
+  IntrinsicEmitter.cpp
+  MacroFusionPredicatorEmitter.cpp
   OptParserEmitter.cpp
   OptRSTEmitter.cpp
-  PredicateExpander.cpp
   PseudoLoweringEmitter.cpp
-  CompressInstEmitter.cpp
-  MacroFusionPredicatorEmitter.cpp
   RegisterBankEmitter.cpp
   RegisterInfoEmitter.cpp
+  RISCVTargetDefEmitter.cpp
   SearchableTableEmitter.cpp
   SubtargetEmitter.cpp
-  SubtargetFeatureInfo.cpp
   TableGen.cpp
-  Types.cpp
-  VarLenCodeEmitterGen.cpp
-  X86DisassemblerTables.cpp
+  VTEmitter.cpp
+  WebAssemblyDisassemblerEmitter.cpp
   X86CompressEVEXTablesEmitter.cpp
+  X86DisassemblerTables.cpp
   X86FoldTablesEmitter.cpp
   X86MnemonicTables.cpp
   X86ModRMFilters.cpp
   X86RecognizableInstr.cpp
-  WebAssemblyDisassemblerEmitter.cpp
   $<TARGET_OBJECTS:obj.LLVMTableGenCommon>
 
   DEPENDS
   intrinsics_gen # via llvm-min-tablegen
   )
-target_link_libraries(llvm-tblgen PRIVATE LLVMTableGenGlobalISel)
+target_link_libraries(llvm-tblgen PRIVATE LLVMTableGenCommon)
 set_target_properties(llvm-tblgen PROPERTIES FOLDER "Tablegenning")
diff --git a/llvm/utils/TableGen/CallingConvEmitter.cpp b/llvm/utils/TableGen/CallingConvEmitter.cpp
index 3c3a2874ce80..ec6ef56a66fa 100644
--- a/llvm/utils/TableGen/CallingConvEmitter.cpp
+++ b/llvm/utils/TableGen/CallingConvEmitter.cpp
@@ -11,7 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenTarget.h"
+#include "Common/CodeGenTarget.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
diff --git a/llvm/utils/TableGen/CodeEmitterGen.cpp b/llvm/utils/TableGen/CodeEmitterGen.cpp
index 9194c13ccdcb..a57885f22d7e 100644
--- a/llvm/utils/TableGen/CodeEmitterGen.cpp
+++ b/llvm/utils/TableGen/CodeEmitterGen.cpp
@@ -22,11 +22,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenHwModes.h"
-#include "CodeGenInstruction.h"
-#include "CodeGenTarget.h"
-#include "InfoByHwMode.h"
-#include "VarLenCodeEmitterGen.h"
+#include "Common/CodeGenHwModes.h"
+#include "Common/CodeGenInstruction.h"
+#include "Common/CodeGenTarget.h"
+#include "Common/InfoByHwMode.h"
+#include "Common/VarLenCodeEmitterGen.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringExtras.h"
diff --git a/llvm/utils/TableGen/CodeGenMapTable.cpp b/llvm/utils/TableGen/CodeGenMapTable.cpp
index 03af0b49ba97..fbf1d47c0327 100644
--- a/llvm/utils/TableGen/CodeGenMapTable.cpp
+++ b/llvm/utils/TableGen/CodeGenMapTable.cpp
@@ -75,8 +75,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenInstruction.h"
-#include "CodeGenTarget.h"
+#include "Common/CodeGenInstruction.h"
+#include "Common/CodeGenTarget.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 using namespace llvm;
diff --git a/llvm/utils/TableGen/AsmWriterInst.cpp b/llvm/utils/TableGen/Common/AsmWriterInst.cpp
similarity index 100%
rename from llvm/utils/TableGen/AsmWriterInst.cpp
rename to llvm/utils/TableGen/Common/AsmWriterInst.cpp
diff --git a/llvm/utils/TableGen/AsmWriterInst.h b/llvm/utils/TableGen/Common/AsmWriterInst.h
similarity index 100%
rename from llvm/utils/TableGen/AsmWriterInst.h
rename to llvm/utils/TableGen/Common/AsmWriterInst.h
diff --git a/llvm/utils/TableGen/Common/CMakeLists.txt b/llvm/utils/TableGen/Common/CMakeLists.txt
new file mode 100644
index 000000000000..491d9bd2949d
--- /dev/null
+++ b/llvm/utils/TableGen/Common/CMakeLists.txt
@@ -0,0 +1,48 @@
+# The common library is similar to the basic library except it can
+# depend on vt_gen.
+#
+# This library contains the bulk of the supporting code for all
+# TableGen backends. It's split off as a separate library to
+# allow unit-testing those components.
+
+set(LLVM_LINK_COMPONENTS
+  Support
+  TableGen
+  )
+
+add_llvm_library(LLVMTableGenCommon STATIC OBJECT EXCLUDE_FROM_ALL
+  GlobalISel/CodeExpander.cpp
+  GlobalISel/CXXPredicates.cpp
+  GlobalISel/GlobalISelMatchTable.cpp
+  GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp
+  GlobalISel/MatchDataInfo.cpp
+  GlobalISel/Patterns.cpp
+
+  AsmWriterInst.cpp
+  CodeGenDAGPatterns.cpp
+  CodeGenHwModes.cpp
+  CodeGenInstAlias.cpp
+  CodeGenInstruction.cpp
+  CodeGenRegisters.cpp
+  CodeGenSchedule.cpp
+  CodeGenTarget.cpp
+  DAGISelMatcher.cpp
+  InfoByHwMode.cpp
+  OptEmitter.cpp
+  PredicateExpander.cpp
+  SubtargetFeatureInfo.cpp
+  Types.cpp
+  VarLenCodeEmitterGen.cpp
+  $<TARGET_OBJECTS:obj.LLVMTableGenBasic>
+
+  DEPENDS
+  vt_gen
+  )
+set_target_properties(LLVMTableGenCommon PROPERTIES FOLDER "Tablegenning")
+target_link_libraries(LLVMTableGenCommon PUBLIC LLVMTableGenBasic)
+
+# Users may include its headers as "Common/*.h"
+target_include_directories(LLVMTableGenCommon
+  PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>
+  )
diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
similarity index 100%
rename from llvm/utils/TableGen/CodeGenDAGPatterns.cpp
rename to llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.h b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h
similarity index 99%
rename from llvm/utils/TableGen/CodeGenDAGPatterns.h
rename to llvm/utils/TableGen/Common/CodeGenDAGPatterns.h
index 823c40c922cb..7fcd39a9e940 100644
--- a/llvm/utils/TableGen/CodeGenDAGPatterns.h
+++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h
@@ -14,9 +14,9 @@
 #ifndef LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H
 #define LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H
 
-#include "CodeGenIntrinsics.h"
+#include "Basic/CodeGenIntrinsics.h"
+#include "Basic/SDNodeProperties.h"
 #include "CodeGenTarget.h"
-#include "SDNodeProperties.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/PointerUnion.h"
diff --git a/llvm/utils/TableGen/CodeGenHwModes.cpp b/llvm/utils/TableGen/Common/CodeGenHwModes.cpp
similarity index 100%
rename from llvm/utils/TableGen/CodeGenHwModes.cpp
rename to llvm/utils/TableGen/Common/CodeGenHwModes.cpp
diff --git a/llvm/utils/TableGen/CodeGenHwModes.h b/llvm/utils/TableGen/Common/CodeGenHwModes.h
similarity index 100%
rename from llvm/utils/TableGen/CodeGenHwModes.h
rename to llvm/utils/TableGen/Common/CodeGenHwModes.h
diff --git a/llvm/utils/TableGen/CodeGenInstAlias.cpp b/llvm/utils/TableGen/Common/CodeGenInstAlias.cpp
similarity index 100%
rename from llvm/utils/TableGen/CodeGenInstAlias.cpp
rename to llvm/utils/TableGen/Common/CodeGenInstAlias.cpp
diff --git a/llvm/utils/TableGen/CodeGenInstAlias.h b/llvm/utils/TableGen/Common/CodeGenInstAlias.h
similarity index 100%
rename from llvm/utils/TableGen/CodeGenInstAlias.h
rename to llvm/utils/TableGen/Common/CodeGenInstAlias.h
diff --git a/llvm/utils/TableGen/CodeGenInstruction.cpp b/llvm/utils/TableGen/Common/CodeGenInstruction.cpp
similarity index 100%
rename from llvm/utils/TableGen/CodeGenInstruction.cpp
rename to llvm/utils/TableGen/Common/CodeGenInstruction.cpp
diff --git a/llvm/utils/TableGen/CodeGenInstruction.h b/llvm/utils/TableGen/Common/CodeGenInstruction.h
similarity index 100%
rename from llvm/utils/TableGen/CodeGenInstruction.h
rename to llvm/utils/TableGen/Common/CodeGenInstruction.h
diff --git a/llvm/utils/TableGen/CodeGenRegisters.cpp b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
similarity index 100%
rename from llvm/utils/TableGen/CodeGenRegisters.cpp
rename to llvm/utils/TableGen/Common/CodeGenRegisters.cpp
diff --git a/llvm/utils/TableGen/CodeGenRegisters.h b/llvm/utils/TableGen/Common/CodeGenRegisters.h
similarity index 100%
rename from llvm/utils/TableGen/CodeGenRegisters.h
rename to llvm/utils/TableGen/Common/CodeGenRegisters.h
diff --git a/llvm/utils/TableGen/CodeGenSchedule.cpp b/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
similarity index 100%
rename from llvm/utils/TableGen/CodeGenSchedule.cpp
rename to llvm/utils/TableGen/Common/CodeGenSchedule.cpp
diff --git a/llvm/utils/TableGen/CodeGenSchedule.h b/llvm/utils/TableGen/Common/CodeGenSchedule.h
similarity index 100%
rename from llvm/utils/TableGen/CodeGenSchedule.h
rename to llvm/utils/TableGen/Common/CodeGenSchedule.h
diff --git a/llvm/utils/TableGen/CodeGenTarget.cpp b/llvm/utils/TableGen/Common/CodeGenTarget.cpp
similarity index 100%
rename from llvm/utils/TableGen/CodeGenTarget.cpp
rename to llvm/utils/TableGen/Common/CodeGenTarget.cpp
diff --git a/llvm/utils/TableGen/CodeGenTarget.h b/llvm/utils/TableGen/Common/CodeGenTarget.h
similarity index 99%
rename from llvm/utils/TableGen/CodeGenTarget.h
rename to llvm/utils/TableGen/Common/CodeGenTarget.h
index e109c717dc01..df4c22ebb379 100644
--- a/llvm/utils/TableGen/CodeGenTarget.h
+++ b/llvm/utils/TableGen/Common/CodeGenTarget.h
@@ -16,10 +16,10 @@
 #ifndef LLVM_UTILS_TABLEGEN_CODEGENTARGET_H
 #define LLVM_UTILS_TABLEGEN_CODEGENTARGET_H
 
+#include "Basic/SDNodeProperties.h"
 #include "CodeGenHwModes.h"
 #include "CodeGenInstruction.h"
 #include "InfoByHwMode.h"
-#include "SDNodeProperties.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
diff --git a/llvm/utils/TableGen/DAGISelMatcher.cpp b/llvm/utils/TableGen/Common/DAGISelMatcher.cpp
similarity index 100%
rename from llvm/utils/TableGen/DAGISelMatcher.cpp
rename to llvm/utils/TableGen/Common/DAGISelMatcher.cpp
diff --git a/llvm/utils/TableGen/DAGISelMatcher.h b/llvm/utils/TableGen/Common/DAGISelMatcher.h
similarity index 100%
rename from llvm/utils/TableGen/DAGISelMatcher.h
rename to llvm/utils/TableGen/Common/DAGISelMatcher.h
diff --git a/llvm/utils/TableGen/GlobalISel/CXXPredicates.cpp b/llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.cpp
similarity index 100%
rename from llvm/utils/TableGen/GlobalISel/CXXPredicates.cpp
rename to llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.cpp
diff --git a/llvm/utils/TableGen/GlobalISel/CXXPredicates.h b/llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.h
similarity index 100%
rename from llvm/utils/TableGen/GlobalISel/CXXPredicates.h
rename to llvm/utils/TableGen/Common/GlobalISel/CXXPredicates.h
diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpander.cpp b/llvm/utils/TableGen/Common/GlobalISel/CodeExpander.cpp
similarity index 100%
rename from llvm/utils/TableGen/GlobalISel/CodeExpander.cpp
rename to llvm/utils/TableGen/Common/GlobalISel/CodeExpander.cpp
diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpander.h b/llvm/utils/TableGen/Common/GlobalISel/CodeExpander.h
similarity index 100%
rename from llvm/utils/TableGen/GlobalISel/CodeExpander.h
rename to llvm/utils/TableGen/Common/GlobalISel/CodeExpander.h
diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpansions.h b/llvm/utils/TableGen/Common/GlobalISel/CodeExpansions.h
similarity index 100%
rename from llvm/utils/TableGen/GlobalISel/CodeExpansions.h
rename to llvm/utils/TableGen/Common/GlobalISel/CodeExpansions.h
diff --git a/llvm/utils/TableGen/GlobalISel/CombinerUtils.h b/llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.h
similarity index 100%
rename from llvm/utils/TableGen/GlobalISel/CombinerUtils.h
rename to llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.h
diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
similarity index 99%
rename from llvm/utils/TableGen/GlobalISelMatchTable.cpp
rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
index 45fb41b89f27..193f95443b16 100644
--- a/llvm/utils/TableGen/GlobalISelMatchTable.cpp
+++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "GlobalISelMatchTable.h"
-#include "CodeGenInstruction.h"
-#include "CodeGenRegisters.h"
+#include "Common/CodeGenInstruction.h"
+#include "Common/CodeGenRegisters.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/LEB128.h"
diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.h b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h
similarity index 99%
rename from llvm/utils/TableGen/GlobalISelMatchTable.h
rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h
index b1ab7da8db44..aa86fad763d1 100644
--- a/llvm/utils/TableGen/GlobalISelMatchTable.h
+++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h
@@ -16,7 +16,7 @@
 #ifndef LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLE_H
 #define LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLE_H
 
-#include "CodeGenDAGPatterns.h"
+#include "Common/CodeGenDAGPatterns.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallPtrSet.h"
diff --git a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp
similarity index 100%
rename from llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp
rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp
diff --git a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h
similarity index 99%
rename from llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h
rename to llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h
index 7e952d6df309..d2b6a74c7577 100644
--- a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h
+++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h
@@ -15,7 +15,7 @@
 #ifndef LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLEEXECUTOREMITTER_H
 #define LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLEEXECUTOREMITTER_H
 
-#include "SubtargetFeatureInfo.h"
+#include "Common/SubtargetFeatureInfo.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
diff --git a/llvm/utils/TableGen/GlobalISel/MatchDataInfo.cpp b/llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.cpp
similarity index 100%
rename from llvm/utils/TableGen/GlobalISel/MatchDataInfo.cpp
rename to llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.cpp
diff --git a/llvm/utils/TableGen/GlobalISel/MatchDataInfo.h b/llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.h
similarity index 100%
rename from llvm/utils/TableGen/GlobalISel/MatchDataInfo.h
rename to llvm/utils/TableGen/Common/GlobalISel/MatchDataInfo.h
diff --git a/llvm/utils/TableGen/GlobalISel/Patterns.cpp b/llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp
similarity index 99%
rename from llvm/utils/TableGen/GlobalISel/Patterns.cpp
rename to llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp
index 758eac2dfebd..388bf7e9e833 100644
--- a/llvm/utils/TableGen/GlobalISel/Patterns.cpp
+++ b/llvm/utils/TableGen/Common/GlobalISel/Patterns.cpp
@@ -7,11 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "Patterns.h"
-#include "../CodeGenInstruction.h"
-#include "../CodeGenIntrinsics.h"
+#include "Basic/CodeGenIntrinsics.h"
 #include "CXXPredicates.h"
 #include "CodeExpander.h"
 #include "CodeExpansions.h"
+#include "Common/CodeGenInstruction.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
diff --git a/llvm/utils/TableGen/GlobalISel/Patterns.h b/llvm/utils/TableGen/Common/GlobalISel/Patterns.h
similarity index 100%
rename from llvm/utils/TableGen/GlobalISel/Patterns.h
rename to llvm/utils/TableGen/Common/GlobalISel/Patterns.h
diff --git a/llvm/utils/TableGen/InfoByHwMode.cpp b/llvm/utils/TableGen/Common/InfoByHwMode.cpp
similarity index 100%
rename from llvm/utils/TableGen/InfoByHwMode.cpp
rename to llvm/utils/TableGen/Common/InfoByHwMode.cpp
diff --git a/llvm/utils/TableGen/InfoByHwMode.h b/llvm/utils/TableGen/Common/InfoByHwMode.h
similarity index 100%
rename from llvm/utils/TableGen/InfoByHwMode.h
rename to llvm/utils/TableGen/Common/InfoByHwMode.h
diff --git a/llvm/utils/TableGen/OptEmitter.cpp b/llvm/utils/TableGen/Common/OptEmitter.cpp
similarity index 100%
rename from llvm/utils/TableGen/OptEmitter.cpp
rename to llvm/utils/TableGen/Common/OptEmitter.cpp
diff --git a/llvm/utils/TableGen/OptEmitter.h b/llvm/utils/TableGen/Common/OptEmitter.h
similarity index 100%
rename from llvm/utils/TableGen/OptEmitter.h
rename to llvm/utils/TableGen/Common/OptEmitter.h
diff --git a/llvm/utils/TableGen/PredicateExpander.cpp b/llvm/utils/TableGen/Common/PredicateExpander.cpp
similarity index 100%
rename from llvm/utils/TableGen/PredicateExpander.cpp
rename to llvm/utils/TableGen/Common/PredicateExpander.cpp
diff --git a/llvm/utils/TableGen/PredicateExpander.h b/llvm/utils/TableGen/Common/PredicateExpander.h
similarity index 100%
rename from llvm/utils/TableGen/PredicateExpander.h
rename to llvm/utils/TableGen/Common/PredicateExpander.h
diff --git a/llvm/utils/TableGen/SubtargetFeatureInfo.cpp b/llvm/utils/TableGen/Common/SubtargetFeatureInfo.cpp
similarity index 100%
rename from llvm/utils/TableGen/SubtargetFeatureInfo.cpp
rename to llvm/utils/TableGen/Common/SubtargetFeatureInfo.cpp
diff --git a/llvm/utils/TableGen/SubtargetFeatureInfo.h b/llvm/utils/TableGen/Common/SubtargetFeatureInfo.h
similarity index 100%
rename from llvm/utils/TableGen/SubtargetFeatureInfo.h
rename to llvm/utils/TableGen/Common/SubtargetFeatureInfo.h
diff --git a/llvm/utils/TableGen/Types.cpp b/llvm/utils/TableGen/Common/Types.cpp
similarity index 100%
rename from llvm/utils/TableGen/Types.cpp
rename to llvm/utils/TableGen/Common/Types.cpp
diff --git a/llvm/utils/TableGen/Types.h b/llvm/utils/TableGen/Common/Types.h
similarity index 100%
rename from llvm/utils/TableGen/Types.h
rename to llvm/utils/TableGen/Common/Types.h
diff --git a/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp
similarity index 97%
rename from llvm/utils/TableGen/VarLenCodeEmitterGen.cpp
rename to llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp
index bfb7e5c33317..4263d8f41715 100644
--- a/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp
+++ b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp
@@ -337,8 +337,8 @@ static void emitInstBits(raw_ostream &IS, raw_ostream &SS, const APInt &Bits,
     return;
   }
 
-  IS.indent(4) << "{/*NumBits*/" << Bits.getBitWidth() << ", "
-               << "/*Index*/" << Index << "},";
+  IS.indent(4) << "{/*NumBits*/" << Bits.getBitWidth() << ", " << "/*Index*/"
+               << Index << "},";
 
   SS.indent(4);
   for (unsigned I = 0; I < Bits.getNumWords(); ++I, ++Index)
@@ -371,8 +371,8 @@ void VarLenCodeEmitterGen::emitInstructionBaseValues(
     if (ModeIt == InstIt->second.end())
       ModeIt = InstIt->second.find(Universal);
     if (ModeIt == InstIt->second.end()) {
-      IS.indent(4) << "{/*NumBits*/0, /*Index*/0},\t"
-                   << "// " << R->getName() << " no encoding\n";
+      IS.indent(4) << "{/*NumBits*/0, /*Index*/0},\t" << "// " << R->getName()
+                   << " no encoding\n";
       continue;
     }
     const VarLenInst &VLI = ModeIt->second;
@@ -492,10 +492,9 @@ std::string VarLenCodeEmitterGen::getInstructionCaseForEncoding(
 
       SS << ", /*Pos=*/" << utostr(Offset) << ", Scratch, Fixups, STI);\n";
 
-      SS.indent(I) << "Inst.insertBits("
-                   << "Scratch.extractBits(" << utostr(NumBits) << ", "
-                   << utostr(LoBit) << ")"
-                   << ", " << Offset << ");\n";
+      SS.indent(I) << "Inst.insertBits(" << "Scratch.extractBits("
+                   << utostr(NumBits) << ", " << utostr(LoBit) << ")" << ", "
+                   << Offset << ");\n";
 
       HighScratchAccess = std::max(HighScratchAccess, NumBits + LoBit);
     }
diff --git a/llvm/utils/TableGen/VarLenCodeEmitterGen.h b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.h
similarity index 100%
rename from llvm/utils/TableGen/VarLenCodeEmitterGen.h
rename to llvm/utils/TableGen/Common/VarLenCodeEmitterGen.h
diff --git a/llvm/utils/TableGen/CompressInstEmitter.cpp b/llvm/utils/TableGen/CompressInstEmitter.cpp
index f703fff0ef3e..fcf77934faac 100644
--- a/llvm/utils/TableGen/CompressInstEmitter.cpp
+++ b/llvm/utils/TableGen/CompressInstEmitter.cpp
@@ -64,9 +64,9 @@
 
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenInstruction.h"
-#include "CodeGenRegisters.h"
-#include "CodeGenTarget.h"
+#include "Common/CodeGenInstruction.h"
+#include "Common/CodeGenRegisters.h"
+#include "Common/CodeGenTarget.h"
 #include "llvm/ADT/IndexedMap.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
diff --git a/llvm/utils/TableGen/DAGISelEmitter.cpp b/llvm/utils/TableGen/DAGISelEmitter.cpp
index 336cee09b90c..b43a8e659dd9 100644
--- a/llvm/utils/TableGen/DAGISelEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelEmitter.cpp
@@ -10,10 +10,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenDAGPatterns.h"
-#include "CodeGenInstruction.h"
-#include "CodeGenTarget.h"
-#include "DAGISelMatcher.h"
+#include "Common/CodeGenDAGPatterns.h"
+#include "Common/CodeGenInstruction.h"
+#include "Common/CodeGenTarget.h"
+#include "Common/DAGISelMatcher.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 533b8c423690..dcecac4380ce 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -10,12 +10,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenDAGPatterns.h"
-#include "CodeGenInstruction.h"
-#include "CodeGenRegisters.h"
-#include "CodeGenTarget.h"
-#include "DAGISelMatcher.h"
-#include "SDNodeProperties.h"
+#include "Basic/SDNodeProperties.h"
+#include "Common/CodeGenDAGPatterns.h"
+#include "Common/CodeGenInstruction.h"
+#include "Common/CodeGenRegisters.h"
+#include "Common/CodeGenTarget.h"
+#include "Common/DAGISelMatcher.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/StringMap.h"
diff --git a/llvm/utils/TableGen/DAGISelMatcherGen.cpp b/llvm/utils/TableGen/DAGISelMatcherGen.cpp
index e8bdabaa0c7e..99babdf07316 100644
--- a/llvm/utils/TableGen/DAGISelMatcherGen.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherGen.cpp
@@ -6,13 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenDAGPatterns.h"
-#include "CodeGenInstruction.h"
-#include "CodeGenRegisters.h"
-#include "CodeGenTarget.h"
-#include "DAGISelMatcher.h"
-#include "InfoByHwMode.h"
-#include "SDNodeProperties.h"
+#include "Basic/SDNodeProperties.h"
+#include "Common/CodeGenDAGPatterns.h"
+#include "Common/CodeGenInstruction.h"
+#include "Common/CodeGenRegisters.h"
+#include "Common/CodeGenTarget.h"
+#include "Common/DAGISelMatcher.h"
+#include "Common/InfoByHwMode.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/TableGen/Error.h"
diff --git a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp
index 047d285f9914..224102e49d98 100644
--- a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp
@@ -10,9 +10,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenDAGPatterns.h"
-#include "DAGISelMatcher.h"
-#include "SDNodeProperties.h"
+#include "Basic/SDNodeProperties.h"
+#include "Common/CodeGenDAGPatterns.h"
+#include "Common/DAGISelMatcher.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
diff --git a/llvm/utils/TableGen/DFAEmitter.cpp b/llvm/utils/TableGen/DFAEmitter.cpp
index ce8cc2a078d7..567184d3d5ee 100644
--- a/llvm/utils/TableGen/DFAEmitter.cpp
+++ b/llvm/utils/TableGen/DFAEmitter.cpp
@@ -21,7 +21,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "DFAEmitter.h"
-#include "SequenceToOffsetTable.h"
+#include "Basic/SequenceToOffsetTable.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/UniqueVector.h"
diff --git a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp
index 26ea1846ffae..3c74df048660 100644
--- a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp
+++ b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp
@@ -14,8 +14,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenSchedule.h"
-#include "CodeGenTarget.h"
+#include "Common/CodeGenSchedule.h"
+#include "Common/CodeGenTarget.h"
 #include "DFAEmitter.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Debug.h"
diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp
index af1efb8aa99f..47c89df35e19 100644
--- a/llvm/utils/TableGen/DXILEmitter.cpp
+++ b/llvm/utils/TableGen/DXILEmitter.cpp
@@ -11,8 +11,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenTarget.h"
-#include "SequenceToOffsetTable.h"
+#include "Basic/SequenceToOffsetTable.h"
+#include "Common/CodeGenTarget.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp
index 732f34ed04c5..494dc93faace 100644
--- a/llvm/utils/TableGen/DecoderEmitter.cpp
+++ b/llvm/utils/TableGen/DecoderEmitter.cpp
@@ -11,12 +11,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenHwModes.h"
-#include "CodeGenInstruction.h"
-#include "CodeGenTarget.h"
-#include "InfoByHwMode.h"
+#include "Common/CodeGenHwModes.h"
+#include "Common/CodeGenInstruction.h"
+#include "Common/CodeGenTarget.h"
+#include "Common/InfoByHwMode.h"
+#include "Common/VarLenCodeEmitterGen.h"
 #include "TableGenBackends.h"
-#include "VarLenCodeEmitterGen.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/CachedHashString.h"
diff --git a/llvm/utils/TableGen/DisassemblerEmitter.cpp b/llvm/utils/TableGen/DisassemblerEmitter.cpp
index 2d653af4d302..d41750075b41 100644
--- a/llvm/utils/TableGen/DisassemblerEmitter.cpp
+++ b/llvm/utils/TableGen/DisassemblerEmitter.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenTarget.h"
+#include "Common/CodeGenTarget.h"
 #include "TableGenBackends.h"
 #include "WebAssemblyDisassemblerEmitter.h"
 #include "X86DisassemblerTables.h"
diff --git a/llvm/utils/TableGen/FastISelEmitter.cpp b/llvm/utils/TableGen/FastISelEmitter.cpp
index f04c6e3b3bf0..acfdc20316b7 100644
--- a/llvm/utils/TableGen/FastISelEmitter.cpp
+++ b/llvm/utils/TableGen/FastISelEmitter.cpp
@@ -16,11 +16,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenDAGPatterns.h"
-#include "CodeGenInstruction.h"
-#include "CodeGenRegisters.h"
-#include "CodeGenTarget.h"
-#include "InfoByHwMode.h"
+#include "Common/CodeGenDAGPatterns.h"
+#include "Common/CodeGenInstruction.h"
+#include "Common/CodeGenRegisters.h"
+#include "Common/CodeGenTarget.h"
+#include "Common/InfoByHwMode.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/TableGen/Error.h"
diff --git a/llvm/utils/TableGen/GlobalISel/CMakeLists.txt b/llvm/utils/TableGen/GlobalISel/CMakeLists.txt
deleted file mode 100644
index 7262c4058399..000000000000
--- a/llvm/utils/TableGen/GlobalISel/CMakeLists.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-set(LLVM_LINK_COMPONENTS
-  Support
-  TableGen
-  )
-
-add_llvm_library(LLVMTableGenGlobalISel STATIC DISABLE_LLVM_LINK_LLVM_DYLIB
-  CodeExpander.cpp
-  CXXPredicates.cpp
-  MatchDataInfo.cpp
-  Patterns.cpp
-
-  DEPENDS
-  vt_gen
-  )
-
-# Users may include its headers as "GlobalISel/*.h"
-target_include_directories(LLVMTableGenGlobalISel
-  INTERFACE
-  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>
-  )
diff --git a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
index dee3cb4d71a4..39b9f8a2ae17 100644
--- a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
@@ -26,18 +26,18 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenInstruction.h"
-#include "CodeGenIntrinsics.h"
-#include "CodeGenTarget.h"
-#include "GlobalISel/CXXPredicates.h"
-#include "GlobalISel/CodeExpander.h"
-#include "GlobalISel/CodeExpansions.h"
-#include "GlobalISel/CombinerUtils.h"
-#include "GlobalISel/MatchDataInfo.h"
-#include "GlobalISel/Patterns.h"
-#include "GlobalISelMatchTable.h"
-#include "GlobalISelMatchTableExecutorEmitter.h"
-#include "SubtargetFeatureInfo.h"
+#include "Basic/CodeGenIntrinsics.h"
+#include "Common/CodeGenInstruction.h"
+#include "Common/CodeGenTarget.h"
+#include "Common/GlobalISel/CXXPredicates.h"
+#include "Common/GlobalISel/CodeExpander.h"
+#include "Common/GlobalISel/CodeExpansions.h"
+#include "Common/GlobalISel/CombinerUtils.h"
+#include "Common/GlobalISel/GlobalISelMatchTable.h"
+#include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h"
+#include "Common/GlobalISel/MatchDataInfo.h"
+#include "Common/GlobalISel/Patterns.h"
+#include "Common/SubtargetFeatureInfo.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/EquivalenceClasses.h"
 #include "llvm/ADT/Hashing.h"
diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp
index e86057422cd7..25e302ce1ca4 100644
--- a/llvm/utils/TableGen/GlobalISelEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp
@@ -30,15 +30,15 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenDAGPatterns.h"
-#include "CodeGenInstruction.h"
-#include "CodeGenIntrinsics.h"
-#include "CodeGenRegisters.h"
-#include "CodeGenTarget.h"
-#include "GlobalISelMatchTable.h"
-#include "GlobalISelMatchTableExecutorEmitter.h"
-#include "InfoByHwMode.h"
-#include "SubtargetFeatureInfo.h"
+#include "Basic/CodeGenIntrinsics.h"
+#include "Common/CodeGenDAGPatterns.h"
+#include "Common/CodeGenInstruction.h"
+#include "Common/CodeGenRegisters.h"
+#include "Common/CodeGenTarget.h"
+#include "Common/GlobalISel/GlobalISelMatchTable.h"
+#include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h"
+#include "Common/InfoByHwMode.h"
+#include "Common/SubtargetFeatureInfo.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGenTypes/LowLevelType.h"
 #include "llvm/CodeGenTypes/MachineValueType.h"
diff --git a/llvm/utils/TableGen/InstrDocsEmitter.cpp b/llvm/utils/TableGen/InstrDocsEmitter.cpp
index efabf6bb7ba6..f948540e18db 100644
--- a/llvm/utils/TableGen/InstrDocsEmitter.cpp
+++ b/llvm/utils/TableGen/InstrDocsEmitter.cpp
@@ -18,9 +18,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenDAGPatterns.h"
-#include "CodeGenInstruction.h"
-#include "CodeGenTarget.h"
+#include "Common/CodeGenDAGPatterns.h"
+#include "Common/CodeGenInstruction.h"
+#include "Common/CodeGenTarget.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
 #include <string>
diff --git a/llvm/utils/TableGen/InstrInfoEmitter.cpp b/llvm/utils/TableGen/InstrInfoEmitter.cpp
index 2d08447429d9..36f8fa146539 100644
--- a/llvm/utils/TableGen/InstrInfoEmitter.cpp
+++ b/llvm/utils/TableGen/InstrInfoEmitter.cpp
@@ -11,15 +11,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenDAGPatterns.h"
-#include "CodeGenInstruction.h"
-#include "CodeGenSchedule.h"
-#include "CodeGenTarget.h"
-#include "PredicateExpander.h"
-#include "SequenceToOffsetTable.h"
-#include "SubtargetFeatureInfo.h"
+#include "Basic/SequenceToOffsetTable.h"
+#include "Common/CodeGenDAGPatterns.h"
+#include "Common/CodeGenInstruction.h"
+#include "Common/CodeGenSchedule.h"
+#include "Common/CodeGenTarget.h"
+#include "Common/PredicateExpander.h"
+#include "Common/SubtargetFeatureInfo.h"
+#include "Common/Types.h"
 #include "TableGenBackends.h"
-#include "Types.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp
index 50a34eac7ca3..a7e99fa4c050 100644
--- a/llvm/utils/TableGen/IntrinsicEmitter.cpp
+++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp
@@ -10,8 +10,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenIntrinsics.h"
-#include "SequenceToOffsetTable.h"
+#include "Basic/CodeGenIntrinsics.h"
+#include "Basic/SequenceToOffsetTable.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
diff --git a/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp
index 91c3b0b4359c..e9e63fa8d0de 100644
--- a/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp
+++ b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp
@@ -38,8 +38,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#include "CodeGenTarget.h"
-#include "PredicateExpander.h"
+#include "Common/CodeGenTarget.h"
+#include "Common/PredicateExpander.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
diff --git a/llvm/utils/TableGen/OptParserEmitter.cpp b/llvm/utils/TableGen/OptParserEmitter.cpp
index c25f6c59cab3..6334af53f88f 100644
--- a/llvm/utils/TableGen/OptParserEmitter.cpp
+++ b/llvm/utils/TableGen/OptParserEmitter.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "OptEmitter.h"
+#include "Common/OptEmitter.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/Twine.h"
diff --git a/llvm/utils/TableGen/OptRSTEmitter.cpp b/llvm/utils/TableGen/OptRSTEmitter.cpp
index 5a7f079dc168..75b7cbdf2988 100644
--- a/llvm/utils/TableGen/OptRSTEmitter.cpp
+++ b/llvm/utils/TableGen/OptRSTEmitter.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "OptEmitter.h"
+#include "Common/OptEmitter.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/TableGen/Record.h"
diff --git a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
index 7f692f29192d..01cfd4a1d982 100644
--- a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
+++ b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenInstruction.h"
-#include "CodeGenTarget.h"
+#include "Common/CodeGenInstruction.h"
+#include "Common/CodeGenTarget.h"
 #include "llvm/ADT/IndexedMap.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
diff --git a/llvm/utils/TableGen/RegisterBankEmitter.cpp b/llvm/utils/TableGen/RegisterBankEmitter.cpp
index 8b59411c5bc3..5546e727af38 100644
--- a/llvm/utils/TableGen/RegisterBankEmitter.cpp
+++ b/llvm/utils/TableGen/RegisterBankEmitter.cpp
@@ -11,9 +11,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenRegisters.h"
-#include "CodeGenTarget.h"
-#include "InfoByHwMode.h"
+#include "Common/CodeGenRegisters.h"
+#include "Common/CodeGenTarget.h"
+#include "Common/InfoByHwMode.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/TableGen/Error.h"
diff --git a/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/llvm/utils/TableGen/RegisterInfoEmitter.cpp
index c4fc1930488c..a1259bff6ba8 100644
--- a/llvm/utils/TableGen/RegisterInfoEmitter.cpp
+++ b/llvm/utils/TableGen/RegisterInfoEmitter.cpp
@@ -12,12 +12,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenHwModes.h"
-#include "CodeGenRegisters.h"
-#include "CodeGenTarget.h"
-#include "InfoByHwMode.h"
-#include "SequenceToOffsetTable.h"
-#include "Types.h"
+#include "Basic/SequenceToOffsetTable.h"
+#include "Common/CodeGenHwModes.h"
+#include "Common/CodeGenRegisters.h"
+#include "Common/CodeGenTarget.h"
+#include "Common/InfoByHwMode.h"
+#include "Common/Types.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/STLExtras.h"
diff --git a/llvm/utils/TableGen/SearchableTableEmitter.cpp b/llvm/utils/TableGen/SearchableTableEmitter.cpp
index 51f18f360ed3..48ee23db957d 100644
--- a/llvm/utils/TableGen/SearchableTableEmitter.cpp
+++ b/llvm/utils/TableGen/SearchableTableEmitter.cpp
@@ -13,8 +13,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenIntrinsics.h"
-#include "CodeGenTarget.h"
+#include "Basic/CodeGenIntrinsics.h"
+#include "Common/CodeGenTarget.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp
index d350d7de139f..2e2c57b802ee 100644
--- a/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -10,10 +10,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenHwModes.h"
-#include "CodeGenSchedule.h"
-#include "CodeGenTarget.h"
-#include "PredicateExpander.h"
+#include "Common/CodeGenHwModes.h"
+#include "Common/CodeGenSchedule.h"
+#include "Common/CodeGenTarget.h"
+#include "Common/PredicateExpander.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/StringExtras.h"
diff --git a/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp b/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp
index 928129f24fcb..e9436ab16e44 100644
--- a/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp
+++ b/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp
@@ -14,7 +14,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "WebAssemblyDisassemblerEmitter.h"
-#include "CodeGenInstruction.h"
+#include "Common/CodeGenInstruction.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Record.h"
diff --git a/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp
index 0a9abbfe186e..c721502a395f 100644
--- a/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp
+++ b/llvm/utils/TableGen/X86CompressEVEXTablesEmitter.cpp
@@ -11,8 +11,8 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenInstruction.h"
-#include "CodeGenTarget.h"
+#include "Common/CodeGenInstruction.h"
+#include "Common/CodeGenTarget.h"
 #include "X86RecognizableInstr.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
diff --git a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
index 1319042e48d0..5871e678b16e 100644
--- a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
+++ b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
@@ -11,8 +11,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenInstruction.h"
-#include "CodeGenTarget.h"
+#include "Common/CodeGenInstruction.h"
+#include "Common/CodeGenTarget.h"
 #include "X86RecognizableInstr.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/FormattedStream.h"
diff --git a/llvm/utils/TableGen/X86MnemonicTables.cpp b/llvm/utils/TableGen/X86MnemonicTables.cpp
index aeafee157462..d9ceed40f7c7 100644
--- a/llvm/utils/TableGen/X86MnemonicTables.cpp
+++ b/llvm/utils/TableGen/X86MnemonicTables.cpp
@@ -11,8 +11,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodeGenInstruction.h"
-#include "CodeGenTarget.h"
+#include "Common/CodeGenInstruction.h"
+#include "Common/CodeGenTarget.h"
 #include "X86RecognizableInstr.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
diff --git a/llvm/utils/TableGen/X86RecognizableInstr.h b/llvm/utils/TableGen/X86RecognizableInstr.h
index 68af68fb5aa0..12fb41750cb3 100644
--- a/llvm/utils/TableGen/X86RecognizableInstr.h
+++ b/llvm/utils/TableGen/X86RecognizableInstr.h
@@ -16,7 +16,7 @@
 #ifndef LLVM_UTILS_TABLEGEN_X86RECOGNIZABLEINSTR_H
 #define LLVM_UTILS_TABLEGEN_X86RECOGNIZABLEINSTR_H
 
-#include "CodeGenInstruction.h"
+#include "Common/CodeGenInstruction.h"
 #include "llvm/Support/X86DisassemblerDecoderCommon.h"
 #include <cstdint>
 #include <string>
-- 
GitLab


From aa962d67ee896f416e285a9298e45fc08ff95eef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= <stefan.graenitz@gmail.com>
Date: Mon, 25 Mar 2024 09:42:41 +0100
Subject: [PATCH 090/404] [clang-repl] Fix Value for platforms where
 unqualified char is unsigned (#86118)

Signedness of unqualified `char` is unspecified and varies between
platforms. This patch adds `Char_U` in `REPL_BUILTIN_TYPES` to account
for platforms that default to `unsigned char`.
---
 clang/include/clang/Interpreter/Value.h         | 1 +
 clang/unittests/Interpreter/InterpreterTest.cpp | 6 ++++++
 2 files changed, 7 insertions(+)

diff --git a/clang/include/clang/Interpreter/Value.h b/clang/include/clang/Interpreter/Value.h
index c380cd91550d..d70e8f871902 100644
--- a/clang/include/clang/Interpreter/Value.h
+++ b/clang/include/clang/Interpreter/Value.h
@@ -76,6 +76,7 @@ class QualType;
   X(bool, Bool)                                                                \
   X(char, Char_S)                                                              \
   X(signed char, SChar)                                                        \
+  X(unsigned char, Char_U)                                                     \
   X(unsigned char, UChar)                                                      \
   X(short, Short)                                                              \
   X(unsigned short, UShort)                                                    \
diff --git a/clang/unittests/Interpreter/InterpreterTest.cpp b/clang/unittests/Interpreter/InterpreterTest.cpp
index e76c0677db5e..69bc2da24288 100644
--- a/clang/unittests/Interpreter/InterpreterTest.cpp
+++ b/clang/unittests/Interpreter/InterpreterTest.cpp
@@ -340,6 +340,12 @@ TEST(InterpreterTest, Value) {
   EXPECT_EQ(V1.getKind(), Value::K_Int);
   EXPECT_FALSE(V1.isManuallyAlloc());
 
+  Value V1b;
+  llvm::cantFail(Interp->ParseAndExecute("char c = 42;"));
+  llvm::cantFail(Interp->ParseAndExecute("c", &V1b));
+  EXPECT_TRUE(V1b.getKind() == Value::K_Char_S ||
+              V1b.getKind() == Value::K_Char_U);
+
   Value V2;
   llvm::cantFail(Interp->ParseAndExecute("double y = 3.14;"));
   llvm::cantFail(Interp->ParseAndExecute("y", &V2));
-- 
GitLab


From 0cf4788d9d0df60980cb48d28aafe7a86aa15a14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= <stefan.graenitz@gmail.com>
Date: Mon, 25 Mar 2024 09:44:25 +0100
Subject: [PATCH 091/404] [clang-repl] Factor out CreateJITBuilder() and allow
 specialization in derived classes (#84461)

The LLJITBuilder interface provides a very convenient way to configure
the ORCv2 JIT engine. IncrementalExecutor already used it internally to
construct the JIT, but didn't provide external access. This patch lifts
control of the creation process to the Interpreter and allows injection
of a custom instance through the extended interface. The Interpreter's
default behavior remains unchanged and the IncrementalExecutor remains
an implementation detail.
---
 clang/include/clang/Interpreter/Interpreter.h |   8 ++
 clang/lib/Interpreter/IncrementalExecutor.cpp |  33 ++---
 clang/lib/Interpreter/IncrementalExecutor.h   |   9 +-
 clang/lib/Interpreter/Interpreter.cpp         |  26 +++-
 .../Interpreter/InterpreterExtensionsTest.cpp | 121 +++++++++++++++++-
 5 files changed, 175 insertions(+), 22 deletions(-)

diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h
index 1dcba1ef9679..970e0245417b 100644
--- a/clang/include/clang/Interpreter/Interpreter.h
+++ b/clang/include/clang/Interpreter/Interpreter.h
@@ -30,6 +30,7 @@
 namespace llvm {
 namespace orc {
 class LLJIT;
+class LLJITBuilder;
 class ThreadSafeContext;
 } // namespace orc
 } // namespace llvm
@@ -127,6 +128,13 @@ protected:
   // custom runtime.
   virtual std::unique_ptr<RuntimeInterfaceBuilder> FindRuntimeInterface();
 
+  // Lazily construct thev ORCv2 JITBuilder. This called when the internal
+  // IncrementalExecutor is created. The default implementation populates an
+  // in-process JIT with debugging support. Override this to configure the JIT
+  // engine used for execution.
+  virtual llvm::Expected<std::unique_ptr<llvm::orc::LLJITBuilder>>
+  CreateJITBuilder(CompilerInstance &CI);
+
 public:
   virtual ~Interpreter();
 
diff --git a/clang/lib/Interpreter/IncrementalExecutor.cpp b/clang/lib/Interpreter/IncrementalExecutor.cpp
index 40bcef94797d..6f036107c14a 100644
--- a/clang/lib/Interpreter/IncrementalExecutor.cpp
+++ b/clang/lib/Interpreter/IncrementalExecutor.cpp
@@ -20,6 +20,7 @@
 #include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h"
 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
+#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h"
@@ -36,26 +37,28 @@ LLVM_ATTRIBUTE_USED void linkComponents() {
 
 namespace clang {
 
+llvm::Expected<std::unique_ptr<llvm::orc::LLJITBuilder>>
+IncrementalExecutor::createDefaultJITBuilder(
+    llvm::orc::JITTargetMachineBuilder JTMB) {
+  auto JITBuilder = std::make_unique<llvm::orc::LLJITBuilder>();
+  JITBuilder->setJITTargetMachineBuilder(std::move(JTMB));
+  JITBuilder->setPrePlatformSetup([](llvm::orc::LLJIT &J) {
+    // Try to enable debugging of JIT'd code (only works with JITLink for
+    // ELF and MachO).
+    consumeError(llvm::orc::enableDebuggerSupport(J));
+    return llvm::Error::success();
+  });
+  return std::move(JITBuilder);
+}
+
 IncrementalExecutor::IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC,
-                                         llvm::Error &Err,
-                                         const clang::TargetInfo &TI)
+                                         llvm::orc::LLJITBuilder &JITBuilder,
+                                         llvm::Error &Err)
     : TSCtx(TSC) {
   using namespace llvm::orc;
   llvm::ErrorAsOutParameter EAO(&Err);
 
-  auto JTMB = JITTargetMachineBuilder(TI.getTriple());
-  JTMB.addFeatures(TI.getTargetOpts().Features);
-  LLJITBuilder Builder;
-  Builder.setJITTargetMachineBuilder(JTMB);
-  Builder.setPrePlatformSetup(
-      [](LLJIT &J) {
-        // Try to enable debugging of JIT'd code (only works with JITLink for
-        // ELF and MachO).
-        consumeError(enableDebuggerSupport(J));
-        return llvm::Error::success();
-      });
-
-  if (auto JitOrErr = Builder.create())
+  if (auto JitOrErr = JITBuilder.create())
     Jit = std::move(*JitOrErr);
   else {
     Err = JitOrErr.takeError();
diff --git a/clang/lib/Interpreter/IncrementalExecutor.h b/clang/lib/Interpreter/IncrementalExecutor.h
index dd0a210a0614..b4347209e14f 100644
--- a/clang/lib/Interpreter/IncrementalExecutor.h
+++ b/clang/lib/Interpreter/IncrementalExecutor.h
@@ -23,7 +23,9 @@
 namespace llvm {
 class Error;
 namespace orc {
+class JITTargetMachineBuilder;
 class LLJIT;
+class LLJITBuilder;
 class ThreadSafeContext;
 } // namespace orc
 } // namespace llvm
@@ -44,8 +46,8 @@ class IncrementalExecutor {
 public:
   enum SymbolNameKind { IRName, LinkerName };
 
-  IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, llvm::Error &Err,
-                      const clang::TargetInfo &TI);
+  IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC,
+                      llvm::orc::LLJITBuilder &JITBuilder, llvm::Error &Err);
   ~IncrementalExecutor();
 
   llvm::Error addModule(PartialTranslationUnit &PTU);
@@ -56,6 +58,9 @@ public:
   getSymbolAddress(llvm::StringRef Name, SymbolNameKind NameKind) const;
 
   llvm::orc::LLJIT &GetExecutionEngine() { return *Jit; }
+
+  static llvm::Expected<std::unique_ptr<llvm::orc::LLJITBuilder>>
+  createDefaultJITBuilder(llvm::orc::JITTargetMachineBuilder JTMB);
 };
 
 } // end namespace clang
diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp
index 7fa52f2f15fc..cf31456b6950 100644
--- a/clang/lib/Interpreter/Interpreter.cpp
+++ b/clang/lib/Interpreter/Interpreter.cpp
@@ -372,15 +372,35 @@ Interpreter::Parse(llvm::StringRef Code) {
   return IncrParser->Parse(Code);
 }
 
+static llvm::Expected<llvm::orc::JITTargetMachineBuilder>
+createJITTargetMachineBuilder(const std::string &TT) {
+  if (TT == llvm::sys::getProcessTriple())
+    // This fails immediately if the target backend is not registered
+    return llvm::orc::JITTargetMachineBuilder::detectHost();
+
+  // If the target backend is not registered, LLJITBuilder::create() will fail
+  return llvm::orc::JITTargetMachineBuilder(llvm::Triple(TT));
+}
+
+llvm::Expected<std::unique_ptr<llvm::orc::LLJITBuilder>>
+Interpreter::CreateJITBuilder(CompilerInstance &CI) {
+  auto JTMB = createJITTargetMachineBuilder(CI.getTargetOpts().Triple);
+  if (!JTMB)
+    return JTMB.takeError();
+  return IncrementalExecutor::createDefaultJITBuilder(std::move(*JTMB));
+}
+
 llvm::Error Interpreter::CreateExecutor() {
-  const clang::TargetInfo &TI =
-      getCompilerInstance()->getASTContext().getTargetInfo();
   if (IncrExecutor)
     return llvm::make_error<llvm::StringError>("Operation failed. "
                                                "Execution engine exists",
                                                std::error_code());
+  llvm::Expected<std::unique_ptr<llvm::orc::LLJITBuilder>> JB =
+      CreateJITBuilder(*getCompilerInstance());
+  if (!JB)
+    return JB.takeError();
   llvm::Error Err = llvm::Error::success();
-  auto Executor = std::make_unique<IncrementalExecutor>(*TSCtx, Err, TI);
+  auto Executor = std::make_unique<IncrementalExecutor>(*TSCtx, **JB, Err);
   if (!Err)
     IncrExecutor = std::move(Executor);
 
diff --git a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp
index b7708616fd24..8bc429d9ec2d 100644
--- a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp
+++ b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp
@@ -18,14 +18,21 @@
 #include "clang/Sema/Sema.h"
 
 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
+#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/Threading.h"
 #include "llvm/Testing/Support/Error.h"
 
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
+
 #include <system_error>
 
+#if defined(_AIX)
+#define CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT
+#endif
+
 using namespace clang;
 namespace {
 
@@ -41,6 +48,10 @@ struct LLVMInitRAII {
   LLVMInitRAII() {
     llvm::InitializeNativeTarget();
     llvm::InitializeNativeTargetAsmPrinter();
+    LLVMInitializeARMTarget();
+    LLVMInitializeARMTargetInfo();
+    LLVMInitializeARMTargetMC();
+    LLVMInitializeARMAsmPrinter();
   }
   ~LLVMInitRAII() { llvm::llvm_shutdown(); }
 } LLVMInit;
@@ -51,12 +62,30 @@ public:
                           llvm::Error &Err)
       : Interpreter(std::move(CI), Err) {}
 
-  llvm::Error testCreateExecutor() { return Interpreter::CreateExecutor(); }
+  llvm::Error testCreateJITBuilderError() {
+    JB = nullptr;
+    return Interpreter::CreateExecutor();
+  }
+
+  llvm::Error testCreateExecutor() {
+    JB = std::make_unique<llvm::orc::LLJITBuilder>();
+    return Interpreter::CreateExecutor();
+  }
 
   void resetExecutor() { Interpreter::ResetExecutor(); }
+
+private:
+  llvm::Expected<std::unique_ptr<llvm::orc::LLJITBuilder>>
+  CreateJITBuilder(CompilerInstance &CI) override {
+    if (JB)
+      return std::move(JB);
+    return llvm::make_error<llvm::StringError>("TestError", std::error_code());
+  }
+
+  std::unique_ptr<llvm::orc::LLJITBuilder> JB;
 };
 
-#ifdef _AIX
+#ifdef CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT
 TEST(InterpreterExtensionsTest, DISABLED_ExecutorCreateReset) {
 #else
 TEST(InterpreterExtensionsTest, ExecutorCreateReset) {
@@ -69,6 +98,8 @@ TEST(InterpreterExtensionsTest, ExecutorCreateReset) {
   llvm::Error ErrOut = llvm::Error::success();
   TestCreateResetExecutor Interp(cantFail(CB.CreateCpp()), ErrOut);
   cantFail(std::move(ErrOut));
+  EXPECT_THAT_ERROR(Interp.testCreateJITBuilderError(),
+                    llvm::FailedWithMessage("TestError"));
   cantFail(Interp.testCreateExecutor());
   Interp.resetExecutor();
   cantFail(Interp.testCreateExecutor());
@@ -126,4 +157,90 @@ TEST(InterpreterExtensionsTest, FindRuntimeInterface) {
   EXPECT_EQ(1U, Interp.RuntimeIBPtr->TransformerQueries);
 }
 
+class CustomJBInterpreter : public Interpreter {
+  using CustomJITBuilderCreatorFunction =
+      std::function<llvm::Expected<std::unique_ptr<llvm::orc::LLJITBuilder>>()>;
+  CustomJITBuilderCreatorFunction JBCreator = nullptr;
+
+public:
+  CustomJBInterpreter(std::unique_ptr<CompilerInstance> CI, llvm::Error &ErrOut)
+      : Interpreter(std::move(CI), ErrOut) {}
+
+  ~CustomJBInterpreter() override {
+    // Skip cleanUp() because it would trigger LLJIT default dtors
+    Interpreter::ResetExecutor();
+  }
+
+  void setCustomJITBuilderCreator(CustomJITBuilderCreatorFunction Fn) {
+    JBCreator = std::move(Fn);
+  }
+
+  llvm::Expected<std::unique_ptr<llvm::orc::LLJITBuilder>>
+  CreateJITBuilder(CompilerInstance &CI) override {
+    if (JBCreator)
+      return JBCreator();
+    return Interpreter::CreateJITBuilder(CI);
+  }
+
+  llvm::Error CreateExecutor() { return Interpreter::CreateExecutor(); }
+};
+
+#ifdef CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT
+TEST(InterpreterExtensionsTest, DISABLED_DefaultCrossJIT) {
+#else
+TEST(InterpreterExtensionsTest, DefaultCrossJIT) {
+#endif
+  IncrementalCompilerBuilder CB;
+  CB.SetTargetTriple("armv6-none-eabi");
+  auto CI = cantFail(CB.CreateCpp());
+  llvm::Error ErrOut = llvm::Error::success();
+  CustomJBInterpreter Interp(std::move(CI), ErrOut);
+  cantFail(std::move(ErrOut));
+  cantFail(Interp.CreateExecutor());
+}
+
+#ifdef CLANG_INTERPRETER_PLATFORM_CANNOT_CREATE_LLJIT
+TEST(InterpreterExtensionsTest, DISABLED_CustomCrossJIT) {
+#else
+TEST(InterpreterExtensionsTest, CustomCrossJIT) {
+#endif
+  std::string TargetTriple = "armv6-none-eabi";
+
+  IncrementalCompilerBuilder CB;
+  CB.SetTargetTriple(TargetTriple);
+  auto CI = cantFail(CB.CreateCpp());
+  llvm::Error ErrOut = llvm::Error::success();
+  CustomJBInterpreter Interp(std::move(CI), ErrOut);
+  cantFail(std::move(ErrOut));
+
+  using namespace llvm::orc;
+  LLJIT *JIT = nullptr;
+  std::vector<std::unique_ptr<llvm::MemoryBuffer>> Objs;
+  Interp.setCustomJITBuilderCreator([&]() {
+    auto JTMB = JITTargetMachineBuilder(llvm::Triple(TargetTriple));
+    JTMB.setCPU("cortex-m0plus");
+    auto JB = std::make_unique<LLJITBuilder>();
+    JB->setJITTargetMachineBuilder(JTMB);
+    JB->setPlatformSetUp(setUpInactivePlatform);
+    JB->setNotifyCreatedCallback([&](LLJIT &J) {
+      ObjectLayer &ObjLayer = J.getObjLinkingLayer();
+      auto *JITLinkObjLayer = llvm::dyn_cast<ObjectLinkingLayer>(&ObjLayer);
+      JITLinkObjLayer->setReturnObjectBuffer(
+          [&Objs](std::unique_ptr<llvm::MemoryBuffer> MB) {
+            Objs.push_back(std::move(MB));
+          });
+      JIT = &J;
+      return llvm::Error::success();
+    });
+    return JB;
+  });
+
+  EXPECT_EQ(0U, Objs.size());
+  cantFail(Interp.CreateExecutor());
+  cantFail(Interp.ParseAndExecute("int a = 1;"));
+  ExecutorAddr Addr = cantFail(JIT->lookup("a"));
+  EXPECT_NE(0U, Addr.getValue());
+  EXPECT_EQ(1U, Objs.size());
+}
+
 } // end anonymous namespace
-- 
GitLab


From 75e528fdd9594ecb6fdb5d9e7bee1506f7e43be0 Mon Sep 17 00:00:00 2001
From: David Stuttard <david.stuttard@amd.com>
Date: Mon, 25 Mar 2024 09:01:46 +0000
Subject: [PATCH 092/404] [AMDGPU] Extend zero initialization of return values
 for TFE (#85759)

buffer_load instructions that use TFE also need to zero initialize
return values similar to how the image instructions currently work. Add
support for this with standard zero init of all results + zero init of
just TFE flag when enable-prt-strict-null subtarget feature is disabled.
---
 .../AMDGPU/AMDGPUInstructionSelector.cpp      |  32 --
 llvm/lib/Target/AMDGPU/BUFInstructions.td     |   5 +-
 llvm/lib/Target/AMDGPU/MIMGInstructions.td    |   1 +
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |  87 ++---
 llvm/lib/Target/AMDGPU/SIISelLowering.h       |   2 +-
 .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp    |   6 +
 llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h |   3 +
 .../llvm.amdgcn.struct.buffer.load.format.ll  |   1 +
 ...vm.amdgcn.struct.ptr.buffer.load.format.ll |   1 +
 .../AMDGPU/llvm.amdgcn.image.msaa.load.ll     | 111 ++++--
 .../llvm.amdgcn.struct.buffer.load.format.ll  | 317 +++++++++++++++++-
 ...vm.amdgcn.struct.ptr.buffer.load.format.ll | 280 ++++++++++++++++
 12 files changed, 741 insertions(+), 105 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 94cc1d90e0ca..e13c13913d4e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -2045,38 +2045,6 @@ bool AMDGPUInstructionSelector::selectImageIntrinsic(
   if (BaseOpcode->HasD16)
     MIB.addImm(IsD16 ? -1 : 0);
 
-  if (IsTexFail) {
-    // An image load instruction with TFE/LWE only conditionally writes to its
-    // result registers. Initialize them to zero so that we always get well
-    // defined result values.
-    assert(VDataOut && !VDataIn);
-    Register Tied = MRI->cloneVirtualRegister(VDataOut);
-    Register Zero = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
-    BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::V_MOV_B32_e32), Zero)
-      .addImm(0);
-    auto Parts = TRI.getRegSplitParts(MRI->getRegClass(Tied), 4);
-    if (STI.usePRTStrictNull()) {
-      // With enable-prt-strict-null enabled, initialize all result registers to
-      // zero.
-      auto RegSeq =
-          BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), Tied);
-      for (auto Sub : Parts)
-        RegSeq.addReg(Zero).addImm(Sub);
-    } else {
-      // With enable-prt-strict-null disabled, only initialize the extra TFE/LWE
-      // result register.
-      Register Undef = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
-      BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::IMPLICIT_DEF), Undef);
-      auto RegSeq =
-          BuildMI(*MBB, *MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), Tied);
-      for (auto Sub : Parts.drop_back(1))
-        RegSeq.addReg(Undef).addImm(Sub);
-      RegSeq.addReg(Zero).addImm(Parts.back());
-    }
-    MIB.addReg(Tied, RegState::Implicit);
-    MIB->tieOperands(0, MIB->getNumOperands() - 1);
-  }
-
   MI.eraseFromParent();
   constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
   TII.enforceOperandRCAlignment(*MIB, AMDGPU::OpName::vaddr);
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index 4ae514ffcf78..273f92abf354 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -86,7 +86,7 @@ class BUF_Pseudo <string opName, dag outs, dag ins,
   bits<1> has_soffset = 1;
   bits<1> has_offset  = 1;
   bits<1> has_slc     = 1;
-  bits<1> tfe         = ?;
+  bits<1> tfe         = 0;
   bits<4> elements    = 0;
   bits<1> has_sccb    = 1;
   bits<1> sccb_value  = 0;
@@ -323,6 +323,7 @@ class MUBUF_Pseudo <string opName, dag outs, dag ins,
   Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret);
   let MUBUF = 1;
   let AsmMatchConverter = "cvtMubuf";
+  let usesCustomInserter = 1;
 }
 
 class MUBUF_Real <MUBUF_Pseudo ps, string real_name = ps.Mnemonic> :
@@ -3369,7 +3370,7 @@ def MUBUFInfoTable : GenericTable {
   let CppTypeName = "MUBUFInfo";
   let Fields = [
     "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset",
-    "IsBufferInv"
+    "IsBufferInv", "tfe"
   ];
 
   let PrimaryKey = ["Opcode"];
diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
index 595ef39ce03e..23e8be0d5e45 100644
--- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td
+++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
@@ -210,6 +210,7 @@ class MIMG <dag outs, string dns = "">
   : MIMG_Base <outs, dns> {
 
   let hasPostISelHook = 1;
+  let usesCustomInserter = 1;
 
   Instruction Opcode = !cast<Instruction>(NAME);
   MIMGBaseOpcode BaseOpcode;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index d437f339a687..81a231f0cade 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -5410,6 +5410,11 @@ MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter(
     return SplitBB;
   }
   default:
+    if (TII->isImage(MI) || TII->isMUBUF(MI)) {
+      if (!MI.mayStore())
+        AddMemOpInit(MI);
+      return BB;
+    }
     return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB);
   }
 }
@@ -15034,60 +15039,67 @@ SDNode *SITargetLowering::PostISelFolding(MachineSDNode *Node,
 // result register that will be written in the case of a memory access failure.
 // The required code is also added to tie this init code to the result of the
 // img instruction.
-void SITargetLowering::AddIMGInit(MachineInstr &MI) const {
+void SITargetLowering::AddMemOpInit(MachineInstr &MI) const {
   const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
   const SIRegisterInfo &TRI = TII->getRegisterInfo();
   MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
   MachineBasicBlock &MBB = *MI.getParent();
 
-  MachineOperand *TFE = TII->getNamedOperand(MI, AMDGPU::OpName::tfe);
-  MachineOperand *LWE = TII->getNamedOperand(MI, AMDGPU::OpName::lwe);
-  MachineOperand *D16 = TII->getNamedOperand(MI, AMDGPU::OpName::d16);
-
-  if (!TFE && !LWE) // intersect_ray
-    return;
+  int DstIdx =
+      AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata);
+  unsigned InitIdx = 0;
 
-  unsigned TFEVal = TFE ? TFE->getImm() : 0;
-  unsigned LWEVal = LWE ? LWE->getImm() : 0;
-  unsigned D16Val = D16 ? D16->getImm() : 0;
+  if (TII->isImage(MI)) {
+    MachineOperand *TFE = TII->getNamedOperand(MI, AMDGPU::OpName::tfe);
+    MachineOperand *LWE = TII->getNamedOperand(MI, AMDGPU::OpName::lwe);
+    MachineOperand *D16 = TII->getNamedOperand(MI, AMDGPU::OpName::d16);
 
-  if (!TFEVal && !LWEVal)
-    return;
+    if (!TFE && !LWE) // intersect_ray
+      return;
 
-  // At least one of TFE or LWE are non-zero
-  // We have to insert a suitable initialization of the result value and
-  // tie this to the dest of the image instruction.
+    unsigned TFEVal = TFE ? TFE->getImm() : 0;
+    unsigned LWEVal = LWE ? LWE->getImm() : 0;
+    unsigned D16Val = D16 ? D16->getImm() : 0;
 
-  const DebugLoc &DL = MI.getDebugLoc();
+    if (!TFEVal && !LWEVal)
+      return;
 
-  int DstIdx =
-      AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata);
+    // At least one of TFE or LWE are non-zero
+    // We have to insert a suitable initialization of the result value and
+    // tie this to the dest of the image instruction.
 
-  // Calculate which dword we have to initialize to 0.
-  MachineOperand *MO_Dmask = TII->getNamedOperand(MI, AMDGPU::OpName::dmask);
+    // Calculate which dword we have to initialize to 0.
+    MachineOperand *MO_Dmask = TII->getNamedOperand(MI, AMDGPU::OpName::dmask);
 
-  // check that dmask operand is found.
-  assert(MO_Dmask && "Expected dmask operand in instruction");
+    // check that dmask operand is found.
+    assert(MO_Dmask && "Expected dmask operand in instruction");
 
-  unsigned dmask = MO_Dmask->getImm();
-  // Determine the number of active lanes taking into account the
-  // Gather4 special case
-  unsigned ActiveLanes = TII->isGather4(MI) ? 4 : llvm::popcount(dmask);
+    unsigned dmask = MO_Dmask->getImm();
+    // Determine the number of active lanes taking into account the
+    // Gather4 special case
+    unsigned ActiveLanes = TII->isGather4(MI) ? 4 : llvm::popcount(dmask);
 
-  bool Packed = !Subtarget->hasUnpackedD16VMem();
+    bool Packed = !Subtarget->hasUnpackedD16VMem();
 
-  unsigned InitIdx =
-      D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1;
+    InitIdx = D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1;
 
-  // Abandon attempt if the dst size isn't large enough
-  // - this is in fact an error but this is picked up elsewhere and
-  // reported correctly.
-  uint32_t DstSize = TRI.getRegSizeInBits(*TII->getOpRegClass(MI, DstIdx)) / 32;
-  if (DstSize < InitIdx)
+    // Abandon attempt if the dst size isn't large enough
+    // - this is in fact an error but this is picked up elsewhere and
+    // reported correctly.
+    uint32_t DstSize =
+        TRI.getRegSizeInBits(*TII->getOpRegClass(MI, DstIdx)) / 32;
+    if (DstSize < InitIdx)
+      return;
+  } else if (TII->isMUBUF(MI) && AMDGPU::getMUBUFTfe(MI.getOpcode())) {
+    InitIdx = TRI.getRegSizeInBits(*TII->getOpRegClass(MI, DstIdx)) / 32;
+  } else {
     return;
+  }
+
+  const DebugLoc &DL = MI.getDebugLoc();
 
   // Create a register for the initialization value.
-  Register PrevDst = MRI.createVirtualRegister(TII->getOpRegClass(MI, DstIdx));
+  Register PrevDst = MRI.cloneVirtualRegister(MI.getOperand(DstIdx).getReg());
   unsigned NewDst = 0; // Final initialized value will be in here
 
   // If PRTStrictNull feature is enabled (the default) then initialize
@@ -15185,11 +15197,8 @@ void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
     return;
   }
 
-  if (TII->isImage(MI)) {
-    if (!MI.mayStore())
-      AddIMGInit(MI);
+  if (TII->isImage(MI))
     TII->enforceOperandRCAlignment(MI, AMDGPU::OpName::vaddr);
-  }
 }
 
 static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL,
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 89da4428e3ab..9856a2923d38 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -466,7 +466,7 @@ public:
 
   SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
   SDNode *PostISelFolding(MachineSDNode *N, SelectionDAG &DAG) const override;
-  void AddIMGInit(MachineInstr &MI) const;
+  void AddMemOpInit(MachineInstr &MI) const;
   void AdjustInstrPostInstrSelection(MachineInstr &MI,
                                      SDNode *Node) const override;
 
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 6d53f68ace70..a90dc32d396f 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -318,6 +318,7 @@ struct MUBUFInfo {
   bool has_srsrc;
   bool has_soffset;
   bool IsBufferInv;
+  bool tfe;
 };
 
 struct MTBUFInfo {
@@ -466,6 +467,11 @@ bool getMUBUFIsBufferInv(unsigned Opc) {
   return Info ? Info->IsBufferInv : false;
 }
 
+bool getMUBUFTfe(unsigned Opc) {
+  const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
+  return Info ? Info->tfe : false;
+}
+
 bool getSMEMIsBuffer(unsigned Opc) {
   const SMInfo *Info = getSMEMOpcodeHelper(Opc);
   return Info ? Info->IsBuffer : false;
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index 29ac402d9535..f4f9a787100b 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -525,6 +525,9 @@ bool getMUBUFHasSoffset(unsigned Opc);
 LLVM_READONLY
 bool getMUBUFIsBufferInv(unsigned Opc);
 
+LLVM_READONLY
+bool getMUBUFTfe(unsigned Opc);
+
 LLVM_READONLY
 bool getSMEMIsBuffer(unsigned Opc);
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll
index 686b849ff58f..06bd45a45cce 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX8 %s
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX12 %s
+; Note that TFE instructions don't have the result initialization to zero due to stopping before finalize-isel - which is where that's inserted
 
 define amdgpu_ps float @struct_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
   ; GFX8-LABEL: name: struct_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll
index 9edc24554911..1e3f94a5e39c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck %s
+; Note that TFE instructions don't have the result initialization to zero due to stopping before finalize-isel - which is where that's inserted
 
 define amdgpu_ps float @struct_ptr_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
   ; CHECK-LABEL: name: struct_ptr_buffer_load_format_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll
index 1348315e72e7..7b1f55e7eeba 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll
@@ -22,18 +22,36 @@ main_body:
 define amdgpu_ps <4 x float> @load_2dmsaa_both(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %fragid) {
 ; GFX11-LABEL: load_2dmsaa_both:
 ; GFX11:       ; %bb.0: ; %main_body
-; GFX11-NEXT:    image_msaa_load v[0:4], v[0:2], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x98,0x02,0x60,0xf0,0x00,0x00,0x60,0x00]
-; GFX11-NEXT:    v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e]
+; GFX11-NEXT:    v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v8, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x08,0x05]
+; GFX11-NEXT:    v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x06,0x07]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; encoding: [0x42,0x02,0x87,0xbf]
+; GFX11-NEXT:    v_mov_b32_e32 v9, v8 ; encoding: [0x08,0x03,0x12,0x7e]
+; GFX11-NEXT:    v_mov_b32_e32 v10, v8 ; encoding: [0x08,0x03,0x14,0x7e]
+; GFX11-NEXT:    v_mov_b32_e32 v11, v8 ; encoding: [0x08,0x03,0x16,0x7e]
+; GFX11-NEXT:    v_mov_b32_e32 v12, v8 ; encoding: [0x08,0x03,0x18,0x7e]
+; GFX11-NEXT:    v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9 ; encoding: [0x08,0x01,0x10,0xca,0x09,0x01,0x00,0x00]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x93,0x01,0x87,0xbf]
+; GFX11-NEXT:    v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11 ; encoding: [0x0a,0x01,0x10,0xca,0x0b,0x01,0x02,0x02]
+; GFX11-NEXT:    v_mov_b32_e32 v4, v12 ; encoding: [0x0c,0x03,0x08,0x7e]
+; GFX11-NEXT:    image_msaa_load v[0:4], v[5:7], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x98,0x02,0x60,0xf0,0x05,0x00,0x60,0x00]
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf]
-; GFX11-NEXT:    global_store_b32 v5, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x05,0x04,0x08,0x00]
+; GFX11-NEXT:    global_store_b32 v8, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x08,0x04,0x08,0x00]
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-LABEL: load_2dmsaa_both:
 ; GFX12:       ; %bb.0: ; %main_body
-; GFX12-NEXT:    image_msaa_load v[0:4], [v0, v1, v2], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x0e,0x20,0x86,0xe4,0x00,0x01,0x00,0x00,0x00,0x01,0x02,0x00]
-; GFX12-NEXT:    v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e]
+; GFX12-NEXT:    v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x08,0x07]
+; GFX12-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v6, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x06,0x05]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x22,0x01,0x87,0xbf]
+; GFX12-NEXT:    v_dual_mov_b32 v9, v8 :: v_dual_mov_b32 v10, v8 ; encoding: [0x08,0x01,0x10,0xca,0x08,0x01,0x0a,0x09]
+; GFX12-NEXT:    v_dual_mov_b32 v11, v8 :: v_dual_mov_b32 v12, v8 ; encoding: [0x08,0x01,0x10,0xca,0x08,0x01,0x0c,0x0b]
+; GFX12-NEXT:    v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9 ; encoding: [0x08,0x01,0x10,0xca,0x09,0x01,0x00,0x00]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x92,0x01,0x87,0xbf]
+; GFX12-NEXT:    v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11 ; encoding: [0x0a,0x01,0x10,0xca,0x0b,0x01,0x02,0x02]
+; GFX12-NEXT:    v_mov_b32_e32 v4, v12 ; encoding: [0x0c,0x03,0x08,0x7e]
+; GFX12-NEXT:    image_msaa_load v[0:4], [v7, v6, v5], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe lwe ; encoding: [0x0e,0x20,0x86,0xe4,0x00,0x01,0x00,0x00,0x07,0x06,0x05,0x00]
 ; GFX12-NEXT:    s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf]
-; GFX12-NEXT:    global_store_b32 v5, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x05,0x00,0x00,0x00]
+; GFX12-NEXT:    global_store_b32 v8, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x08,0x00,0x00,0x00]
 ; GFX12-NEXT:    ; return to shader part epilog
 main_body:
   %v = call {<4 x float>,i32} @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32i32.i32(i32 2, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 3, i32 0)
@@ -63,18 +81,37 @@ main_body:
 define amdgpu_ps <4 x float> @load_2darraymsaa_tfe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %slice, i32 %fragid) {
 ; GFX11-LABEL: load_2darraymsaa_tfe:
 ; GFX11:       ; %bb.0: ; %main_body
-; GFX11-NEXT:    image_msaa_load v[0:4], v[0:3], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x9c,0x08,0x60,0xf0,0x00,0x00,0x20,0x00]
-; GFX11-NEXT:    v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e]
+; GFX11-NEXT:    v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v8, v3 ; encoding: [0x80,0x00,0x10,0xca,0x03,0x01,0x08,0x09]
+; GFX11-NEXT:    v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x06,0x07]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4) ; encoding: [0x42,0x02,0x87,0xbf]
+; GFX11-NEXT:    v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v10, v9 ; encoding: [0x00,0x01,0x10,0xca,0x09,0x01,0x0a,0x05]
+; GFX11-NEXT:    v_mov_b32_e32 v11, v9 ; encoding: [0x09,0x03,0x16,0x7e]
+; GFX11-NEXT:    v_mov_b32_e32 v12, v9 ; encoding: [0x09,0x03,0x18,0x7e]
+; GFX11-NEXT:    v_mov_b32_e32 v13, v9 ; encoding: [0x09,0x03,0x1a,0x7e]
+; GFX11-NEXT:    v_dual_mov_b32 v0, v9 :: v_dual_mov_b32 v1, v10 ; encoding: [0x09,0x01,0x10,0xca,0x0a,0x01,0x00,0x00]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x93,0x01,0x87,0xbf]
+; GFX11-NEXT:    v_dual_mov_b32 v2, v11 :: v_dual_mov_b32 v3, v12 ; encoding: [0x0b,0x01,0x10,0xca,0x0c,0x01,0x02,0x02]
+; GFX11-NEXT:    v_mov_b32_e32 v4, v13 ; encoding: [0x0d,0x03,0x08,0x7e]
+; GFX11-NEXT:    image_msaa_load v[0:4], v[5:8], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x9c,0x08,0x60,0xf0,0x05,0x00,0x20,0x00]
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf]
-; GFX11-NEXT:    global_store_b32 v5, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x05,0x04,0x08,0x00]
+; GFX11-NEXT:    global_store_b32 v9, v4, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x09,0x04,0x08,0x00]
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-LABEL: load_2darraymsaa_tfe:
 ; GFX12:       ; %bb.0: ; %main_body
-; GFX12-NEXT:    image_msaa_load v[0:4], [v0, v1, v2, v3], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x0f,0x20,0x06,0xe6,0x00,0x00,0x00,0x00,0x00,0x01,0x02,0x03]
-; GFX12-NEXT:    v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e]
+; GFX12-NEXT:    v_mov_b32_e32 v9, 0 ; encoding: [0x80,0x02,0x12,0x7e]
+; GFX12-NEXT:    v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v6, v2 ; encoding: [0x03,0x01,0x10,0xca,0x02,0x01,0x06,0x05]
+; GFX12-NEXT:    v_dual_mov_b32 v7, v1 :: v_dual_mov_b32 v8, v0 ; encoding: [0x01,0x01,0x10,0xca,0x00,0x01,0x08,0x07]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x23,0x01,0x87,0xbf]
+; GFX12-NEXT:    v_dual_mov_b32 v10, v9 :: v_dual_mov_b32 v11, v9 ; encoding: [0x09,0x01,0x10,0xca,0x09,0x01,0x0a,0x0a]
+; GFX12-NEXT:    v_dual_mov_b32 v12, v9 :: v_dual_mov_b32 v13, v9 ; encoding: [0x09,0x01,0x10,0xca,0x09,0x01,0x0c,0x0c]
+; GFX12-NEXT:    v_dual_mov_b32 v0, v9 :: v_dual_mov_b32 v1, v10 ; encoding: [0x09,0x01,0x10,0xca,0x0a,0x01,0x00,0x00]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3) ; encoding: [0x92,0x01,0x87,0xbf]
+; GFX12-NEXT:    v_dual_mov_b32 v2, v11 :: v_dual_mov_b32 v3, v12 ; encoding: [0x0b,0x01,0x10,0xca,0x0c,0x01,0x02,0x02]
+; GFX12-NEXT:    v_mov_b32_e32 v4, v13 ; encoding: [0x0d,0x03,0x08,0x7e]
+; GFX12-NEXT:    image_msaa_load v[0:4], [v8, v7, v6, v5], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x0f,0x20,0x06,0xe6,0x00,0x00,0x00,0x00,0x08,0x07,0x06,0x05]
 ; GFX12-NEXT:    s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf]
-; GFX12-NEXT:    global_store_b32 v5, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x05,0x00,0x00,0x00]
+; GFX12-NEXT:    global_store_b32 v9, v4, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x02,0x09,0x00,0x00,0x00]
 ; GFX12-NEXT:    ; return to shader part epilog
 main_body:
   %v = call {<4 x float>,i32} @llvm.amdgcn.image.msaa.load.2darraymsaa.v4f32i32.i32(i32 8, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 1, i32 0)
@@ -155,18 +192,31 @@ main_body:
 define amdgpu_ps <4 x half> @load_2dmsaa_tfe_d16(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %fragid) {
 ; GFX11-LABEL: load_2dmsaa_tfe_d16:
 ; GFX11:       ; %bb.0: ; %main_body
-; GFX11-NEXT:    image_msaa_load v[0:2], v[0:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x98,0x01,0x62,0xf0,0x00,0x00,0x20,0x00]
-; GFX11-NEXT:    v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e]
+; GFX11-NEXT:    v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v6, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x03]
+; GFX11-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x05]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x22,0x01,0x87,0xbf]
+; GFX11-NEXT:    v_mov_b32_e32 v7, v6 ; encoding: [0x06,0x03,0x0e,0x7e]
+; GFX11-NEXT:    v_mov_b32_e32 v8, v6 ; encoding: [0x06,0x03,0x10,0x7e]
+; GFX11-NEXT:    v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7 ; encoding: [0x06,0x01,0x10,0xca,0x07,0x01,0x00,0x00]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf]
+; GFX11-NEXT:    v_mov_b32_e32 v2, v8 ; encoding: [0x08,0x03,0x04,0x7e]
+; GFX11-NEXT:    image_msaa_load v[0:2], v[3:5], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x98,0x01,0x62,0xf0,0x03,0x00,0x20,0x00]
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf]
-; GFX11-NEXT:    global_store_b32 v3, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x03,0x02,0x08,0x00]
+; GFX11-NEXT:    global_store_b32 v6, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x06,0x02,0x08,0x00]
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-LABEL: load_2dmsaa_tfe_d16:
 ; GFX12:       ; %bb.0: ; %main_body
-; GFX12-NEXT:    image_msaa_load v[0:2], [v0, v1, v2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x2e,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x00,0x01,0x02,0x00]
-; GFX12-NEXT:    v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e]
+; GFX12-NEXT:    v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x05]
+; GFX12-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x03]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; encoding: [0x92,0x00,0x87,0xbf]
+; GFX12-NEXT:    v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v8, v6 ; encoding: [0x06,0x01,0x10,0xca,0x06,0x01,0x08,0x07]
+; GFX12-NEXT:    v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7 ; encoding: [0x06,0x01,0x10,0xca,0x07,0x01,0x00,0x00]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf]
+; GFX12-NEXT:    v_mov_b32_e32 v2, v8 ; encoding: [0x08,0x03,0x04,0x7e]
+; GFX12-NEXT:    image_msaa_load v[0:2], [v5, v4, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm tfe d16 ; encoding: [0x2e,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x05,0x04,0x03,0x00]
 ; GFX12-NEXT:    s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf]
-; GFX12-NEXT:    global_store_b32 v3, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x03,0x00,0x00,0x00]
+; GFX12-NEXT:    global_store_b32 v6, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x06,0x00,0x00,0x00]
 ; GFX12-NEXT:    ; return to shader part epilog
 main_body:
   %v = call {<4 x half>,i32} @llvm.amdgcn.image.msaa.load.2dmsaa.v4f16i32.i32(i32 1, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 1, i32 0)
@@ -196,18 +246,31 @@ main_body:
 define amdgpu_ps <4 x half> @load_2darraymsaa_tfe_d16(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %slice, i32 %fragid) {
 ; GFX11-LABEL: load_2darraymsaa_tfe_d16:
 ; GFX11:       ; %bb.0: ; %main_body
-; GFX11-NEXT:    image_msaa_load v[0:2], v[0:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x9c,0x01,0x62,0xf0,0x00,0x00,0x20,0x00]
-; GFX11-NEXT:    v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e]
+; GFX11-NEXT:    v_dual_mov_b32 v6, v0 :: v_dual_mov_b32 v7, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x06]
+; GFX11-NEXT:    v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x04]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; encoding: [0x22,0x01,0x87,0xbf]
+; GFX11-NEXT:    v_mov_b32_e32 v8, v7 ; encoding: [0x07,0x03,0x10,0x7e]
+; GFX11-NEXT:    v_mov_b32_e32 v9, v7 ; encoding: [0x07,0x03,0x12,0x7e]
+; GFX11-NEXT:    v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8 ; encoding: [0x07,0x01,0x10,0xca,0x08,0x01,0x00,0x00]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf]
+; GFX11-NEXT:    v_mov_b32_e32 v2, v9 ; encoding: [0x09,0x03,0x04,0x7e]
+; GFX11-NEXT:    image_msaa_load v[0:2], [v6, v5, v4, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x9d,0x01,0x62,0xf0,0x06,0x00,0x20,0x00,0x05,0x04,0x03,0x00]
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0xf7,0x03,0x89,0xbf]
-; GFX11-NEXT:    global_store_b32 v3, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x03,0x02,0x08,0x00]
+; GFX11-NEXT:    global_store_b32 v7, v2, s[8:9] ; encoding: [0x00,0x00,0x6a,0xdc,0x07,0x02,0x08,0x00]
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-LABEL: load_2darraymsaa_tfe_d16:
 ; GFX12:       ; %bb.0: ; %main_body
-; GFX12-NEXT:    image_msaa_load v[0:2], [v0, v1, v2, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x2f,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x00,0x01,0x02,0x03]
-; GFX12-NEXT:    v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e]
+; GFX12-NEXT:    v_dual_mov_b32 v6, v0 :: v_dual_mov_b32 v7, 0 ; encoding: [0x00,0x01,0x10,0xca,0x80,0x00,0x06,0x06]
+; GFX12-NEXT:    v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v1 ; encoding: [0x02,0x01,0x10,0xca,0x01,0x01,0x04,0x04]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; encoding: [0x92,0x00,0x87,0xbf]
+; GFX12-NEXT:    v_dual_mov_b32 v8, v7 :: v_dual_mov_b32 v9, v7 ; encoding: [0x07,0x01,0x10,0xca,0x07,0x01,0x08,0x08]
+; GFX12-NEXT:    v_dual_mov_b32 v0, v7 :: v_dual_mov_b32 v1, v8 ; encoding: [0x07,0x01,0x10,0xca,0x08,0x01,0x00,0x00]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf]
+; GFX12-NEXT:    v_mov_b32_e32 v2, v9 ; encoding: [0x09,0x03,0x04,0x7e]
+; GFX12-NEXT:    image_msaa_load v[0:2], [v6, v5, v4, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe d16 ; encoding: [0x2f,0x20,0x46,0xe4,0x00,0x00,0x00,0x00,0x06,0x05,0x04,0x03]
 ; GFX12-NEXT:    s_wait_loadcnt 0x0 ; encoding: [0x00,0x00,0xc0,0xbf]
-; GFX12-NEXT:    global_store_b32 v3, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x03,0x00,0x00,0x00]
+; GFX12-NEXT:    global_store_b32 v7, v2, s[8:9] ; encoding: [0x08,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x07,0x00,0x00,0x00]
 ; GFX12-NEXT:    ; return to shader part epilog
 main_body:
   %v = call {<4 x half>,i32} @llvm.amdgcn.image.msaa.load.2darraymsaa.v4f16i32.i32(i32 1, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 1, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll
index 00be32b06de0..ba3d306cc0cf 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll
@@ -2,6 +2,7 @@
 ;RUN: llc < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefixes=GFX6 %s
 ;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefixes=GFX8PLUS %s
 ;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs | FileCheck --check-prefixes=GFX11 %s
+;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-enable-prt-strict-null -verify-machineinstrs | FileCheck --check-prefixes=NOPRT %s
 ;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs | FileCheck --check-prefixes=GFX12,GFX12-SDAG %s
 ;RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs | FileCheck --check-prefixes=GFX12,GFX12-GISEL %s
 
@@ -34,6 +35,16 @@ define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(<4 x i32>
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    v_mov_b32_e32 v8, 0
+; NOPRT-NEXT:    s_clause 0x2
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v8, s[0:3], 0 idxen
+; NOPRT-NEXT:    buffer_load_format_xyzw v[4:7], v8, s[0:3], 0 idxen glc
+; NOPRT-NEXT:    buffer_load_format_xyzw v[8:11], v8, s[0:3], 0 idxen slc
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load:
 ; GFX12:       ; %bb.0: ; %main_body
 ; GFX12-NEXT:    v_mov_b32_e32 v8, 0
@@ -75,6 +86,13 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs(<4 x i32> inreg) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_immoffs:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    v_mov_b32_e32 v0, 0
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:42
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_immoffs:
 ; GFX12:       ; %bb.0: ; %main_body
 ; GFX12-NEXT:    v_mov_b32_e32 v0, 0
@@ -146,6 +164,25 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs_large(<4 x i32> inreg) {
 ; GFX11-NEXT:    v_add_f32_e32 v2, v10, v2
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_immoffs_large:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    v_mov_b32_e32 v8, 0
+; NOPRT-NEXT:    s_movk_i32 s4, 0x7ffc
+; NOPRT-NEXT:    s_clause 0x1
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v8, s[0:3], 60 idxen offset:4092
+; NOPRT-NEXT:    buffer_load_format_xyzw v[4:7], v8, s[0:3], s4 idxen offset:4092
+; NOPRT-NEXT:    s_mov_b32 s4, 0x8ffc
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    v_add_f32_e32 v1, v1, v5
+; NOPRT-NEXT:    buffer_load_format_xyzw v[8:11], v8, s[0:3], s4 idxen offset:4
+; NOPRT-NEXT:    v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v3, v3, v7
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v1, v9, v1
+; NOPRT-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; NOPRT-NEXT:    v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3
+; NOPRT-NEXT:    v_add_f32_e32 v2, v10, v2
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_immoffs_large:
 ; GFX12:       ; %bb.0: ; %main_body
 ; GFX12-NEXT:    v_mov_b32_e32 v8, 0
@@ -196,6 +233,13 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_12bit(<4 x i32> inreg) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_voffset_large_12bit:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    v_mov_b32_e32 v0, 0
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:4092
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_voffset_large_12bit:
 ; GFX12:       ; %bb.0: ; %main_body
 ; GFX12-NEXT:    v_mov_b32_e32 v0, 0
@@ -235,6 +279,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_13bit(<4 x i32> inreg) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_voffset_large_13bit:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    s_mov_b32 s4, 0
+; NOPRT-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; NOPRT-NEXT:    v_dual_mov_b32 v1, 0x1000 :: v_dual_mov_b32 v0, s4
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_voffset_large_13bit:
 ; GFX12:       ; %bb.0: ; %main_body
 ; GFX12-NEXT:    v_mov_b32_e32 v0, 0
@@ -274,6 +327,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_16bit(<4 x i32> inreg) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_voffset_large_16bit:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    s_mov_b32 s4, 0
+; NOPRT-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; NOPRT-NEXT:    v_dual_mov_b32 v1, 0xf000 :: v_dual_mov_b32 v0, s4
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_voffset_large_16bit:
 ; GFX12:       ; %bb.0: ; %main_body
 ; GFX12-NEXT:    v_mov_b32_e32 v0, 0
@@ -313,6 +375,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_23bit(<4 x i32> inreg) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_voffset_large_23bit:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    s_mov_b32 s4, 0
+; NOPRT-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; NOPRT-NEXT:    v_dual_mov_b32 v1, 0x7ff000 :: v_dual_mov_b32 v0, s4
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_voffset_large_23bit:
 ; GFX12:       ; %bb.0: ; %main_body
 ; GFX12-NEXT:    v_mov_b32_e32 v0, 0
@@ -352,6 +423,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_24bit(<4 x i32> inreg) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_voffset_large_24bit:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    s_mov_b32 s4, 0
+; NOPRT-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; NOPRT-NEXT:    v_dual_mov_b32 v1, 0xfff000 :: v_dual_mov_b32 v0, s4
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-SDAG-LABEL: buffer_load_voffset_large_24bit:
 ; GFX12-SDAG:       ; %bb.0: ; %main_body
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, 0x800000 :: v_dual_mov_b32 v0, 0
@@ -389,6 +469,12 @@ define amdgpu_ps <4 x float> @buffer_load_idx(<4 x i32> inreg, i32) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_idx:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_idx:
 ; GFX12:       ; %bb.0: ; %main_body
 ; GFX12-NEXT:    buffer_load_format_xyzw v[0:3], v0, s[0:3], null idxen
@@ -427,6 +513,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs(<4 x i32> inreg, i32) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_ofs:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    s_mov_b32 s4, 0
+; NOPRT-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; NOPRT-NEXT:    v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_ofs:
 ; GFX12:       ; %bb.0: ; %main_body
 ; GFX12-NEXT:    v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, 0
@@ -466,6 +561,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs_imm(<4 x i32> inreg, i32) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_ofs_imm:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    s_mov_b32 s4, 0
+; NOPRT-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; NOPRT-NEXT:    v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:60
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_ofs_imm:
 ; GFX12:       ; %bb.0: ; %main_body
 ; GFX12-NEXT:    v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, 0
@@ -497,6 +601,12 @@ define amdgpu_ps <4 x float> @buffer_load_both(<4 x i32> inreg, i32, i32) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_both:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_both:
 ; GFX12:       ; %bb.0: ; %main_body
 ; GFX12-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], null idxen offen
@@ -529,6 +639,13 @@ define amdgpu_ps <4 x float> @buffer_load_both_reversed(<4 x i32> inreg, i32, i3
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_both_reversed:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    v_mov_b32_e32 v2, v0
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v[1:2], s[0:3], 0 idxen offen
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_both_reversed:
 ; GFX12:       ; %bb.0: ; %main_body
 ; GFX12-NEXT:    v_mov_b32_e32 v2, v0
@@ -562,6 +679,13 @@ define amdgpu_ps float @buffer_load_x(<4 x i32> inreg %rsrc) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_x:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    v_mov_b32_e32 v0, 0
+; NOPRT-NEXT:    buffer_load_format_x v0, v0, s[0:3], 0 idxen
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_x:
 ; GFX12:       ; %bb.0: ; %main_body
 ; GFX12-NEXT:    v_mov_b32_e32 v0, 0
@@ -595,6 +719,13 @@ define amdgpu_ps float @buffer_load_x_i32(<4 x i32> inreg %rsrc) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_x_i32:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    v_mov_b32_e32 v0, 0
+; NOPRT-NEXT:    buffer_load_format_x v0, v0, s[0:3], 0 idxen
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_x_i32:
 ; GFX12:       ; %bb.0: ; %main_body
 ; GFX12-NEXT:    v_mov_b32_e32 v0, 0
@@ -629,6 +760,13 @@ define amdgpu_ps <2 x float> @buffer_load_xy(<4 x i32> inreg %rsrc) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_xy:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    v_mov_b32_e32 v0, 0
+; NOPRT-NEXT:    buffer_load_format_xy v[0:1], v0, s[0:3], 0 idxen
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_xy:
 ; GFX12:       ; %bb.0: ; %main_body
 ; GFX12-NEXT:    v_mov_b32_e32 v0, 0
@@ -644,7 +782,12 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
 ; GFX6-LABEL: buffer_load_v4i32_tfe:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
-; GFX6-NEXT:    buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe
+; GFX6-NEXT:    v_mov_b32_e32 v7, 2
+; GFX6-NEXT:    v_mov_b32_e32 v3, v2
+; GFX6-NEXT:    v_mov_b32_e32 v4, v2
+; GFX6-NEXT:    v_mov_b32_e32 v5, v2
+; GFX6-NEXT:    v_mov_b32_e32 v6, v2
+; GFX6-NEXT:    buffer_load_format_xyzw v[2:6], v7, s[0:3], 0 idxen tfe
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s0, s2
@@ -658,7 +801,12 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
 ; GFX8PLUS-LABEL: buffer_load_v4i32_tfe:
 ; GFX8PLUS:       ; %bb.0:
 ; GFX8PLUS-NEXT:    v_mov_b32_e32 v2, 0
-; GFX8PLUS-NEXT:    buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v7, 2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v3, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v5, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v6, v2
+; GFX8PLUS-NEXT:    buffer_load_format_xyzw v[2:6], v7, s[0:3], 0 idxen tfe
 ; GFX8PLUS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8PLUS-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
 ; GFX8PLUS-NEXT:    v_mov_b32_e32 v0, v6
@@ -667,22 +815,40 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
 ;
 ; GFX11-LABEL: buffer_load_v4i32_tfe:
 ; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_mov_b32_e32 v2, 0
-; GFX11-NEXT:    buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe
+; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v7, 2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_mov_b32_e32 v3, v2
+; GFX11-NEXT:    v_mov_b32_e32 v4, v2
+; GFX11-NEXT:    v_mov_b32_e32 v5, v2
+; GFX11-NEXT:    v_mov_b32_e32 v6, v2
+; GFX11-NEXT:    buffer_load_format_xyzw v[2:6], v7, s[0:3], 0 idxen tfe
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    global_store_b128 v[0:1], v[2:5], off
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v6
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_v4i32_tfe:
+; NOPRT:       ; %bb.0:
+; NOPRT-NEXT:    v_mov_b32_e32 v2, 2
+; NOPRT-NEXT:    v_mov_b32_e32 v6, 0
+; NOPRT-NEXT:    buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; NOPRT-NEXT:    v_mov_b32_e32 v0, v6
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_v4i32_tfe:
 ; GFX12:       ; %bb.0:
-; GFX12-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-NEXT:    buffer_load_format_xyzw v[2:6], v2, s[0:3], null idxen tfe
+; GFX12-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v7, 2
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2
+; GFX12-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v6, v2
+; GFX12-NEXT:    buffer_load_format_xyzw v[2:6], v7, s[0:3], null idxen tfe
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_store_b128 v[0:1], v[2:5], off
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v6
 ; GFX12-NEXT:    ; return to shader part epilog
-  %load = call { <4 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.format.sl_v4i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
+  %load = call { <4 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.format.sl_v4i32i32s(<4 x i32> %rsrc, i32 2, i32 0, i32 0, i32 0)
   %data = extractvalue { <4 x i32>, i32 } %load, 0
   store <4 x i32> %data, ptr addrspace(1) %out
   %status = extractvalue { <4 x i32>, i32 } %load, 1
@@ -694,6 +860,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
 ; GFX6-LABEL: buffer_load_v4f32_tfe:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, v2
+; GFX6-NEXT:    v_mov_b32_e32 v4, v2
+; GFX6-NEXT:    v_mov_b32_e32 v5, v2
+; GFX6-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX6-NEXT:    buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
@@ -708,6 +878,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
 ; GFX8PLUS-LABEL: buffer_load_v4f32_tfe:
 ; GFX8PLUS:       ; %bb.0:
 ; GFX8PLUS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v3, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v5, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX8PLUS-NEXT:    buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe
 ; GFX8PLUS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8PLUS-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
@@ -718,15 +892,32 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
 ; GFX11-LABEL: buffer_load_v4f32_tfe:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_mov_b32_e32 v3, v2
+; GFX11-NEXT:    v_mov_b32_e32 v4, v2
+; GFX11-NEXT:    v_mov_b32_e32 v5, v2
+; GFX11-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX11-NEXT:    buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    global_store_b128 v[0:1], v[2:5], off
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v6
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_v4f32_tfe:
+; NOPRT:       ; %bb.0:
+; NOPRT-NEXT:    v_mov_b32_e32 v6, 0
+; NOPRT-NEXT:    buffer_load_format_xyzw v[2:6], v6, s[0:3], 0 idxen tfe
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; NOPRT-NEXT:    v_mov_b32_e32 v0, v6
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_v4f32_tfe:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2
+; GFX12-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v6, v2
 ; GFX12-NEXT:    buffer_load_format_xyzw v[2:6], v2, s[0:3], null idxen tfe
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_store_b128 v[0:1], v[2:5], off
@@ -744,6 +935,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
 ; GFX6-LABEL: buffer_load_v3i32_tfe:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, v2
+; GFX6-NEXT:    v_mov_b32_e32 v4, v2
+; GFX6-NEXT:    v_mov_b32_e32 v5, v2
 ; GFX6-NEXT:    buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
@@ -759,6 +953,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
 ; GFX8PLUS-LABEL: buffer_load_v3i32_tfe:
 ; GFX8PLUS:       ; %bb.0:
 ; GFX8PLUS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v3, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v5, v2
 ; GFX8PLUS-NEXT:    buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe
 ; GFX8PLUS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8PLUS-NEXT:    flat_store_dwordx3 v[0:1], v[2:4]
@@ -769,15 +966,31 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
 ; GFX11-LABEL: buffer_load_v3i32_tfe:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_mov_b32_e32 v3, v2
+; GFX11-NEXT:    v_mov_b32_e32 v4, v2
+; GFX11-NEXT:    v_mov_b32_e32 v5, v2
 ; GFX11-NEXT:    buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    global_store_b96 v[0:1], v[2:4], off
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v5
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_v3i32_tfe:
+; NOPRT:       ; %bb.0:
+; NOPRT-NEXT:    v_mov_b32_e32 v5, 0
+; NOPRT-NEXT:    buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    global_store_b96 v[0:1], v[2:4], off
+; NOPRT-NEXT:    v_mov_b32_e32 v0, v5
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_v3i32_tfe:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2
+; GFX12-NEXT:    v_mov_b32_e32 v5, v2
 ; GFX12-NEXT:    buffer_load_format_xyz v[2:5], v2, s[0:3], null idxen tfe
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_store_b96 v[0:1], v[2:4], off
@@ -795,6 +1008,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
 ; GFX6-LABEL: buffer_load_v3f32_tfe:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, v2
+; GFX6-NEXT:    v_mov_b32_e32 v4, v2
+; GFX6-NEXT:    v_mov_b32_e32 v5, v2
 ; GFX6-NEXT:    buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
@@ -810,6 +1026,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
 ; GFX8PLUS-LABEL: buffer_load_v3f32_tfe:
 ; GFX8PLUS:       ; %bb.0:
 ; GFX8PLUS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v3, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v5, v2
 ; GFX8PLUS-NEXT:    buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe
 ; GFX8PLUS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8PLUS-NEXT:    flat_store_dwordx3 v[0:1], v[2:4]
@@ -820,15 +1039,31 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
 ; GFX11-LABEL: buffer_load_v3f32_tfe:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_mov_b32_e32 v3, v2
+; GFX11-NEXT:    v_mov_b32_e32 v4, v2
+; GFX11-NEXT:    v_mov_b32_e32 v5, v2
 ; GFX11-NEXT:    buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    global_store_b96 v[0:1], v[2:4], off
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v5
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_v3f32_tfe:
+; NOPRT:       ; %bb.0:
+; NOPRT-NEXT:    v_mov_b32_e32 v5, 0
+; NOPRT-NEXT:    buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    global_store_b96 v[0:1], v[2:4], off
+; NOPRT-NEXT:    v_mov_b32_e32 v0, v5
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_v3f32_tfe:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2
+; GFX12-NEXT:    v_mov_b32_e32 v5, v2
 ; GFX12-NEXT:    buffer_load_format_xyz v[2:5], v2, s[0:3], null idxen tfe
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_store_b96 v[0:1], v[2:4], off
@@ -846,6 +1081,9 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
 ; GFX6-LABEL: buffer_load_v2i32_tfe:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, v2
+; GFX6-NEXT:    v_mov_b32_e32 v4, v2
+; GFX6-NEXT:    v_mov_b32_e32 v5, v2
 ; GFX6-NEXT:    buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
@@ -860,6 +1098,8 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
 ; GFX8PLUS-LABEL: buffer_load_v2i32_tfe:
 ; GFX8PLUS:       ; %bb.0:
 ; GFX8PLUS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v3, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v4, v2
 ; GFX8PLUS-NEXT:    buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe
 ; GFX8PLUS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8PLUS-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
@@ -870,15 +1110,29 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
 ; GFX11-LABEL: buffer_load_v2i32_tfe:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_mov_b32_e32 v3, v2
+; GFX11-NEXT:    v_mov_b32_e32 v4, v2
 ; GFX11-NEXT:    buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    global_store_b64 v[0:1], v[2:3], off
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_v2i32_tfe:
+; NOPRT:       ; %bb.0:
+; NOPRT-NEXT:    v_mov_b32_e32 v4, 0
+; NOPRT-NEXT:    buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    global_store_b64 v[0:1], v[2:3], off
+; NOPRT-NEXT:    v_mov_b32_e32 v0, v4
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_v2i32_tfe:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2
 ; GFX12-NEXT:    buffer_load_format_xy v[2:4], v2, s[0:3], null idxen tfe
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_store_b64 v[0:1], v[2:3], off
@@ -896,6 +1150,9 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
 ; GFX6-LABEL: buffer_load_v2f32_tfe:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, v2
+; GFX6-NEXT:    v_mov_b32_e32 v4, v2
+; GFX6-NEXT:    v_mov_b32_e32 v5, v2
 ; GFX6-NEXT:    buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
@@ -910,6 +1167,8 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
 ; GFX8PLUS-LABEL: buffer_load_v2f32_tfe:
 ; GFX8PLUS:       ; %bb.0:
 ; GFX8PLUS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v3, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v4, v2
 ; GFX8PLUS-NEXT:    buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe
 ; GFX8PLUS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8PLUS-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
@@ -920,15 +1179,29 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
 ; GFX11-LABEL: buffer_load_v2f32_tfe:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_mov_b32_e32 v3, v2
+; GFX11-NEXT:    v_mov_b32_e32 v4, v2
 ; GFX11-NEXT:    buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    global_store_b64 v[0:1], v[2:3], off
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_v2f32_tfe:
+; NOPRT:       ; %bb.0:
+; NOPRT-NEXT:    v_mov_b32_e32 v4, 0
+; NOPRT-NEXT:    buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    global_store_b64 v[0:1], v[2:3], off
+; NOPRT-NEXT:    v_mov_b32_e32 v0, v4
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_v2f32_tfe:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2
 ; GFX12-NEXT:    buffer_load_format_xy v[2:4], v2, s[0:3], null idxen tfe
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_store_b64 v[0:1], v[2:3], off
@@ -946,6 +1219,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrspace
 ; GFX6-LABEL: buffer_load_i32_tfe:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX6-NEXT:    buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
@@ -960,6 +1234,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrspace
 ; GFX8PLUS-LABEL: buffer_load_i32_tfe:
 ; GFX8PLUS:       ; %bb.0:
 ; GFX8PLUS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX8PLUS-NEXT:    buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe
 ; GFX8PLUS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8PLUS-NEXT:    flat_store_dword v[0:1], v2
@@ -970,15 +1245,28 @@ define amdgpu_cs float @buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrspace
 ; GFX11-LABEL: buffer_load_i32_tfe:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX11-NEXT:    buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    global_store_b32 v[0:1], v2, off
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_i32_tfe:
+; NOPRT:       ; %bb.0:
+; NOPRT-NEXT:    v_mov_b32_e32 v3, 0
+; NOPRT-NEXT:    buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    global_store_b32 v[0:1], v2, off
+; NOPRT-NEXT:    v_mov_b32_e32 v0, v3
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_i32_tfe:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX12-NEXT:    buffer_load_format_x v[2:3], v2, s[0:3], null idxen tfe
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_store_b32 v[0:1], v2, off
@@ -996,6 +1284,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(<4 x i32> inreg %rsrc, ptr addrspace
 ; GFX6-LABEL: buffer_load_f32_tfe:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX6-NEXT:    buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
@@ -1010,6 +1299,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(<4 x i32> inreg %rsrc, ptr addrspace
 ; GFX8PLUS-LABEL: buffer_load_f32_tfe:
 ; GFX8PLUS:       ; %bb.0:
 ; GFX8PLUS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX8PLUS-NEXT:    buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe
 ; GFX8PLUS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8PLUS-NEXT:    flat_store_dword v[0:1], v2
@@ -1020,15 +1310,28 @@ define amdgpu_cs float @buffer_load_f32_tfe(<4 x i32> inreg %rsrc, ptr addrspace
 ; GFX11-LABEL: buffer_load_f32_tfe:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX11-NEXT:    buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    global_store_b32 v[0:1], v2, off
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX11-NEXT:    ; return to shader part epilog
 ;
+; NOPRT-LABEL: buffer_load_f32_tfe:
+; NOPRT:       ; %bb.0:
+; NOPRT-NEXT:    v_mov_b32_e32 v3, 0
+; NOPRT-NEXT:    buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    global_store_b32 v[0:1], v2, off
+; NOPRT-NEXT:    v_mov_b32_e32 v0, v3
+; NOPRT-NEXT:    ; return to shader part epilog
+;
 ; GFX12-LABEL: buffer_load_f32_tfe:
 ; GFX12:       ; %bb.0:
 ; GFX12-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX12-NEXT:    buffer_load_format_x v[2:3], v2, s[0:3], null idxen tfe
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_store_b32 v[0:1], v2, off
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll
index b0bd4e428ef2..c5202b84fa1e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll
@@ -2,6 +2,7 @@
 ;RUN: llc < %s -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefixes=GFX6 %s
 ;RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefixes=GFX8PLUS %s
 ;RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs | FileCheck --check-prefixes=GFX11 %s
+;RUN: llc < %s -mtriple=amdgcn -mattr=-enable-prt-strict-null -mcpu=gfx1100 -verify-machineinstrs | FileCheck --check-prefixes=NOPRT %s
 
 define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(ptr addrspace(8) inreg) {
 ; GFX6-LABEL: buffer_load:
@@ -31,6 +32,16 @@ define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(ptr addrsp
 ; GFX11-NEXT:    buffer_load_format_xyzw v[8:11], v8, s[0:3], 0 idxen slc
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    v_mov_b32_e32 v8, 0
+; NOPRT-NEXT:    s_clause 0x2
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v8, s[0:3], 0 idxen
+; NOPRT-NEXT:    buffer_load_format_xyzw v[4:7], v8, s[0:3], 0 idxen glc
+; NOPRT-NEXT:    buffer_load_format_xyzw v[8:11], v8, s[0:3], 0 idxen slc
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
 main_body:
   %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 0, i32 0)
   %data_glc = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 0, i32 0, i32 1)
@@ -62,6 +73,13 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs(ptr addrspace(8) inreg) {
 ; GFX11-NEXT:    buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:42
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_immoffs:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    v_mov_b32_e32 v0, 0
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:42
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
 main_body:
   %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 42, i32 0, i32 0)
   ret <4 x float> %data
@@ -126,6 +144,25 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs_large(ptr addrspace(8) inreg)
 ; GFX11-NEXT:    v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3
 ; GFX11-NEXT:    v_add_f32_e32 v2, v10, v2
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_immoffs_large:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    v_mov_b32_e32 v8, 0
+; NOPRT-NEXT:    s_movk_i32 s4, 0x7ffc
+; NOPRT-NEXT:    s_clause 0x1
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v8, s[0:3], 60 idxen offset:4092
+; NOPRT-NEXT:    buffer_load_format_xyzw v[4:7], v8, s[0:3], s4 idxen offset:4092
+; NOPRT-NEXT:    s_mov_b32 s4, 0x8ffc
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    v_add_f32_e32 v1, v1, v5
+; NOPRT-NEXT:    buffer_load_format_xyzw v[8:11], v8, s[0:3], s4 idxen offset:4
+; NOPRT-NEXT:    v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v3, v3, v7
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v1, v9, v1
+; NOPRT-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; NOPRT-NEXT:    v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3
+; NOPRT-NEXT:    v_add_f32_e32 v2, v10, v2
+; NOPRT-NEXT:    ; return to shader part epilog
 main_body:
   %d.0 = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 4092, i32 60, i32 0)
   %d.1 = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 4092, i32 32764, i32 0)
@@ -156,6 +193,13 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_12bit(ptr addrspace(8) i
 ; GFX11-NEXT:    buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:4092
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_voffset_large_12bit:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    v_mov_b32_e32 v0, 0
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen offset:4092
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
 main_body:
   %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 4092, i32 0, i32 0)
   ret <4 x float> %data
@@ -188,6 +232,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_13bit(ptr addrspace(8) i
 ; GFX11-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_voffset_large_13bit:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    s_mov_b32 s4, 0
+; NOPRT-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; NOPRT-NEXT:    v_dual_mov_b32 v1, 0x1000 :: v_dual_mov_b32 v0, s4
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
 main_body:
   %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 8188, i32 0, i32 0)
   ret <4 x float> %data
@@ -220,6 +273,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_16bit(ptr addrspace(8) i
 ; GFX11-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_voffset_large_16bit:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    s_mov_b32 s4, 0
+; NOPRT-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; NOPRT-NEXT:    v_dual_mov_b32 v1, 0xf000 :: v_dual_mov_b32 v0, s4
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
 main_body:
   %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 65532, i32 0, i32 0)
   ret <4 x float> %data
@@ -252,6 +314,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_23bit(ptr addrspace(8) i
 ; GFX11-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_voffset_large_23bit:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    s_mov_b32 s4, 0
+; NOPRT-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; NOPRT-NEXT:    v_dual_mov_b32 v1, 0x7ff000 :: v_dual_mov_b32 v0, s4
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
 main_body:
   %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 8388604, i32 0, i32 0)
   ret <4 x float> %data
@@ -284,6 +355,15 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_24bit(ptr addrspace(8) i
 ; GFX11-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_voffset_large_24bit:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    s_mov_b32 s4, 0
+; NOPRT-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; NOPRT-NEXT:    v_dual_mov_b32 v1, 0xfff000 :: v_dual_mov_b32 v0, s4
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:4092
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
 main_body:
   %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 16777212, i32 0, i32 0)
   ret <4 x float> %data
@@ -307,6 +387,12 @@ define amdgpu_ps <4 x float> @buffer_load_idx(ptr addrspace(8) inreg, i32) {
 ; GFX11-NEXT:    buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_idx:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 idxen
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
 main_body:
   %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 %1, i32 0, i32 0, i32 0)
   ret <4 x float> %data
@@ -339,6 +425,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs(ptr addrspace(8) inreg, i32) {
 ; GFX11-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_ofs:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    s_mov_b32 s4, 0
+; NOPRT-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; NOPRT-NEXT:    v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
 main_body:
   %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 %1, i32 0, i32 0)
   ret <4 x float> %data
@@ -371,6 +466,15 @@ define amdgpu_ps <4 x float> @buffer_load_ofs_imm(ptr addrspace(8) inreg, i32) {
 ; GFX11-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:60
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_ofs_imm:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    s_mov_b32 s4, 0
+; NOPRT-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; NOPRT-NEXT:    v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s4
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen offset:60
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
 main_body:
   %ofs = add i32 %1, 60
   %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 %ofs, i32 0, i32 0)
@@ -395,6 +499,12 @@ define amdgpu_ps <4 x float> @buffer_load_both(ptr addrspace(8) inreg, i32, i32)
 ; GFX11-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_both:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v[0:1], s[0:3], 0 idxen offen
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
 main_body:
   %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 %1, i32 %2, i32 0, i32 0)
   ret <4 x float> %data
@@ -421,6 +531,13 @@ define amdgpu_ps <4 x float> @buffer_load_both_reversed(ptr addrspace(8) inreg,
 ; GFX11-NEXT:    buffer_load_format_xyzw v[0:3], v[1:2], s[0:3], 0 idxen offen
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_both_reversed:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    v_mov_b32_e32 v2, v0
+; NOPRT-NEXT:    buffer_load_format_xyzw v[0:3], v[1:2], s[0:3], 0 idxen offen
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
 main_body:
   %data = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 %2, i32 %1, i32 0, i32 0)
   ret <4 x float> %data
@@ -447,6 +564,13 @@ define amdgpu_ps float @buffer_load_x(ptr addrspace(8) inreg %rsrc) {
 ; GFX11-NEXT:    buffer_load_format_x v0, v0, s[0:3], 0 idxen
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_x:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    v_mov_b32_e32 v0, 0
+; NOPRT-NEXT:    buffer_load_format_x v0, v0, s[0:3], 0 idxen
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
 main_body:
   %data = call float @llvm.amdgcn.struct.ptr.buffer.load.format.f32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
   ret float %data
@@ -473,6 +597,13 @@ define amdgpu_ps float @buffer_load_x_i32(ptr addrspace(8) inreg %rsrc) {
 ; GFX11-NEXT:    buffer_load_format_x v0, v0, s[0:3], 0 idxen
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_x_i32:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    v_mov_b32_e32 v0, 0
+; NOPRT-NEXT:    buffer_load_format_x v0, v0, s[0:3], 0 idxen
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
 main_body:
   %data = call i32 @llvm.amdgcn.struct.ptr.buffer.load.format.i32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
   %fdata = bitcast i32 %data to float
@@ -500,6 +631,13 @@ define amdgpu_ps <2 x float> @buffer_load_xy(ptr addrspace(8) inreg %rsrc) {
 ; GFX11-NEXT:    buffer_load_format_xy v[0:1], v0, s[0:3], 0 idxen
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_xy:
+; NOPRT:       ; %bb.0: ; %main_body
+; NOPRT-NEXT:    v_mov_b32_e32 v0, 0
+; NOPRT-NEXT:    buffer_load_format_xy v[0:1], v0, s[0:3], 0 idxen
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    ; return to shader part epilog
 main_body:
   %data = call <2 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v2f32(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
   ret <2 x float> %data
@@ -509,6 +647,10 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(ptr addrspace(8) inreg %rsrc, ptr
 ; GFX6-LABEL: buffer_load_v4i32_tfe:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, v2
+; GFX6-NEXT:    v_mov_b32_e32 v4, v2
+; GFX6-NEXT:    v_mov_b32_e32 v5, v2
+; GFX6-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX6-NEXT:    buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
@@ -523,6 +665,10 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(ptr addrspace(8) inreg %rsrc, ptr
 ; GFX8PLUS-LABEL: buffer_load_v4i32_tfe:
 ; GFX8PLUS:       ; %bb.0:
 ; GFX8PLUS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v3, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v5, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX8PLUS-NEXT:    buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe
 ; GFX8PLUS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8PLUS-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
@@ -533,11 +679,25 @@ define amdgpu_cs float @buffer_load_v4i32_tfe(ptr addrspace(8) inreg %rsrc, ptr
 ; GFX11-LABEL: buffer_load_v4i32_tfe:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_mov_b32_e32 v3, v2
+; GFX11-NEXT:    v_mov_b32_e32 v4, v2
+; GFX11-NEXT:    v_mov_b32_e32 v5, v2
+; GFX11-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX11-NEXT:    buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    global_store_b128 v[0:1], v[2:5], off
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v6
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_v4i32_tfe:
+; NOPRT:       ; %bb.0:
+; NOPRT-NEXT:    v_mov_b32_e32 v6, 0
+; NOPRT-NEXT:    buffer_load_format_xyzw v[2:6], v6, s[0:3], 0 idxen tfe
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; NOPRT-NEXT:    v_mov_b32_e32 v0, v6
+; NOPRT-NEXT:    ; return to shader part epilog
   %load = call { <4 x i32>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v4i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
   %data = extractvalue { <4 x i32>, i32 } %load, 0
   store <4 x i32> %data, ptr addrspace(1) %out
@@ -550,6 +710,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(ptr addrspace(8) inreg %rsrc, ptr
 ; GFX6-LABEL: buffer_load_v4f32_tfe:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, v2
+; GFX6-NEXT:    v_mov_b32_e32 v4, v2
+; GFX6-NEXT:    v_mov_b32_e32 v5, v2
+; GFX6-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX6-NEXT:    buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
@@ -564,6 +728,10 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(ptr addrspace(8) inreg %rsrc, ptr
 ; GFX8PLUS-LABEL: buffer_load_v4f32_tfe:
 ; GFX8PLUS:       ; %bb.0:
 ; GFX8PLUS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v3, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v5, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX8PLUS-NEXT:    buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe
 ; GFX8PLUS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8PLUS-NEXT:    flat_store_dwordx4 v[0:1], v[2:5]
@@ -574,11 +742,25 @@ define amdgpu_cs float @buffer_load_v4f32_tfe(ptr addrspace(8) inreg %rsrc, ptr
 ; GFX11-LABEL: buffer_load_v4f32_tfe:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_mov_b32_e32 v3, v2
+; GFX11-NEXT:    v_mov_b32_e32 v4, v2
+; GFX11-NEXT:    v_mov_b32_e32 v5, v2
+; GFX11-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX11-NEXT:    buffer_load_format_xyzw v[2:6], v2, s[0:3], 0 idxen tfe
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    global_store_b128 v[0:1], v[2:5], off
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v6
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_v4f32_tfe:
+; NOPRT:       ; %bb.0:
+; NOPRT-NEXT:    v_mov_b32_e32 v6, 0
+; NOPRT-NEXT:    buffer_load_format_xyzw v[2:6], v6, s[0:3], 0 idxen tfe
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; NOPRT-NEXT:    v_mov_b32_e32 v0, v6
+; NOPRT-NEXT:    ; return to shader part epilog
   %load = call { <4 x float>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v4f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
   %data = extractvalue { <4 x float>, i32 } %load, 0
   store <4 x float> %data, ptr addrspace(1) %out
@@ -591,6 +773,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(ptr addrspace(8) inreg %rsrc, ptr
 ; GFX6-LABEL: buffer_load_v3i32_tfe:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, v2
+; GFX6-NEXT:    v_mov_b32_e32 v4, v2
+; GFX6-NEXT:    v_mov_b32_e32 v5, v2
 ; GFX6-NEXT:    buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
@@ -606,6 +791,9 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(ptr addrspace(8) inreg %rsrc, ptr
 ; GFX8PLUS-LABEL: buffer_load_v3i32_tfe:
 ; GFX8PLUS:       ; %bb.0:
 ; GFX8PLUS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v3, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v5, v2
 ; GFX8PLUS-NEXT:    buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe
 ; GFX8PLUS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8PLUS-NEXT:    flat_store_dwordx3 v[0:1], v[2:4]
@@ -616,11 +804,24 @@ define amdgpu_cs float @buffer_load_v3i32_tfe(ptr addrspace(8) inreg %rsrc, ptr
 ; GFX11-LABEL: buffer_load_v3i32_tfe:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_mov_b32_e32 v3, v2
+; GFX11-NEXT:    v_mov_b32_e32 v4, v2
+; GFX11-NEXT:    v_mov_b32_e32 v5, v2
 ; GFX11-NEXT:    buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    global_store_b96 v[0:1], v[2:4], off
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v5
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_v3i32_tfe:
+; NOPRT:       ; %bb.0:
+; NOPRT-NEXT:    v_mov_b32_e32 v5, 0
+; NOPRT-NEXT:    buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    global_store_b96 v[0:1], v[2:4], off
+; NOPRT-NEXT:    v_mov_b32_e32 v0, v5
+; NOPRT-NEXT:    ; return to shader part epilog
   %load = call { <3 x i32>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
   %data = extractvalue { <3 x i32>, i32 } %load, 0
   store <3 x i32> %data, ptr addrspace(1) %out
@@ -633,6 +834,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(ptr addrspace(8) inreg %rsrc, ptr
 ; GFX6-LABEL: buffer_load_v3f32_tfe:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, v2
+; GFX6-NEXT:    v_mov_b32_e32 v4, v2
+; GFX6-NEXT:    v_mov_b32_e32 v5, v2
 ; GFX6-NEXT:    buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
@@ -648,6 +852,9 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(ptr addrspace(8) inreg %rsrc, ptr
 ; GFX8PLUS-LABEL: buffer_load_v3f32_tfe:
 ; GFX8PLUS:       ; %bb.0:
 ; GFX8PLUS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v3, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v5, v2
 ; GFX8PLUS-NEXT:    buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe
 ; GFX8PLUS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8PLUS-NEXT:    flat_store_dwordx3 v[0:1], v[2:4]
@@ -658,11 +865,24 @@ define amdgpu_cs float @buffer_load_v3f32_tfe(ptr addrspace(8) inreg %rsrc, ptr
 ; GFX11-LABEL: buffer_load_v3f32_tfe:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_mov_b32_e32 v3, v2
+; GFX11-NEXT:    v_mov_b32_e32 v4, v2
+; GFX11-NEXT:    v_mov_b32_e32 v5, v2
 ; GFX11-NEXT:    buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    global_store_b96 v[0:1], v[2:4], off
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v5
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_v3f32_tfe:
+; NOPRT:       ; %bb.0:
+; NOPRT-NEXT:    v_mov_b32_e32 v5, 0
+; NOPRT-NEXT:    buffer_load_format_xyz v[2:5], v5, s[0:3], 0 idxen tfe
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    global_store_b96 v[0:1], v[2:4], off
+; NOPRT-NEXT:    v_mov_b32_e32 v0, v5
+; NOPRT-NEXT:    ; return to shader part epilog
   %load = call { <3 x float>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v3f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
   %data = extractvalue { <3 x float>, i32 } %load, 0
   store <3 x float> %data, ptr addrspace(1) %out
@@ -675,6 +895,9 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(ptr addrspace(8) inreg %rsrc, ptr
 ; GFX6-LABEL: buffer_load_v2i32_tfe:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, v2
+; GFX6-NEXT:    v_mov_b32_e32 v4, v2
+; GFX6-NEXT:    v_mov_b32_e32 v5, v2
 ; GFX6-NEXT:    buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
@@ -689,6 +912,8 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(ptr addrspace(8) inreg %rsrc, ptr
 ; GFX8PLUS-LABEL: buffer_load_v2i32_tfe:
 ; GFX8PLUS:       ; %bb.0:
 ; GFX8PLUS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v3, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v4, v2
 ; GFX8PLUS-NEXT:    buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe
 ; GFX8PLUS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8PLUS-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
@@ -699,11 +924,23 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(ptr addrspace(8) inreg %rsrc, ptr
 ; GFX11-LABEL: buffer_load_v2i32_tfe:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_mov_b32_e32 v3, v2
+; GFX11-NEXT:    v_mov_b32_e32 v4, v2
 ; GFX11-NEXT:    buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    global_store_b64 v[0:1], v[2:3], off
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_v2i32_tfe:
+; NOPRT:       ; %bb.0:
+; NOPRT-NEXT:    v_mov_b32_e32 v4, 0
+; NOPRT-NEXT:    buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    global_store_b64 v[0:1], v[2:3], off
+; NOPRT-NEXT:    v_mov_b32_e32 v0, v4
+; NOPRT-NEXT:    ; return to shader part epilog
   %load = call { <2 x i32>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v2i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
   %data = extractvalue { <2 x i32>, i32 } %load, 0
   store <2 x i32> %data, ptr addrspace(1) %out
@@ -716,6 +953,9 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(ptr addrspace(8) inreg %rsrc, ptr
 ; GFX6-LABEL: buffer_load_v2f32_tfe:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, v2
+; GFX6-NEXT:    v_mov_b32_e32 v4, v2
+; GFX6-NEXT:    v_mov_b32_e32 v5, v2
 ; GFX6-NEXT:    buffer_load_format_xyz v[2:5], v2, s[0:3], 0 idxen tfe
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
@@ -730,6 +970,8 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(ptr addrspace(8) inreg %rsrc, ptr
 ; GFX8PLUS-LABEL: buffer_load_v2f32_tfe:
 ; GFX8PLUS:       ; %bb.0:
 ; GFX8PLUS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v3, v2
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v4, v2
 ; GFX8PLUS-NEXT:    buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe
 ; GFX8PLUS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8PLUS-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
@@ -740,11 +982,23 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(ptr addrspace(8) inreg %rsrc, ptr
 ; GFX11-LABEL: buffer_load_v2f32_tfe:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_mov_b32_e32 v3, v2
+; GFX11-NEXT:    v_mov_b32_e32 v4, v2
 ; GFX11-NEXT:    buffer_load_format_xy v[2:4], v2, s[0:3], 0 idxen tfe
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    global_store_b64 v[0:1], v[2:3], off
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_v2f32_tfe:
+; NOPRT:       ; %bb.0:
+; NOPRT-NEXT:    v_mov_b32_e32 v4, 0
+; NOPRT-NEXT:    buffer_load_format_xy v[2:4], v4, s[0:3], 0 idxen tfe
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    global_store_b64 v[0:1], v[2:3], off
+; NOPRT-NEXT:    v_mov_b32_e32 v0, v4
+; NOPRT-NEXT:    ; return to shader part epilog
   %load = call { <2 x float>, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_v2f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
   %data = extractvalue { <2 x float>, i32 } %load, 0
   store <2 x float> %data, ptr addrspace(1) %out
@@ -757,6 +1011,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad
 ; GFX6-LABEL: buffer_load_i32_tfe:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX6-NEXT:    buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
@@ -771,6 +1026,7 @@ define amdgpu_cs float @buffer_load_i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad
 ; GFX8PLUS-LABEL: buffer_load_i32_tfe:
 ; GFX8PLUS:       ; %bb.0:
 ; GFX8PLUS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX8PLUS-NEXT:    buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe
 ; GFX8PLUS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8PLUS-NEXT:    flat_store_dword v[0:1], v2
@@ -781,11 +1037,22 @@ define amdgpu_cs float @buffer_load_i32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad
 ; GFX11-LABEL: buffer_load_i32_tfe:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX11-NEXT:    buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    global_store_b32 v[0:1], v2, off
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_i32_tfe:
+; NOPRT:       ; %bb.0:
+; NOPRT-NEXT:    v_mov_b32_e32 v3, 0
+; NOPRT-NEXT:    buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    global_store_b32 v[0:1], v2, off
+; NOPRT-NEXT:    v_mov_b32_e32 v0, v3
+; NOPRT-NEXT:    ; return to shader part epilog
   %load = call { i32, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_i32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
   %data = extractvalue { i32, i32 } %load, 0
   store i32 %data, ptr addrspace(1) %out
@@ -798,6 +1065,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad
 ; GFX6-LABEL: buffer_load_f32_tfe:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX6-NEXT:    buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
@@ -812,6 +1080,7 @@ define amdgpu_cs float @buffer_load_f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad
 ; GFX8PLUS-LABEL: buffer_load_f32_tfe:
 ; GFX8PLUS:       ; %bb.0:
 ; GFX8PLUS-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8PLUS-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX8PLUS-NEXT:    buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe
 ; GFX8PLUS-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8PLUS-NEXT:    flat_store_dword v[0:1], v2
@@ -822,11 +1091,22 @@ define amdgpu_cs float @buffer_load_f32_tfe(ptr addrspace(8) inreg %rsrc, ptr ad
 ; GFX11-LABEL: buffer_load_f32_tfe:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX11-NEXT:    buffer_load_format_x v[2:3], v2, s[0:3], 0 idxen tfe
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    global_store_b32 v[0:1], v2, off
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX11-NEXT:    ; return to shader part epilog
+;
+; NOPRT-LABEL: buffer_load_f32_tfe:
+; NOPRT:       ; %bb.0:
+; NOPRT-NEXT:    v_mov_b32_e32 v3, 0
+; NOPRT-NEXT:    buffer_load_format_x v[2:3], v3, s[0:3], 0 idxen tfe
+; NOPRT-NEXT:    s_waitcnt vmcnt(0)
+; NOPRT-NEXT:    global_store_b32 v[0:1], v2, off
+; NOPRT-NEXT:    v_mov_b32_e32 v0, v3
+; NOPRT-NEXT:    ; return to shader part epilog
   %load = call { float, i32 } @llvm.amdgcn.struct.ptr.buffer.load.format.sl_f32i32s(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 0)
   %data = extractvalue { float, i32 } %load, 0
   store float %data, ptr addrspace(1) %out
-- 
GitLab


From 99c40f6ba60ea6e5dbd3f015956c7d2f6e25e54c Mon Sep 17 00:00:00 2001
From: Vyacheslav Levytskyy <vyacheslav.levytskyy@intel.com>
Date: Mon, 25 Mar 2024 10:13:42 +0100
Subject: [PATCH 093/404] [SPIR-V] Introduce a command line option to support
 compatibility with Khronos SPIRV Translator (#86101)

SPIRV-LLVM-Translator project
(https://github.com/KhronosGroup/SPIRV-LLVM-Translator) from Khronos
Group is a tool and a library for bi-directional translation between
SPIR-V and LLVM IR. In its backward translation from SPIR-V to LLVM IR
SPIRV-LLVM-Translator isn't necessarily able to cover the same SPIR-V
patterns/instructions set that SPIRV Backend produces, even if we target
the same SPIR-V version in both SPIRV-LLVM-Translator and SPIRV Backend
projects.

To improve interoperability and ability to apply SPIRV Backend output in
different products this PR introduces a notion of a mode of SPIR-V
output that is compatible with a subset of SPIR-V supported by
SPIRV-LLVM-Translator. This includes a new command line option that
doesn't influence default behavior of SPIRV Backend and one test case
that demonstrates how this command line option may be used to get a
practical benefit of producing that one of two possible and similar
output options that can be understood by SPIRV-LLVM-Translator.
---
 llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp       |  8 +++++++-
 llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll | 12 ++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp
index 38caa7c8ea0a..09a029a35a74 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp
@@ -27,6 +27,11 @@ using namespace llvm;
 #define GET_SUBTARGETINFO_CTOR
 #include "SPIRVGenSubtargetInfo.inc"
 
+static cl::opt<bool>
+    SPVTranslatorCompat("translator-compatibility-mode",
+                        cl::desc("SPIR-V Translator compatibility mode"),
+                        cl::Optional, cl::init(false));
+
 cl::list<SPIRV::Extension::Extension> Extensions(
     "spirv-extensions", cl::desc("SPIR-V extensions"), cl::ZeroOrMore,
     cl::Hidden,
@@ -157,8 +162,9 @@ bool SPIRVSubtarget::isAtLeastOpenCLVer(uint32_t VerToCompareTo) const {
 }
 
 // If the SPIR-V version is >= 1.4 we can call OpPtrEqual and OpPtrNotEqual.
+// In SPIR-V Translator compatibility mode this feature is not available.
 bool SPIRVSubtarget::canDirectlyComparePointers() const {
-  return isAtLeastVer(SPIRVVersion, 14);
+  return !SPVTranslatorCompat && isAtLeastVer(SPIRVVersion, 14);
 }
 
 void SPIRVSubtarget::initAvailableExtensions() {
diff --git a/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll b/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll
index 641e2bf0649c..31cd8bd45929 100644
--- a/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll
+++ b/llvm/test/CodeGen/SPIRV/instructions/ptrcmp.ll
@@ -1,7 +1,13 @@
 ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s --translator-compatibility-mode -o - | FileCheck %s --check-prefix=CHECK-COMPAT
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s --translator-compatibility-mode -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: OpName [[EQ:%.*]] "test_eq"
 ; CHECK-DAG: OpName [[NE:%.*]] "test_ne"
+; CHECK-COMPAT-DAG: OpName [[EQ:%.*]] "test_eq"
+; CHECK-COMPAT-DAG: OpName [[NE:%.*]] "test_ne"
 ; CHECK-DAG: OpName [[ULT:%.*]] "test_ult"
 ; CHECK-DAG: OpName [[SLT:%.*]] "test_slt"
 ; CHECK-DAG: OpName [[ULE:%.*]] "test_ule"
@@ -19,6 +25,9 @@
 ; CHECK-NEXT: [[R:%.*]] = OpPtrEqual {{%.+}} [[A]] [[B]]
 ; CHECK-NEXT: OpReturnValue [[R]]
 ; CHECK-NEXT: OpFunctionEnd
+; CHECK-COMPAT: [[EQ]] = OpFunction
+; CHECK-COMPAT-NOT: OpPtrEqual
+; CHECK-COMPAT: OpFunctionEnd
 define i1 @test_eq(i16* %a, i16* %b) {
   %r = icmp eq i16* %a, %b
   ret i1 %r
@@ -31,6 +40,9 @@ define i1 @test_eq(i16* %a, i16* %b) {
 ; CHECK-NEXT: [[R:%.*]] = OpPtrNotEqual {{%.+}} [[A]] [[B]]
 ; CHECK-NEXT: OpReturnValue [[R]]
 ; CHECK-NEXT: OpFunctionEnd
+; CHECK-COMPAT: [[NE]] = OpFunction
+; CHECK-COMPAT-NOT: OpPtrNotEqual
+; CHECK-COMPAT: OpFunctionEnd
 define i1 @test_ne(i16* %a, i16* %b) {
   %r = icmp ne i16* %a, %b
   ret i1 %r
-- 
GitLab


From 1d250d9099a9ba8b53add7eb7db6827e8fc0c8fd Mon Sep 17 00:00:00 2001
From: Vyacheslav Levytskyy <vyacheslav.levytskyy@intel.com>
Date: Mon, 25 Mar 2024 10:14:08 +0100
Subject: [PATCH 094/404] [SPIR-V] Improve type inference in SPIR-V Backend for
 opaque pointers (#86283)

This PR improves type inference in SPIR-V Backend for opaque pointers,
accounting or a case when there is a chain of function calls that allows
to deduce formal parameter types from actual arguments. The attached
test demonstrates the case.
---
 llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 123 +++++++++++-------
 llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp |   1 +
 .../Target/SPIRV/SPIRVInstructionSelector.cpp |  12 +-
 .../pointers/type-deduce-by-call-chain.ll     |  52 ++++++++
 4 files changed, 138 insertions(+), 50 deletions(-)
 create mode 100644 llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll

diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 458af9229ed7..5828db6669ff 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -92,6 +92,9 @@ class SPIRVEmitIntrinsics
   void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B);
   void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B);
   void processParamTypes(Function *F, IRBuilder<> &B);
+  Type *deduceFunParamType(Function *F, unsigned OpIdx);
+  Type *deduceFunParamType(Function *F, unsigned OpIdx,
+                           std::unordered_set<Function *> &FVisited);
 
 public:
   static char ID;
@@ -169,6 +172,10 @@ static inline void reportFatalOnTokenType(const Instruction *I) {
 static Type *deduceElementTypeHelper(Value *I,
                                      std::unordered_set<Value *> &Visited,
                                      DenseMap<Value *, Type *> &DeducedElTys) {
+  // allow to pass nullptr as an argument
+  if (!I)
+    return nullptr;
+
   // maybe already known
   auto It = DeducedElTys.find(I);
   if (It != DeducedElTys.end())
@@ -182,15 +189,20 @@ static Type *deduceElementTypeHelper(Value *I,
   // fallback value in case when we fail to deduce a type
   Type *Ty = nullptr;
   // look for known basic patterns of type inference
-  if (auto *Ref = dyn_cast<AllocaInst>(I))
+  if (auto *Ref = dyn_cast<AllocaInst>(I)) {
     Ty = Ref->getAllocatedType();
-  else if (auto *Ref = dyn_cast<GetElementPtrInst>(I))
+  } else if (auto *Ref = dyn_cast<GetElementPtrInst>(I)) {
     Ty = Ref->getResultElementType();
-  else if (auto *Ref = dyn_cast<GlobalValue>(I))
+  } else if (auto *Ref = dyn_cast<GlobalValue>(I)) {
     Ty = Ref->getValueType();
-  else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(I))
+  } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(I)) {
     Ty = deduceElementTypeHelper(Ref->getPointerOperand(), Visited,
                                  DeducedElTys);
+  } else if (auto *Ref = dyn_cast<BitCastInst>(I)) {
+    if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy();
+        isPointerTy(Src) && isPointerTy(Dest))
+      Ty = deduceElementTypeHelper(Ref->getOperand(0), Visited, DeducedElTys);
+  }
 
   // remember the found relationship
   if (Ty)
@@ -795,61 +807,80 @@ void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I,
   }
 }
 
-void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) {
-  DenseMap<unsigned, Argument *> Args;
-  unsigned i = 0;
-  for (Argument &Arg : F->args()) {
-    if (isUntypedPointerTy(Arg.getType()) &&
-        DeducedElTys.find(&Arg) == DeducedElTys.end() &&
-        !HasPointeeTypeAttr(&Arg))
-      Args[i] = &Arg;
-    i++;
-  }
-  if (Args.size() == 0)
-    return;
+Type *SPIRVEmitIntrinsics::deduceFunParamType(Function *F, unsigned OpIdx) {
+  std::unordered_set<Function *> FVisited;
+  return deduceFunParamType(F, OpIdx, FVisited);
+}
+
+Type *SPIRVEmitIntrinsics::deduceFunParamType(
+    Function *F, unsigned OpIdx, std::unordered_set<Function *> &FVisited) {
+  // maybe a cycle
+  if (FVisited.find(F) != FVisited.end())
+    return nullptr;
+  FVisited.insert(F);
 
-  // Args contains opaque pointers without element type definition
-  B.SetInsertPointPastAllocas(F);
   std::unordered_set<Value *> Visited;
+  SmallVector<std::pair<Function *, unsigned>> Lookup;
+  // search in function's call sites
   for (User *U : F->users()) {
     CallInst *CI = dyn_cast<CallInst>(U);
-    if (!CI)
+    if (!CI || OpIdx >= CI->arg_size())
       continue;
-    for (unsigned OpIdx = 0; OpIdx < CI->arg_size() && Args.size() > 0;
-         OpIdx++) {
-      auto It = Args.find(OpIdx);
-      Argument *Arg = It == Args.end() ? nullptr : It->second;
-      if (!Arg)
-        continue;
-      Value *OpArg = CI->getArgOperand(OpIdx);
-      if (!isPointerTy(OpArg->getType()))
+    Value *OpArg = CI->getArgOperand(OpIdx);
+    if (!isPointerTy(OpArg->getType()))
+      continue;
+    // maybe we already know operand's element type
+    if (auto It = DeducedElTys.find(OpArg); It != DeducedElTys.end())
+      return It->second;
+    // search in actual parameter's users
+    for (User *OpU : OpArg->users()) {
+      Instruction *Inst = dyn_cast<Instruction>(OpU);
+      if (!Inst || Inst == CI)
         continue;
-      // maybe we already know the operand's element type
-      auto DeducedIt = DeducedElTys.find(OpArg);
-      Type *ElemTy =
-          DeducedIt == DeducedElTys.end() ? nullptr : DeducedIt->second;
-      if (!ElemTy) {
-        for (User *OpU : OpArg->users()) {
-          if (Instruction *Inst = dyn_cast<Instruction>(OpU)) {
-            Visited.clear();
-            ElemTy = deduceElementTypeHelper(Inst, Visited, DeducedElTys);
-            if (ElemTy)
-              break;
-          }
-        }
+      Visited.clear();
+      if (Type *Ty = deduceElementTypeHelper(Inst, Visited, DeducedElTys))
+        return Ty;
+    }
+    // check if it's a formal parameter of the outer function
+    if (!CI->getParent() || !CI->getParent()->getParent())
+      continue;
+    Function *OuterF = CI->getParent()->getParent();
+    if (FVisited.find(OuterF) != FVisited.end())
+      continue;
+    for (unsigned i = 0; i < OuterF->arg_size(); ++i) {
+      if (OuterF->getArg(i) == OpArg) {
+        Lookup.push_back(std::make_pair(OuterF, i));
+        break;
       }
-      if (ElemTy) {
-        unsigned AddressSpace = getPointerAddressSpace(Arg->getType());
+    }
+  }
+
+  // search in function parameters
+  for (auto &Pair : Lookup) {
+    if (Type *Ty = deduceFunParamType(Pair.first, Pair.second, FVisited))
+      return Ty;
+  }
+
+  return nullptr;
+}
+
+void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) {
+  B.SetInsertPointPastAllocas(F);
+  DenseMap<Argument *, Type *> Args;
+  for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
+    Argument *Arg = F->getArg(OpIdx);
+    if (isUntypedPointerTy(Arg->getType()) &&
+        DeducedElTys.find(Arg) == DeducedElTys.end() &&
+        !HasPointeeTypeAttr(Arg)) {
+      if (Type *ElemTy = deduceFunParamType(F, OpIdx)) {
         CallInst *AssignPtrTyCI = buildIntrWithMD(
             Intrinsic::spv_assign_ptr_type, {Arg->getType()},
-            Constant::getNullValue(ElemTy), Arg, {B.getInt32(AddressSpace)}, B);
+            Constant::getNullValue(ElemTy), Arg,
+            {B.getInt32(getPointerAddressSpace(Arg->getType()))}, B);
         DeducedElTys[AssignPtrTyCI] = ElemTy;
         DeducedElTys[Arg] = ElemTy;
-        Args.erase(It);
       }
     }
-    if (Args.size() == 0)
-      break;
   }
 }
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index 42f8397a3023..ee52163a5d12 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -479,6 +479,7 @@ Register SPIRVGlobalRegistry::buildGlobalVariable(
     GVar = M->getGlobalVariable(Name);
     if (GVar == nullptr) {
       const Type *Ty = getTypeForSPIRVType(BaseType); // TODO: check type.
+      // Module takes ownership of the global var.
       GVar = new GlobalVariable(*M, const_cast<Type *>(Ty), false,
                                 GlobalValue::ExternalLinkage, nullptr,
                                 Twine(Name));
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 5bb8f6084f96..39228e2196b3 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -499,6 +499,7 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
     assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
     Register GV = I.getOperand(1).getReg();
     MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV);
+    (void)II;
     assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
             (*II).getOpcode() == TargetOpcode::COPY ||
             (*II).getOpcode() == SPIRV::OpVariable) &&
@@ -771,10 +772,13 @@ bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg,
     SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType(
         ArrTy, I, TII, SPIRV::StorageClass::UniformConstant);
     // TODO: check if we have such GV, add init, use buildGlobalVariable.
-    Type *LLVMArrTy = ArrayType::get(
-        IntegerType::get(GR.CurMF->getFunction().getContext(), 8), Num);
-    GlobalVariable *GV =
-        new GlobalVariable(LLVMArrTy, true, GlobalValue::InternalLinkage);
+    Function &CurFunction = GR.CurMF->getFunction();
+    Type *LLVMArrTy =
+        ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num);
+    // Module takes ownership of the global var.
+    GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy,
+                                            true, GlobalValue::InternalLinkage,
+                                            Constant::getNullValue(LLVMArrTy));
     Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(32));
     GR.add(GV, GR.CurMF, VarReg);
 
diff --git a/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll
new file mode 100644
index 000000000000..703f1e22a032
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/type-deduce-by-call-chain.ll
@@ -0,0 +1,52 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-SPIRV-DAG: OpName %[[ArgCum:.*]] "_arg_cum"
+; CHECK-SPIRV-DAG: OpName %[[FunTest:.*]] "test"
+; CHECK-SPIRV-DAG: OpName %[[Addr:.*]] "addr"
+; CHECK-SPIRV-DAG: OpName %[[StubObj:.*]] "stub_object"
+; CHECK-SPIRV-DAG: OpName %[[MemOrder:.*]] "mem_order"
+; CHECK-SPIRV-DAG: OpName %[[FooStub:.*]] "foo_stub"
+; CHECK-SPIRV-DAG: OpName %[[FooObj:.*]] "foo_object"
+; CHECK-SPIRV-DAG: OpName %[[FooMemOrder:.*]] "mem_order"
+; CHECK-SPIRV-DAG: OpName %[[FooFunc:.*]] "foo"
+; CHECK-SPIRV-DAG: %[[TyLong:.*]] = OpTypeInt 32 0
+; CHECK-SPIRV-DAG: %[[TyVoid:.*]] = OpTypeVoid
+; CHECK-SPIRV-DAG: %[[TyPtrLong:.*]] = OpTypePointer CrossWorkgroup %[[TyLong]]
+; CHECK-SPIRV-DAG: %[[TyFunPtrLong:.*]] = OpTypeFunction %[[TyVoid]] %[[TyPtrLong]]
+; CHECK-SPIRV-DAG: %[[TyGenPtrLong:.*]] = OpTypePointer Generic %[[TyLong]]
+; CHECK-SPIRV-DAG: %[[TyFunGenPtrLongLong:.*]] = OpTypeFunction %[[TyVoid]] %[[TyGenPtrLong]] %[[TyLong]]
+; CHECK-SPIRV-DAG: %[[Const3:.*]] = OpConstant %[[TyLong]] 3
+; CHECK-SPIRV: %[[FunTest]] = OpFunction %[[TyVoid]] None %[[TyFunPtrLong]]
+; CHECK-SPIRV: %[[ArgCum]] = OpFunctionParameter %[[TyPtrLong]]
+; CHECK-SPIRV: OpFunctionCall %[[TyVoid]] %[[FooFunc]] %[[Addr]] %[[Const3]]
+; CHECK-SPIRV: %[[FooStub]] = OpFunction %[[TyVoid]] None %[[TyFunGenPtrLongLong]]
+; CHECK-SPIRV: %[[StubObj]] = OpFunctionParameter %[[TyGenPtrLong]]
+; CHECK-SPIRV: %[[MemOrder]] = OpFunctionParameter %[[TyLong]]
+; CHECK-SPIRV: %[[FooFunc]] = OpFunction %[[TyVoid]] None %[[TyFunGenPtrLongLong]]
+; CHECK-SPIRV: %[[FooObj]] = OpFunctionParameter %[[TyGenPtrLong]]
+; CHECK-SPIRV: %[[FooMemOrder]] = OpFunctionParameter %[[TyLong]]
+; CHECK-SPIRV: OpFunctionCall %[[TyVoid]] %[[FooStub]] %[[FooObj]] %[[FooMemOrder]]
+
+define spir_kernel void @test(ptr addrspace(1) noundef align 4 %_arg_cum) {
+entry:
+  %lptr = getelementptr inbounds i32, ptr addrspace(1) %_arg_cum, i64 1
+  %addr = addrspacecast ptr addrspace(1) %lptr to ptr addrspace(4)
+  %object = bitcast ptr addrspace(4) %addr to ptr addrspace(4)
+  call spir_func void @foo(ptr addrspace(4) %object, i32 3)
+  ret void
+}
+
+define void @foo_stub(ptr addrspace(4) noundef %stub_object, i32 noundef %mem_order) {
+entry:
+  %object.addr = alloca ptr addrspace(4)
+  %object.addr.ascast = addrspacecast ptr %object.addr to ptr addrspace(4)
+  store ptr addrspace(4) %stub_object, ptr addrspace(4) %object.addr.ascast
+  ret void
+}
+
+define void @foo(ptr addrspace(4) noundef %foo_object, i32 noundef %mem_order) {
+  tail call void @foo_stub(ptr addrspace(4) noundef %foo_object, i32 noundef %mem_order)
+  ret void
+}
+
-- 
GitLab


From b0d03ccc0855f2bff39160f25fcde06aae07cace Mon Sep 17 00:00:00 2001
From: Vyacheslav Levytskyy <vyacheslav.levytskyy@intel.com>
Date: Mon, 25 Mar 2024 10:14:46 +0100
Subject: [PATCH 095/404] [SPIR-V] Fix illegal OpConstantComposite instruction
 with non-const constituents in SPIR-V Backend (#86352)

This PR fixes illegal use of OpConstantComposite with non-constant
constituents. The test attached to the PR is able now to satisfy
`spirv-val` check. Before the fix SPIR-V Backend produced for the
attached test case a pattern like
```
%a = OpVariable %_ptr_CrossWorkgroup_uint CrossWorkgroup %uint_123
%11 = OpConstantComposite %_struct_6 %a %a
```
so that `spirv-val` complained with
```
error: line 25: OpConstantComposite Constituent <id> '10[%a]' is not a constant or undef.
  %11 = OpConstantComposite %_struct_6 %a %a
```
---
 .../Target/SPIRV/SPIRVDuplicatesTracker.cpp   |  1 +
 .../lib/Target/SPIRV/SPIRVDuplicatesTracker.h |  9 ++
 llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp |  1 +
 llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h   |  8 ++
 .../Target/SPIRV/SPIRVInstructionSelector.cpp | 90 +++++++++++++++----
 llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp   |  1 +
 .../lib/Target/SPIRV/SPIRVSymbolicOperands.td |  1 +
 .../SPIRV/pointers/struct-opaque-pointers.ll  |  2 +-
 8 files changed, 96 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp
index d82fb2df4539..7c32bb1968ef 100644
--- a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.cpp
@@ -39,6 +39,7 @@ void SPIRVGeneralDuplicatesTracker::buildDepsGraph(
   prebuildReg2Entry(GT, Reg2Entry);
   prebuildReg2Entry(FT, Reg2Entry);
   prebuildReg2Entry(AT, Reg2Entry);
+  prebuildReg2Entry(MT, Reg2Entry);
   prebuildReg2Entry(ST, Reg2Entry);
 
   for (auto &Op2E : Reg2Entry) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h
index 96cc621791e9..2ec3fb35ca04 100644
--- a/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h
+++ b/llvm/lib/Target/SPIRV/SPIRVDuplicatesTracker.h
@@ -262,6 +262,7 @@ class SPIRVGeneralDuplicatesTracker {
   SPIRVDuplicatesTracker<GlobalVariable> GT;
   SPIRVDuplicatesTracker<Function> FT;
   SPIRVDuplicatesTracker<Argument> AT;
+  SPIRVDuplicatesTracker<MachineInstr> MT;
   SPIRVDuplicatesTracker<SPIRV::SpecialTypeDescriptor> ST;
 
   // NOTE: using MOs instead of regs to get rid of MF dependency to be able
@@ -306,6 +307,10 @@ public:
     AT.add(Arg, MF, R);
   }
 
+  void add(const MachineInstr *MI, const MachineFunction *MF, Register R) {
+    MT.add(MI, MF, R);
+  }
+
   void add(const SPIRV::SpecialTypeDescriptor &TD, const MachineFunction *MF,
            Register R) {
     ST.add(TD, MF, R);
@@ -337,6 +342,10 @@ public:
     return AT.find(const_cast<Argument *>(Arg), MF);
   }
 
+  Register find(const MachineInstr *MI, const MachineFunction *MF) {
+    return MT.find(const_cast<MachineInstr *>(MI), MF);
+  }
+
   Register find(const SPIRV::SpecialTypeDescriptor &TD,
                 const MachineFunction *MF) {
     return ST.find(TD, MF);
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index ee52163a5d12..db66ed4f0e0f 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -123,6 +123,7 @@ SPIRVType *SPIRVGlobalRegistry::getOpTypeVector(uint32_t NumElems,
                                                 SPIRVType *ElemType,
                                                 MachineIRBuilder &MIRBuilder) {
   auto EleOpc = ElemType->getOpcode();
+  (void)EleOpc;
   assert((EleOpc == SPIRV::OpTypeInt || EleOpc == SPIRV::OpTypeFloat ||
           EleOpc == SPIRV::OpTypeBool) &&
          "Invalid vector element type");
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
index da480b22a525..ed0f90ff89ce 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
@@ -94,6 +94,14 @@ public:
     DT.add(Arg, MF, R);
   }
 
+  void add(const MachineInstr *MI, MachineFunction *MF, Register R) {
+    DT.add(MI, MF, R);
+  }
+
+  Register find(const MachineInstr *MI, MachineFunction *MF) {
+    return DT.find(MI, MF);
+  }
+
   Register find(const Constant *C, MachineFunction *MF) {
     return DT.find(C, MF);
   }
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 39228e2196b3..505b19a4d66e 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -231,6 +231,9 @@ private:
   Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const;
   Register buildOnesVal(bool AllOnes, const SPIRVType *ResType,
                         MachineInstr &I) const;
+
+  bool wrapIntoSpecConstantOp(MachineInstr &I,
+                              SmallVector<Register> &CompositeArgs) const;
 };
 
 } // end anonymous namespace
@@ -1249,6 +1252,24 @@ static unsigned getArrayComponentCount(MachineRegisterInfo *MRI,
   return N;
 }
 
+// Return true if the type represents a constant register
+static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef) {
+  if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE &&
+      OpDef->getOperand(1).isReg()) {
+    if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg()))
+      OpDef = RefDef;
+  }
+  return OpDef->getOpcode() == TargetOpcode::G_CONSTANT ||
+         OpDef->getOpcode() == TargetOpcode::G_FCONSTANT;
+}
+
+// Return true if the virtual register represents a constant
+static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) {
+  if (SPIRVType *OpDef = MRI->getVRegDef(OpReg))
+    return isConstReg(MRI, OpDef);
+  return false;
+}
+
 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg,
                                                  const SPIRVType *ResType,
                                                  MachineInstr &I) const {
@@ -1266,16 +1287,7 @@ bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg,
 
   // check if we may construct a constant vector
   Register OpReg = I.getOperand(OpIdx).getReg();
-  bool IsConst = false;
-  if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) {
-    if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE &&
-        OpDef->getOperand(1).isReg()) {
-      if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg()))
-        OpDef = RefDef;
-    }
-    IsConst = OpDef->getOpcode() == TargetOpcode::G_CONSTANT ||
-              OpDef->getOpcode() == TargetOpcode::G_FCONSTANT;
-  }
+  bool IsConst = isConstReg(MRI, OpReg);
 
   if (!IsConst && N < 2)
     report_fatal_error(
@@ -1628,6 +1640,48 @@ bool SPIRVInstructionSelector::selectGEP(Register ResVReg,
   return Res.constrainAllUses(TII, TRI, RBI);
 }
 
+// Maybe wrap a value into OpSpecConstantOp
+bool SPIRVInstructionSelector::wrapIntoSpecConstantOp(
+    MachineInstr &I, SmallVector<Register> &CompositeArgs) const {
+  bool Result = true;
+  unsigned Lim = I.getNumExplicitOperands();
+  for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) {
+    Register OpReg = I.getOperand(i).getReg();
+    SPIRVType *OpDefine = MRI->getVRegDef(OpReg);
+    SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg);
+    if (!OpDefine || !OpType || isConstReg(MRI, OpDefine) ||
+        OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST) {
+      // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed
+      // by selectAddrSpaceCast()
+      CompositeArgs.push_back(OpReg);
+      continue;
+    }
+    MachineFunction *MF = I.getMF();
+    Register WrapReg = GR.find(OpDefine, MF);
+    if (WrapReg.isValid()) {
+      CompositeArgs.push_back(WrapReg);
+      continue;
+    }
+    // Create a new register for the wrapper
+    WrapReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
+    GR.add(OpDefine, MF, WrapReg);
+    CompositeArgs.push_back(WrapReg);
+    // Decorate the wrapper register and generate a new instruction
+    MRI->setType(WrapReg, LLT::pointer(0, 32));
+    GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF);
+    MachineBasicBlock &BB = *I.getParent();
+    Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
+                 .addDef(WrapReg)
+                 .addUse(GR.getSPIRVTypeID(OpType))
+                 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast))
+                 .addUse(OpReg)
+                 .constrainAllUses(TII, TRI, RBI);
+    if (!Result)
+      break;
+  }
+  return Result;
+}
+
 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
                                                const SPIRVType *ResType,
                                                MachineInstr &I) const {
@@ -1666,17 +1720,21 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
   case Intrinsic::spv_const_composite: {
     // If no values are attached, the composite is null constant.
     bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands();
-    unsigned Opcode =
-        IsNull ? SPIRV::OpConstantNull : SPIRV::OpConstantComposite;
+    // Select a proper instruction.
+    unsigned Opcode = SPIRV::OpConstantNull;
+    SmallVector<Register> CompositeArgs;
+    if (!IsNull) {
+      Opcode = SPIRV::OpConstantComposite;
+      if (!wrapIntoSpecConstantOp(I, CompositeArgs))
+        return false;
+    }
     auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
                    .addDef(ResVReg)
                    .addUse(GR.getSPIRVTypeID(ResType));
     // skip type MD node we already used when generated assign.type for this
     if (!IsNull) {
-      for (unsigned i = I.getNumExplicitDefs() + 1;
-           i < I.getNumExplicitOperands(); ++i) {
-        MIB.addUse(I.getOperand(i).getReg());
-      }
+      for (Register OpReg : CompositeArgs)
+        MIB.addUse(OpReg);
     }
     return MIB.constrainAllUses(TII, TRI, RBI);
   }
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index d547f91ba4a5..1f0d8d8cd43a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -543,6 +543,7 @@ static void processSwitches(MachineFunction &MF, SPIRVGlobalRegistry *GR,
       Register Dst = ICMP->getOperand(0).getReg();
       MachineOperand &PredOp = ICMP->getOperand(1);
       const auto CC = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
+      (void)CC;
       assert((CC == CmpInst::ICMP_EQ || CC == CmpInst::ICMP_ULE) &&
              MRI.hasOneUse(Dst) && MRI.hasOneDef(CompareReg));
       uint64_t Value = getIConstVal(ICMP->getOperand(3).getReg(), &MRI);
diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
index 8dbbd9049844..ff102e318469 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
+++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
@@ -1611,3 +1611,4 @@ multiclass OpcodeOperand<bits<32> value> {
 // TODO: implement other mnemonics.
 defm InBoundsPtrAccessChain : OpcodeOperand<70>;
 defm PtrCastToGeneric : OpcodeOperand<121>;
+defm Bitcast : OpcodeOperand<124>;
diff --git a/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll b/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll
index d426fc4dfd4e..ce3ab8895a59 100644
--- a/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll
+++ b/llvm/test/CodeGen/SPIRV/pointers/struct-opaque-pointers.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
-; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK: %[[TyInt8:.*]] = OpTypeInt 8 0
 ; CHECK: %[[TyInt8Ptr:.*]] = OpTypePointer {{[a-zA-Z]+}} %[[TyInt8]]
-- 
GitLab


From cbbbab349e9c412729c3969008cdcb677cc55790 Mon Sep 17 00:00:00 2001
From: Orlando Cazalet-Hyams <orlando.hyams@sony.com>
Date: Fri, 15 Mar 2024 12:41:32 +0000
Subject: [PATCH 096/404] [RemoveDIs] Enable direct-to-bitcode writing by
 default

Follow on from #83251. This patch simply enables the behaviour by default in
order to provide an easily revertible capstone.
---
 llvm/lib/IR/BasicBlock.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp
index f088c7a2cc4e..ae99267f5ba8 100644
--- a/llvm/lib/IR/BasicBlock.cpp
+++ b/llvm/lib/IR/BasicBlock.cpp
@@ -39,7 +39,7 @@ cl::opt<bool>
 bool WriteNewDbgInfoFormatToBitcode /*set default value in cl::init() below*/;
 cl::opt<bool, true> WriteNewDbgInfoFormatToBitcode2(
     "write-experimental-debuginfo-iterators-to-bitcode", cl::Hidden,
-    cl::location(WriteNewDbgInfoFormatToBitcode), cl::init(false));
+    cl::location(WriteNewDbgInfoFormatToBitcode), cl::init(true));
 
 DbgMarker *BasicBlock::createMarker(Instruction *I) {
   assert(IsNewDbgInfoFormat &&
-- 
GitLab


From 3cb024198f6a028089779c0a3cb4d8e753e87c73 Mon Sep 17 00:00:00 2001
From: Shih-Po Hung <shihpo.hung@sifive.com>
Date: Mon, 25 Mar 2024 17:17:36 +0800
Subject: [PATCH 097/404] [RISCV][CostModel] Estimate cost of
 llvm.vector.reduce.fmaximum/fminimum (#80697)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The ‘llvm.vector.reduce.fmaximum/fminimum.*’ intrinsics propagate NaNs
if any element of the vector is a NaN.
Following #79402, the patch adds the cost for NaN check (vmfne + vcpop)
---
 .../Target/RISCV/RISCVTargetTransformInfo.cpp | 45 +++++++++
 .../CostModel/RISCV/reduce-fmaximum.ll        | 91 +++++++++++++------
 .../CostModel/RISCV/reduce-fminimum.ll        | 52 +++++------
 3 files changed, 136 insertions(+), 52 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 8f46fdc2f7ca..f75b3d3caa62 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1001,6 +1001,51 @@ RISCVTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
       return getArithmeticReductionCost(Instruction::And, Ty, FMF, CostKind);
   }
 
+  if (IID == Intrinsic::maximum || IID == Intrinsic::minimum) {
+    SmallVector<unsigned, 3> Opcodes;
+    InstructionCost ExtraCost = 0;
+    switch (IID) {
+    case Intrinsic::maximum:
+      if (FMF.noNaNs()) {
+        Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S};
+      } else {
+        Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMAX_VS,
+                   RISCV::VFMV_F_S};
+        // Cost of Canonical Nan + branch
+        // lui a0, 523264
+        // fmv.w.x fa0, a0
+        Type *DstTy = Ty->getScalarType();
+        const unsigned EltTyBits = DstTy->getScalarSizeInBits();
+        Type *SrcTy = IntegerType::getIntNTy(DstTy->getContext(), EltTyBits);
+        ExtraCost = 1 +
+                    getCastInstrCost(Instruction::UIToFP, DstTy, SrcTy,
+                                     TTI::CastContextHint::None, CostKind) +
+                    getCFInstrCost(Instruction::Br, CostKind);
+      }
+      break;
+
+    case Intrinsic::minimum:
+      if (FMF.noNaNs()) {
+        Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S};
+      } else {
+        Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMIN_VS,
+                   RISCV::VFMV_F_S};
+        // Cost of Canonical Nan + branch
+        // lui a0, 523264
+        // fmv.w.x fa0, a0
+        Type *DstTy = Ty->getScalarType();
+        const unsigned EltTyBits = DL.getTypeSizeInBits(DstTy);
+        Type *SrcTy = IntegerType::getIntNTy(DstTy->getContext(), EltTyBits);
+        ExtraCost = 1 +
+                    getCastInstrCost(Instruction::UIToFP, DstTy, SrcTy,
+                                     TTI::CastContextHint::None, CostKind) +
+                    getCFInstrCost(Instruction::Br, CostKind);
+      }
+      break;
+    }
+    return ExtraCost + getRISCVInstructionCost(Opcodes, LT.second, CostKind);
+  }
+
   // IR Reduction is composed by two vmv and one rvv reduction instruction.
   InstructionCost BaseCost = 2;
 
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll
index 1618c3833a97..f91f13b2d9ec 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-fmaximum.ll
@@ -6,23 +6,37 @@
 
 define float @reduce_fmaximum_f32(float %arg) {
 ; CHECK-LABEL: 'reduce_fmaximum_f32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %2 = call fast float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %3 = call fast float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %4 = call fast float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %5 = call fast float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %6 = call fast float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %7 = call fast float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float undef
 ;
 ; SIZE-LABEL: 'reduce_fmaximum_f32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16 = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V128 = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = call fast float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %3 = call fast float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = call fast float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = call fast float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %6 = call fast float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %7 = call fast float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float undef
 ;
 %V2   = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef)
@@ -32,6 +46,13 @@ define float @reduce_fmaximum_f32(float %arg) {
 %V32   = call float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef)
 %V64   = call float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef)
 %V128   = call float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef)
+call fast float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef)
+call fast float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef)
+call fast float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef)
+call fast float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> undef)
+call fast float @llvm.vector.reduce.fmaximum.v32f32(<32 x float> undef)
+call fast float @llvm.vector.reduce.fmaximum.v64f32(<64 x float> undef)
+call fast float @llvm.vector.reduce.fmaximum.v128f32(<128 x float> undef)
 ret float undef
 }
 declare float @llvm.vector.reduce.fmaximum.v2f32(<2 x float>)
@@ -44,21 +65,33 @@ declare float @llvm.vector.reduce.fmaximum.v128f32(<128 x float>)
 
 define double @reduce_fmaximum_f64(double %arg) {
 ; CHECK-LABEL: 'reduce_fmaximum_f64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %2 = call fast double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %3 = call fast double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %4 = call fast double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %5 = call fast double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %6 = call fast double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double undef
 ;
 ; SIZE-LABEL: 'reduce_fmaximum_f64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16 = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = call fast double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = call fast double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %3 = call fast double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = call fast double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = call fast double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %6 = call fast double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret double undef
 ;
 %V2   = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef)
@@ -67,6 +100,12 @@ define double @reduce_fmaximum_f64(double %arg) {
 %V16   = call double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef)
 %V32   = call double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef)
 %V64   = call double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef)
+call fast double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef)
+call fast double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef)
+call fast double @llvm.vector.reduce.fmaximum.v8f64(<8 x double> undef)
+call fast double @llvm.vector.reduce.fmaximum.v16f64(<16 x double> undef)
+call fast double @llvm.vector.reduce.fmaximum.v32f64(<32 x double> undef)
+call fast double @llvm.vector.reduce.fmaximum.v64f64(<64 x double> undef)
 ret double undef
 }
 declare double @llvm.vector.reduce.fmaximum.v2f64(<2 x double>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll
index 35b18645b1f2..86b84025ad54 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-fminimum.ll
@@ -6,23 +6,23 @@
 
 define float @reduce_fmaximum_f32(float %arg) {
 ; CHECK-LABEL: 'reduce_fmaximum_f32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float undef
 ;
 ; SIZE-LABEL: 'reduce_fmaximum_f32'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16 = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call float @llvm.vector.reduce.fminimum.v32f32(<32 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = call float @llvm.vector.reduce.fminimum.v64f32(<64 x float> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V128 = call float @llvm.vector.reduce.fminimum.v128f32(<128 x float> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float undef
 ;
 %V2   = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef)
@@ -44,21 +44,21 @@ declare float @llvm.vector.reduce.fminimum.v128f32(<128 x float>)
 
 define double @reduce_fmaximum_f64(double %arg) {
 ; CHECK-LABEL: 'reduce_fmaximum_f64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double undef
 ;
 ; SIZE-LABEL: 'reduce_fmaximum_f64'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef)
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2 = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8 = call double @llvm.vector.reduce.fminimum.v8f64(<8 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16 = call double @llvm.vector.reduce.fminimum.v16f64(<16 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32 = call double @llvm.vector.reduce.fminimum.v32f64(<32 x double> undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64 = call double @llvm.vector.reduce.fminimum.v64f64(<64 x double> undef)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret double undef
 ;
 %V2   = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef)
-- 
GitLab


From e550022b68fc75d32e44faa59ed8f02036cd8f26 Mon Sep 17 00:00:00 2001
From: Orlando Cazalet-Hyams <orlando.hyams@sony.com>
Date: Mon, 25 Mar 2024 09:26:27 +0000
Subject: [PATCH 098/404] [RemoveDIs] Load into new debug info format by
 default in llvm-dis (#86276)

Directly load all bitcode into the new debug info format in llvm-dis.
This means that new-mode bitcode no longer round-trips back to old-mode
after parsing, and that old-mode bitcode gets auto-upgraded to new-mode
debug info (which is the current in-memory default in LLVM).
---
 llvm/tools/llvm-dis/llvm-dis.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/llvm/tools/llvm-dis/llvm-dis.cpp b/llvm/tools/llvm-dis/llvm-dis.cpp
index 8e443318dd7d..49154dc46c57 100644
--- a/llvm/tools/llvm-dis/llvm-dis.cpp
+++ b/llvm/tools/llvm-dis/llvm-dis.cpp
@@ -82,6 +82,8 @@ static cl::opt<bool> PrintThinLTOIndexOnly(
 
 extern cl::opt<bool> WriteNewDbgInfoFormat;
 
+extern cl::opt<cl::boolOrDefault> LoadBitcodeIntoNewDbgInfoFormat;
+
 namespace {
 
 static void printDebugLoc(const DebugLoc &DL, formatted_raw_ostream &OS) {
@@ -169,6 +171,10 @@ int main(int argc, char **argv) {
   cl::HideUnrelatedOptions({&DisCategory, &getColorCategory()});
   cl::ParseCommandLineOptions(argc, argv, "llvm .bc -> .ll disassembler\n");
 
+  // Load bitcode into the new debug info format by default.
+  if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET)
+    LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_TRUE;
+
   LLVMContext Context;
   Context.setDiagnosticHandler(
       std::make_unique<LLVMDisDiagnosticHandler>(argv[0]));
-- 
GitLab


From 2ef612050844355906e4b67d892a00bbb58c41d6 Mon Sep 17 00:00:00 2001
From: Orlando Cazalet-Hyams <orlando.hyams@sony.com>
Date: Mon, 25 Mar 2024 09:28:01 +0000
Subject: [PATCH 099/404] [RemoveDIs] Do not load into new debug info format
 from bitcode by default (#86268)

This is NFC right now, as the global default behaviour is also "do not
load into the new debug info format by default", but we want to change
that soon.

Additionally unconditionally convert from the new debug info format into
if we've loaded into it (e.g., if the bitcode file loaded was already in
the new format).

The latter change is needed because verify-uselistorder doesn't yet
understand DbgRecords (it doesn't know how to map them).

The former change is needed because if we load from an old debug format
bitcode file but load directly into the new format _and then convert
back to the old mode after_, the use-lists of the debug intrinsic
functions (the functions' global value uses) change.
---
 .../verify-uselistorder/verify-uselistorder.cpp     | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp
index d929ae09958a..8b299c394e4e 100644
--- a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp
+++ b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp
@@ -68,6 +68,8 @@ static cl::opt<unsigned>
                 cl::desc("Number of times to shuffle and verify use-lists"),
                 cl::init(1), cl::cat(Cat));
 
+extern cl::opt<cl::boolOrDefault> LoadBitcodeIntoNewDbgInforFormat;
+
 namespace {
 
 struct TempFile {
@@ -169,8 +171,7 @@ std::unique_ptr<Module> TempFile::readBitcode(LLVMContext &Context) const {
 
   // verify-uselistoder currently only supports old-style debug info mode.
   // FIXME: Update mapping code for RemoveDIs.
-  assert(!ModuleOr.get()->IsNewDbgInfoFormat &&
-         "Unexpectedly in new debug info mode");
+  ModuleOr.get()->setIsNewDbgInfoFormat(false);
   return std::move(ModuleOr.get());
 }
 
@@ -182,7 +183,7 @@ std::unique_ptr<Module> TempFile::readAssembly(LLVMContext &Context) const {
     Err.print("verify-uselistorder", errs());
   // verify-uselistoder currently only supports old-style debug info mode.
   // FIXME: Update mapping code for RemoveDIs.
-  assert(!M->IsNewDbgInfoFormat && "Unexpectedly in new debug info mode");
+  M->setIsNewDbgInfoFormat(false);
   return M;
 }
 
@@ -544,6 +545,10 @@ int main(int argc, char **argv) {
   cl::ParseCommandLineOptions(argc, argv,
                               "llvm tool to verify use-list order\n");
 
+  // Do not load bitcode into the new debug info format by default.
+  if (LoadBitcodeIntoNewDbgInforFormat == cl::boolOrDefault::BOU_UNSET)
+    LoadBitcodeIntoNewDbgInforFormat = cl::boolOrDefault::BOU_FALSE;
+
   LLVMContext Context;
   SMDiagnostic Err;
 
@@ -551,7 +556,7 @@ int main(int argc, char **argv) {
   std::unique_ptr<Module> M = parseIRFile(InputFilename, Err, Context);
   // verify-uselistoder currently only supports old-style debug info mode.
   // FIXME: Update mapping code for RemoveDIs.
-  assert(!M->IsNewDbgInfoFormat && "Unexpectedly in new debug info mode");
+  M->setIsNewDbgInfoFormat(false);
 
   if (!M.get()) {
     Err.print(argv[0], errs());
-- 
GitLab


From 8263a883342d9925a4a1fd9752efc8deda5840fc Mon Sep 17 00:00:00 2001
From: Orlando Cazalet-Hyams <orlando.hyams@sony.com>
Date: Mon, 25 Mar 2024 09:29:19 +0000
Subject: [PATCH 100/404] [RemoveDIs] Load into new debug info format by
 default in llvm-link (#86274)

Directly load all bitcode into the new debug info format in llvm-link.
This means that new-mode bitcode no longer round-trips back to old-mode
after parsing, and that old-mode bitcode gets auto-upgraded to new-mode
debug info (which is the current in-memory default in LLVM).
---
 llvm/tools/llvm-link/llvm-link.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/llvm/tools/llvm-link/llvm-link.cpp b/llvm/tools/llvm-link/llvm-link.cpp
index 9e7f2c3ebac4..9049cb5e8580 100644
--- a/llvm/tools/llvm-link/llvm-link.cpp
+++ b/llvm/tools/llvm-link/llvm-link.cpp
@@ -136,6 +136,8 @@ static cl::opt<bool> TryUseNewDbgInfoFormat(
 
 extern cl::opt<bool> UseNewDbgInfoFormat;
 
+extern cl::opt<cl::boolOrDefault> LoadBitcodeIntoNewDbgInfoFormat;
+
 static ExitOnError ExitOnErr;
 
 // Read the specified bitcode file in and return it. This routine searches the
@@ -480,6 +482,10 @@ int main(int argc, char **argv) {
   cl::HideUnrelatedOptions({&LinkCategory, &getColorCategory()});
   cl::ParseCommandLineOptions(argc, argv, "llvm linker\n");
 
+  // Load bitcode into the new debug info format by default.
+  if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET)
+    LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_TRUE;
+
   // RemoveDIs debug-info transition: tests may request that we /try/ to use the
   // new debug-info format.
   if (TryUseNewDbgInfoFormat) {
-- 
GitLab


From 772e316457ef94759804d9f4da0af70d8d2ca4d4 Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas@arm.com>
Date: Mon, 25 Mar 2024 09:43:41 +0000
Subject: [PATCH 101/404] [FMV] Allow multi versioning without default
 declaration. (#85454)

This was a limitation which has now been lifted. Please read the
thread below for more details:

https://github.com/llvm/llvm-project/pull/84405#discussion_r1525583647

Basically it allows to separate versioned implementations across
different TUs without having to share private header files which
contain the default declaration.

The ACLE spec has been updated accordingly to make this explicit:
"Each version declaration should be visible at the translation
 unit in which the corresponding function version resides."

https://github.com/ARM-software/acle/pull/310

If a resolver is required (because there is a caller in the TU),
then a default declaration is implicitly generated.
---
 clang/lib/CodeGen/CodeGenModule.cpp           | 131 ++++---
 clang/lib/Sema/SemaDecl.cpp                   |   6 +-
 clang/lib/Sema/SemaOverload.cpp               |  44 ++-
 clang/test/CodeGen/attr-target-version.c      | 368 ++++++++++++------
 clang/test/CodeGenCXX/attr-target-version.cpp |  62 +--
 clang/test/Sema/aarch64-sme-func-attrs.c      |   8 +-
 clang/test/Sema/attr-target-version.c         |   8 +-
 clang/test/SemaCXX/attr-target-version.cpp    |  11 +-
 8 files changed, 403 insertions(+), 235 deletions(-)

diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index cb153066b28d..ac81df8cf7ad 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -3711,7 +3711,8 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) {
 
     // Forward declarations are emitted lazily on first use.
     if (!FD->doesThisDeclarationHaveABody()) {
-      if (!FD->doesDeclarationForceExternallyVisibleDefinition())
+      if (!FD->doesDeclarationForceExternallyVisibleDefinition() &&
+          !FD->isTargetVersionMultiVersion())
         return;
 
       StringRef MangledName = getMangledName(GD);
@@ -4092,6 +4093,23 @@ llvm::GlobalValue::LinkageTypes getMultiversionLinkage(CodeGenModule &CGM,
   return llvm::GlobalValue::WeakODRLinkage;
 }
 
+static FunctionDecl *createDefaultTargetVersionFrom(const FunctionDecl *FD) {
+  DeclContext *DeclCtx = FD->getASTContext().getTranslationUnitDecl();
+  TypeSourceInfo *TInfo = FD->getTypeSourceInfo();
+  StorageClass SC = FD->getStorageClass();
+  DeclarationName Name = FD->getNameInfo().getName();
+
+  FunctionDecl *NewDecl =
+      FunctionDecl::Create(FD->getASTContext(), DeclCtx, FD->getBeginLoc(),
+                           FD->getEndLoc(), Name, TInfo->getType(), TInfo, SC);
+
+  NewDecl->setIsMultiVersion();
+  NewDecl->addAttr(TargetVersionAttr::CreateImplicit(
+      NewDecl->getASTContext(), "default", NewDecl->getSourceRange()));
+
+  return NewDecl;
+}
+
 void CodeGenModule::emitMultiVersionFunctions() {
   std::vector<GlobalDecl> MVFuncsToEmit;
   MultiVersionFuncs.swap(MVFuncsToEmit);
@@ -4099,70 +4117,54 @@ void CodeGenModule::emitMultiVersionFunctions() {
     const auto *FD = cast<FunctionDecl>(GD.getDecl());
     assert(FD && "Expected a FunctionDecl");
 
-    bool EmitResolver = !FD->isTargetVersionMultiVersion();
+    auto createFunction = [&](const FunctionDecl *Decl, unsigned MVIdx = 0) {
+      GlobalDecl CurGD{Decl->isDefined() ? Decl->getDefinition() : Decl, MVIdx};
+      StringRef MangledName = getMangledName(CurGD);
+      llvm::Constant *Func = GetGlobalValue(MangledName);
+      if (!Func) {
+        if (Decl->isDefined()) {
+          EmitGlobalFunctionDefinition(CurGD, nullptr);
+          Func = GetGlobalValue(MangledName);
+        } else {
+          const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(CurGD);
+          llvm::FunctionType *Ty = getTypes().GetFunctionType(FI);
+          Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false,
+                                   /*DontDefer=*/false, ForDefinition);
+        }
+        assert(Func && "This should have just been created");
+      }
+      return cast<llvm::Function>(Func);
+    };
+
+    bool HasDefaultDecl = !FD->isTargetVersionMultiVersion();
+    bool ShouldEmitResolver = !FD->isTargetVersionMultiVersion();
     SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
     if (FD->isTargetMultiVersion()) {
       getContext().forEachMultiversionedFunctionVersion(
-          FD, [this, &GD, &Options, &EmitResolver](const FunctionDecl *CurFD) {
-            GlobalDecl CurGD{
-                (CurFD->isDefined() ? CurFD->getDefinition() : CurFD)};
-            StringRef MangledName = getMangledName(CurGD);
-            llvm::Constant *Func = GetGlobalValue(MangledName);
-            if (!Func) {
-              if (CurFD->isDefined()) {
-                EmitGlobalFunctionDefinition(CurGD, nullptr);
-                Func = GetGlobalValue(MangledName);
-              } else {
-                const CGFunctionInfo &FI =
-                    getTypes().arrangeGlobalDeclaration(GD);
-                llvm::FunctionType *Ty = getTypes().GetFunctionType(FI);
-                Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false,
-                                         /*DontDefer=*/false, ForDefinition);
-              }
-              assert(Func && "This should have just been created");
-            }
-            if (CurFD->getMultiVersionKind() == MultiVersionKind::Target) {
-              const auto *TA = CurFD->getAttr<TargetAttr>();
-              llvm::SmallVector<StringRef, 8> Feats;
+          FD, [&](const FunctionDecl *CurFD) {
+            llvm::SmallVector<StringRef, 8> Feats;
+            llvm::Function *Func = createFunction(CurFD);
+
+            if (const auto *TA = CurFD->getAttr<TargetAttr>()) {
               TA->getAddedFeatures(Feats);
-              Options.emplace_back(cast<llvm::Function>(Func),
-                                   TA->getArchitecture(), Feats);
-            } else {
-              const auto *TVA = CurFD->getAttr<TargetVersionAttr>();
-              if (CurFD->isUsed() || (TVA->isDefaultVersion() &&
-                                      CurFD->doesThisDeclarationHaveABody()))
-                EmitResolver = true;
-              llvm::SmallVector<StringRef, 8> Feats;
+              Options.emplace_back(Func, TA->getArchitecture(), Feats);
+            } else if (const auto *TVA = CurFD->getAttr<TargetVersionAttr>()) {
+              bool HasDefaultDef = TVA->isDefaultVersion() &&
+                                   CurFD->doesThisDeclarationHaveABody();
+              HasDefaultDecl |= TVA->isDefaultVersion();
+              ShouldEmitResolver |= (CurFD->isUsed() || HasDefaultDef);
               TVA->getFeatures(Feats);
-              Options.emplace_back(cast<llvm::Function>(Func),
-                                   /*Architecture*/ "", Feats);
-            }
+              Options.emplace_back(Func, /*Architecture*/ "", Feats);
+            } else
+              llvm_unreachable("unexpected MultiVersionKind");
           });
-    } else if (FD->isTargetClonesMultiVersion()) {
-      const auto *TC = FD->getAttr<TargetClonesAttr>();
-      for (unsigned VersionIndex = 0; VersionIndex < TC->featuresStrs_size();
-           ++VersionIndex) {
-        if (!TC->isFirstOfVersion(VersionIndex))
+    } else if (const auto *TC = FD->getAttr<TargetClonesAttr>()) {
+      for (unsigned I = 0; I < TC->featuresStrs_size(); ++I) {
+        if (!TC->isFirstOfVersion(I))
           continue;
-        GlobalDecl CurGD{(FD->isDefined() ? FD->getDefinition() : FD),
-                         VersionIndex};
-        StringRef Version = TC->getFeatureStr(VersionIndex);
-        StringRef MangledName = getMangledName(CurGD);
-        llvm::Constant *Func = GetGlobalValue(MangledName);
-        if (!Func) {
-          if (FD->isDefined()) {
-            EmitGlobalFunctionDefinition(CurGD, nullptr);
-            Func = GetGlobalValue(MangledName);
-          } else {
-            const CGFunctionInfo &FI =
-                getTypes().arrangeGlobalDeclaration(CurGD);
-            llvm::FunctionType *Ty = getTypes().GetFunctionType(FI);
-            Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false,
-                                     /*DontDefer=*/false, ForDefinition);
-          }
-          assert(Func && "This should have just been created");
-        }
 
+        llvm::Function *Func = createFunction(FD, I);
+        StringRef Version = TC->getFeatureStr(I);
         StringRef Architecture;
         llvm::SmallVector<StringRef, 1> Feature;
 
@@ -4180,16 +4182,23 @@ void CodeGenModule::emitMultiVersionFunctions() {
             Feature.push_back(Version);
         }
 
-        Options.emplace_back(cast<llvm::Function>(Func), Architecture, Feature);
+        Options.emplace_back(Func, Architecture, Feature);
       }
     } else {
       assert(0 && "Expected a target or target_clones multiversion function");
       continue;
     }
 
-    if (!EmitResolver)
+    if (!ShouldEmitResolver)
       continue;
 
+    if (!HasDefaultDecl) {
+      FunctionDecl *NewFD = createDefaultTargetVersionFrom(FD);
+      llvm::Function *Func = createFunction(NewFD);
+      llvm::SmallVector<StringRef, 1> Feats;
+      Options.emplace_back(Func, /*Architecture*/ "", Feats);
+    }
+
     llvm::Constant *ResolverConstant = GetOrCreateMultiVersionResolver(GD);
     if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(ResolverConstant)) {
       ResolverConstant = IFunc->getResolver();
@@ -4480,7 +4489,9 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction(
 
     if (FD->isMultiVersion()) {
       UpdateMultiVersionNames(GD, FD, MangledName);
-      if (!IsForDefinition)
+      if (FD->isTargetVersionMultiVersion() && !FD->isUsed())
+        AddDeferredMultiVersionResolverToEmit(GD);
+      else if (!IsForDefinition)
         return GetOrCreateMultiVersionResolver(GD);
     }
   }
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index aa754d47a0c4..73ea155053d7 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -11441,9 +11441,9 @@ static bool CheckMultiVersionFirstFunction(Sema &S, FunctionDecl *FD) {
          "Function lacks multiversion attribute");
   const auto *TA = FD->getAttr<TargetAttr>();
   const auto *TVA = FD->getAttr<TargetVersionAttr>();
-  // Target and target_version only causes MV if it is default, otherwise this
-  // is a normal function.
-  if ((TA && !TA->isDefaultVersion()) || (TVA && !TVA->isDefaultVersion()))
+  // The target attribute only causes MV if this declaration is the default,
+  // otherwise it is treated as a normal function.
+  if (TA && !TA->isDefaultVersion())
     return false;
 
   if ((TA || TVA) && CheckMultiVersionValue(S, FD)) {
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index f6bd85bdc646..51450e486eae 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -6865,6 +6865,32 @@ static bool IsAcceptableNonMemberOperatorCandidate(ASTContext &Context,
   return false;
 }
 
+static bool isNonViableMultiVersionOverload(FunctionDecl *FD) {
+  if (FD->isTargetMultiVersionDefault())
+    return false;
+
+  if (!FD->getASTContext().getTargetInfo().getTriple().isAArch64())
+    return FD->isTargetMultiVersion();
+
+  if (!FD->isMultiVersion())
+    return false;
+
+  // Among multiple target versions consider either the default,
+  // or the first non-default in the absence of default version.
+  unsigned SeenAt = 0;
+  unsigned I = 0;
+  bool HasDefault = false;
+  FD->getASTContext().forEachMultiversionedFunctionVersion(
+      FD, [&](const FunctionDecl *CurFD) {
+        if (FD == CurFD)
+          SeenAt = I;
+        else if (CurFD->isTargetMultiVersionDefault())
+          HasDefault = true;
+        ++I;
+      });
+  return HasDefault || SeenAt != 0;
+}
+
 /// AddOverloadCandidate - Adds the given function to the set of
 /// candidate functions, using the given function call arguments.  If
 /// @p SuppressUserConversions, then don't allow user-defined
@@ -6970,11 +6996,7 @@ void Sema::AddOverloadCandidate(
     }
   }
 
-  if (Function->isMultiVersion() &&
-      ((Function->hasAttr<TargetAttr>() &&
-        !Function->getAttr<TargetAttr>()->isDefaultVersion()) ||
-       (Function->hasAttr<TargetVersionAttr>() &&
-        !Function->getAttr<TargetVersionAttr>()->isDefaultVersion()))) {
+  if (isNonViableMultiVersionOverload(Function)) {
     Candidate.Viable = false;
     Candidate.FailureKind = ovl_non_default_multiversion_function;
     return;
@@ -7637,11 +7659,7 @@ Sema::AddMethodCandidate(CXXMethodDecl *Method, DeclAccessPair FoundDecl,
     return;
   }
 
-  if (Method->isMultiVersion() &&
-      ((Method->hasAttr<TargetAttr>() &&
-        !Method->getAttr<TargetAttr>()->isDefaultVersion()) ||
-       (Method->hasAttr<TargetVersionAttr>() &&
-        !Method->getAttr<TargetVersionAttr>()->isDefaultVersion()))) {
+  if (isNonViableMultiVersionOverload(Method)) {
     Candidate.Viable = false;
     Candidate.FailureKind = ovl_non_default_multiversion_function;
   }
@@ -8127,11 +8145,7 @@ void Sema::AddConversionCandidate(
     return;
   }
 
-  if (Conversion->isMultiVersion() &&
-      ((Conversion->hasAttr<TargetAttr>() &&
-        !Conversion->getAttr<TargetAttr>()->isDefaultVersion()) ||
-       (Conversion->hasAttr<TargetVersionAttr>() &&
-        !Conversion->getAttr<TargetVersionAttr>()->isDefaultVersion()))) {
+  if (isNonViableMultiVersionOverload(Conversion)) {
     Candidate.Viable = false;
     Candidate.FailureKind = ovl_non_default_multiversion_function;
   }
diff --git a/clang/test/CodeGen/attr-target-version.c b/clang/test/CodeGen/attr-target-version.c
index 25129605e76c..dd4cbbf5a898 100644
--- a/clang/test/CodeGen/attr-target-version.c
+++ b/clang/test/CodeGen/attr-target-version.c
@@ -109,21 +109,47 @@ int unused_with_implicit_default_def(void) { return 1; }
 int unused_with_implicit_forward_default_def(void) { return 0; }
 __attribute__((target_version("lse"))) int unused_with_implicit_forward_default_def(void) { return 1; }
 
-// This should generate a normal function.
+// This should generate a target version despite the default not being declared.
 __attribute__((target_version("rdm"))) int unused_without_default(void) { return 0; }
 
+// These shouldn't generate anything.
+int unused_version_declarations(void);
+__attribute__((target_version("jscvt"))) int unused_version_declarations(void);
+__attribute__((target_version("rdma"))) int unused_version_declarations(void);
+
+// These should generate the default (mangled) version and the resolver.
+int default_def_with_version_decls(void) { return 0; }
+__attribute__((target_version("jscvt"))) int default_def_with_version_decls(void);
+__attribute__((target_version("rdma"))) int default_def_with_version_decls(void);
+
+// The following is guarded because in NOFMV we get errors for calling undeclared functions.
+#ifdef __HAVE_FUNCTION_MULTI_VERSIONING
+// This should generate a default declaration, two target versions and the resolver.
+__attribute__((target_version("jscvt"))) int used_def_without_default_decl(void) { return 1; }
+__attribute__((target_version("rdma"))) int used_def_without_default_decl(void) { return 2; }
+
+// This should generate a default declaration and the resolver.
+__attribute__((target_version("jscvt"))) int used_decl_without_default_decl(void);
+__attribute__((target_version("rdma"))) int used_decl_without_default_decl(void);
+
+int caller(void) { return used_def_without_default_decl() + used_decl_without_default_decl(); }
+#endif
+
 //.
 // CHECK: @__aarch64_cpu_features = external dso_local global { i64 }
 // CHECK: @fmv.ifunc = weak_odr alias i32 (), ptr @fmv
 // CHECK: @fmv_one.ifunc = weak_odr alias i32 (), ptr @fmv_one
 // CHECK: @fmv_two.ifunc = weak_odr alias i32 (), ptr @fmv_two
 // CHECK: @fmv_e.ifunc = weak_odr alias i32 (), ptr @fmv_e
+// CHECK: @fmv_d.ifunc = internal alias i32 (), ptr @fmv_d
 // CHECK: @fmv_c.ifunc = weak_odr alias void (), ptr @fmv_c
 // CHECK: @fmv_inline.ifunc = weak_odr alias i32 (), ptr @fmv_inline
-// CHECK: @fmv_d.ifunc = internal alias i32 (), ptr @fmv_d
 // CHECK: @unused_with_default_def.ifunc = weak_odr alias i32 (), ptr @unused_with_default_def
 // CHECK: @unused_with_implicit_default_def.ifunc = weak_odr alias i32 (), ptr @unused_with_implicit_default_def
 // CHECK: @unused_with_implicit_forward_default_def.ifunc = weak_odr alias i32 (), ptr @unused_with_implicit_forward_default_def
+// CHECK: @default_def_with_version_decls.ifunc = weak_odr alias i32 (), ptr @default_def_with_version_decls
+// CHECK: @used_def_without_default_decl.ifunc = weak_odr alias i32 (), ptr @used_def_without_default_decl
+// CHECK: @used_decl_without_default_decl.ifunc = weak_odr alias i32 (), ptr @used_decl_without_default_decl
 // CHECK: @fmv = weak_odr ifunc i32 (), ptr @fmv.resolver
 // CHECK: @fmv_one = weak_odr ifunc i32 (), ptr @fmv_one.resolver
 // CHECK: @fmv_two = weak_odr ifunc i32 (), ptr @fmv_two.resolver
@@ -131,97 +157,121 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return
 // CHECK: @fmv_e = weak_odr ifunc i32 (), ptr @fmv_e.resolver
 // CHECK: @fmv_d = internal ifunc i32 (), ptr @fmv_d.resolver
 // CHECK: @fmv_c = weak_odr ifunc void (), ptr @fmv_c.resolver
+// CHECK: @used_def_without_default_decl = weak_odr ifunc i32 (), ptr @used_def_without_default_decl.resolver
+// CHECK: @used_decl_without_default_decl = weak_odr ifunc i32 (), ptr @used_decl_without_default_decl.resolver
 // CHECK: @unused_with_default_def = weak_odr ifunc i32 (), ptr @unused_with_default_def.resolver
 // CHECK: @unused_with_implicit_default_def = weak_odr ifunc i32 (), ptr @unused_with_implicit_default_def.resolver
 // CHECK: @unused_with_implicit_forward_default_def = weak_odr ifunc i32 (), ptr @unused_with_implicit_forward_default_def.resolver
+// CHECK: @default_def_with_version_decls = weak_odr ifunc i32 (), ptr @default_def_with_version_decls.resolver
 //.
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv._Mflagm2Msme-i16i64
+// CHECK-LABEL: define {{[^@]+}}@fmv._MflagmMfp16fmlMrng
 // CHECK-SAME: () #[[ATTR0:[0-9]+]] {
 // CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv._Mflagm2Msme-i16i64
+// CHECK-SAME: () #[[ATTR1:[0-9]+]] {
+// CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 2
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv._MlseMsha2
-// CHECK-SAME: () #[[ATTR1:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR2:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 3
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv._MdotprodMls64_accdata
-// CHECK-SAME: () #[[ATTR2:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR3:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 4
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv._Mfp16fmlMmemtag
-// CHECK-SAME: () #[[ATTR3:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR4:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 5
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv._MaesMfp
-// CHECK-SAME: () #[[ATTR4:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR5:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 6
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv._McrcMls64_v
-// CHECK-SAME: () #[[ATTR5:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR6:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 7
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv._Mbti
-// CHECK-SAME: () #[[ATTR6:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR7:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 8
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv._Msme2
-// CHECK-SAME: () #[[ATTR7:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR8:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 9
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_one._Mls64Msimd
+// CHECK-SAME: () #[[ATTR5]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_one._Mdpb
-// CHECK-SAME: () #[[ATTR8:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR10:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 2
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp
+// CHECK-SAME: () #[[ATTR5]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_two._Msimd
-// CHECK-SAME: () #[[ATTR4]] {
+// CHECK-SAME: () #[[ATTR5]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 2
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_two._Mdgh
-// CHECK-SAME: () #[[ATTR9:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR11:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 3
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp16Msimd
-// CHECK-SAME: () #[[ATTR10:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR12:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 4
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@foo
-// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-SAME: () #[[ATTR11]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[CALL:%.*]] = call i32 @fmv()
 // CHECK-NEXT:    [[CALL1:%.*]] = call i32 @fmv_one()
@@ -371,35 +421,49 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_e.default
-// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-SAME: () #[[ATTR11]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 20
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_d._Msb
+// CHECK-SAME: () #[[ATTR13:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 0
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@fmv_d.default
+// CHECK-SAME: () #[[ATTR11]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_default
-// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-SAME: () #[[ATTR11]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 111
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_c._Mssbs
-// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-SAME: () #[[ATTR11]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret void
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_c.default
-// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-SAME: () #[[ATTR11]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret void
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@goo
-// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-SAME: () #[[ATTR11]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[CALL:%.*]] = call i32 @fmv_inline()
 // CHECK-NEXT:    [[CALL1:%.*]] = call i32 @fmv_e()
@@ -587,7 +651,7 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@recur
-// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-SAME: () #[[ATTR11]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    call void @reca()
 // CHECK-NEXT:    ret void
@@ -595,7 +659,7 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@main
-// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-SAME: () #[[ATTR11]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    store i32 0, ptr [[RETVAL]], align 4
@@ -606,7 +670,7 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@hoo
-// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-SAME: () #[[ATTR11]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[FP1:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    [[FP2:%.*]] = alloca ptr, align 8
@@ -623,228 +687,268 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@unused_with_forward_default_decl._Mmops
-// CHECK-SAME: () #[[ATTR12:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR14:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 0
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_extern_forward_default_decl._Mdotprod
-// CHECK-SAME: () #[[ATTR13:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR15:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 0
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.default
-// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-LABEL: define {{[^@]+}}@unused_with_default_decl._Maes
+// CHECK-SAME: () #[[ATTR5]] {
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
+// CHECK-NEXT:    ret i32 0
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def.default
-// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def._Msve
+// CHECK-SAME: () #[[ATTR16:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 0
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.default
+// CHECK-SAME: () #[[ATTR11]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 1
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def.default
-// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def._Mfp16
+// CHECK-SAME: () #[[ATTR12]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 0
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def._Mlse
-// CHECK-SAME: () #[[ATTR14:[0-9]+]] {
+// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def.default
+// CHECK-SAME: () #[[ATTR11]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 1
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv._MflagmMfp16fmlMrng
-// CHECK-SAME: () #[[ATTR15:[0-9]+]] {
+// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def.default
+// CHECK-SAME: () #[[ATTR11]] {
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
+// CHECK-NEXT:    ret i32 0
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_one._Mls64Msimd
-// CHECK-SAME: () #[[ATTR4]] {
+// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def._Mlse
+// CHECK-SAME: () #[[ATTR17:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 1
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp
-// CHECK-SAME: () #[[ATTR4]] {
+// CHECK-LABEL: define {{[^@]+}}@unused_without_default._Mrdm
+// CHECK-SAME: () #[[ATTR18:[0-9]+]] {
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
+// CHECK-NEXT:    ret i32 0
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@unused_with_default_decl._Maes
-// CHECK-SAME: () #[[ATTR4]] {
+// CHECK-LABEL: define {{[^@]+}}@default_def_with_version_decls.default
+// CHECK-SAME: () #[[ATTR11]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 0
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def._Msve
-// CHECK-SAME: () #[[ATTR16:[0-9]+]] {
+// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl._Mjscvt
+// CHECK-SAME: () #[[ATTR21:[0-9]+]] {
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 0
+// CHECK-NEXT:    ret i32 1
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def._Mfp16
-// CHECK-SAME: () #[[ATTR10]] {
+// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl._Mrdm
+// CHECK-SAME: () #[[ATTR18]] {
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 0
+// CHECK-NEXT:    ret i32 2
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@unused_without_default
-// CHECK-SAME: () #[[ATTR17:[0-9]+]] {
+// CHECK-LABEL: define {{[^@]+}}@caller
+// CHECK-SAME: () #[[ATTR11]] {
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 0
+// CHECK-NEXT:    [[CALL:%.*]] = call i32 @used_def_without_default_decl()
+// CHECK-NEXT:    [[CALL1:%.*]] = call i32 @used_decl_without_default_decl()
+// CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[CALL]], [[CALL1]]
+// CHECK-NEXT:    ret i32 [[ADD]]
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl.resolver() comdat {
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1048576
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK:       resolver_return:
+// CHECK-NEXT:    ret ptr @used_def_without_default_decl._Mjscvt
+// CHECK:       resolver_else:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 64
+// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 64
+// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK:       resolver_return1:
+// CHECK-NEXT:    ret ptr @used_def_without_default_decl._Mrdm
+// CHECK:       resolver_else2:
+// CHECK-NEXT:    ret ptr @used_def_without_default_decl.default
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@used_decl_without_default_decl.resolver() comdat {
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1048576
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK:       resolver_return:
+// CHECK-NEXT:    ret ptr @used_decl_without_default_decl._Mjscvt
+// CHECK:       resolver_else:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 64
+// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 64
+// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK:       resolver_return1:
+// CHECK-NEXT:    ret ptr @used_decl_without_default_decl._Mrdm
+// CHECK:       resolver_else2:
+// CHECK-NEXT:    ret ptr @used_decl_without_default_decl.default
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf64mmMpmullMsha1
-// CHECK-SAME: () #[[ATTR18:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR22:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 1
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfcmaMfp16MrdmMsme
-// CHECK-SAME: () #[[ATTR19:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR23:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 2
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf32mmMi8mmMsha3
-// CHECK-SAME: () #[[ATTR20:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR24:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 12
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MditMsve-ebf16
-// CHECK-SAME: () #[[ATTR21:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR25:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 8
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MdpbMrcpc2
-// CHECK-SAME: () #[[ATTR22:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR26:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 6
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mdpb2Mjscvt
-// CHECK-SAME: () #[[ATTR23:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR27:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 7
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfrinttsMrcpc
-// CHECK-SAME: () #[[ATTR24:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR28:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 3
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MsveMsve-bf16
-// CHECK-SAME: () #[[ATTR25:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR29:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 4
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2-aesMsve2-sha3
-// CHECK-SAME: () #[[ATTR26:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR30:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 5
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2Msve2-bitpermMsve2-pmull128
-// CHECK-SAME: () #[[ATTR27:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR31:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 9
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag2Msve2-sm4
-// CHECK-SAME: () #[[ATTR28:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR32:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 10
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag3MmopsMrcpc3
-// CHECK-SAME: () #[[ATTR29:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR33:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 11
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MaesMdotprod
-// CHECK-SAME: () #[[ATTR13]] {
+// CHECK-SAME: () #[[ATTR15]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 13
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mfp16fmlMsimd
-// CHECK-SAME: () #[[ATTR3]] {
+// CHECK-SAME: () #[[ATTR4]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 14
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfpMsm4
-// CHECK-SAME: () #[[ATTR30:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR34:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 15
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MlseMrdm
-// CHECK-SAME: () #[[ATTR31:[0-9]+]] {
+// CHECK-SAME: () #[[ATTR35:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 16
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: define {{[^@]+}}@fmv_inline.default
-// CHECK-SAME: () #[[ATTR9]] {
+// CHECK-SAME: () #[[ATTR11]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 3
 //
 //
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_d._Msb
-// CHECK-SAME: () #[[ATTR32:[0-9]+]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 0
-//
-//
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_d.default
-// CHECK-SAME: () #[[ATTR9]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
 // CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.resolver() comdat {
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
@@ -887,6 +991,28 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return
 // CHECK-NEXT:    ret ptr @unused_with_implicit_forward_default_def.default
 //
 //
+// CHECK-LABEL: define {{[^@]+}}@default_def_with_version_decls.resolver() comdat {
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1048576
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK:       resolver_return:
+// CHECK-NEXT:    ret ptr @default_def_with_version_decls._Mjscvt
+// CHECK:       resolver_else:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 64
+// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 64
+// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK:       resolver_return1:
+// CHECK-NEXT:    ret ptr @default_def_with_version_decls._Mrdm
+// CHECK:       resolver_else2:
+// CHECK-NEXT:    ret ptr @default_def_with_version_decls.default
+//
+//
 // CHECK-NOFMV: Function Attrs: noinline nounwind optnone
 // CHECK-NOFMV-LABEL: define {{[^@]+}}@foo
 // CHECK-NOFMV-SAME: () #[[ATTR0:[0-9]+]] {
@@ -995,40 +1121,50 @@ __attribute__((target_version("rdm"))) int unused_without_default(void) { return
 // CHECK-NOFMV-NEXT:  entry:
 // CHECK-NOFMV-NEXT:    ret i32 0
 //
+//
+// CHECK-NOFMV: Function Attrs: noinline nounwind optnone
+// CHECK-NOFMV-LABEL: define {{[^@]+}}@default_def_with_version_decls
+// CHECK-NOFMV-SAME: () #[[ATTR0]] {
+// CHECK-NOFMV-NEXT:  entry:
+// CHECK-NOFMV-NEXT:    ret i32 0
+//
 //.
-// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+altnzcv,+bf16,+flagm,+sme,+sme-i16i64,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+sha2,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+ls64,+neon,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp16fml,+fullfp16,+neon,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme2,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR9]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR11:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR12]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+neon,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+flagm,+fp16fml,+fullfp16,+neon,+rand,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+flagm,+fp16fml,+fullfp16,+neon,+rand,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+altnzcv,+bf16,+flagm,+sme,+sme-i16i64,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+sha2,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+ls64,+neon,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp16fml,+fullfp16,+neon,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme2,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR9:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR11]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR12]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+neon,-fp-armv8,-v9.5a" }
 // CHECK: attributes #[[ATTR16]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR18]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR19]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fullfp16,+neon,+rdm,+sme,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR20]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+jsconv,+neon,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+mte,+neon,+sve,+sve2,+sve2-sm4,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+mte,+rcpc,+rcpc3,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+sm4,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+rdm,-fp-armv8,-v9.5a" }
-// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR18]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR19:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR20:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fullfp16,+neon,+rdm,+sme,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+jsconv,+neon,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+mte,+neon,+sve,+sve2,+sve2-sm4,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR33]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+mte,+rcpc,+rcpc3,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR34]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+sm4,-fp-armv8,-v9.5a" }
+// CHECK: attributes #[[ATTR35]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+rdm,-fp-armv8,-v9.5a" }
 //.
 // CHECK-NOFMV: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" }
 // CHECK-NOFMV: attributes #[[ATTR1:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" }
diff --git a/clang/test/CodeGenCXX/attr-target-version.cpp b/clang/test/CodeGenCXX/attr-target-version.cpp
index e06121d1a719..8b7273fe3bb5 100644
--- a/clang/test/CodeGenCXX/attr-target-version.cpp
+++ b/clang/test/CodeGenCXX/attr-target-version.cpp
@@ -35,7 +35,7 @@ struct MyClass {
   int unused_with_implicit_forward_default_def(void);
   int __attribute__((target_version("lse"))) unused_with_implicit_forward_default_def(void);
 
-  // This should generate a normal function.
+  // This should generate a target version despite the default not being declared.
   int __attribute__((target_version("rdm"))) unused_without_default(void);
 };
 
@@ -75,6 +75,13 @@ int bar() {
 // CHECK: @_ZN7MyClass32unused_with_implicit_default_defEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClass32unused_with_implicit_default_defEv.resolver
 // CHECK: @_ZN7MyClass40unused_with_implicit_forward_default_defEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClass40unused_with_implicit_forward_default_defEv.resolver
 //.
+// CHECK-LABEL: @_Z3fooi._Mbf16Msme-f64f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DOTADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4
+// CHECK-NEXT:    ret i32 1
+//
+//
 // CHECK-LABEL: @_Z3fooi.default(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[DOTADDR:%.*]] = alloca i32, align 4
@@ -82,6 +89,11 @@ int bar() {
 // CHECK-NEXT:    ret i32 2
 //
 //
+// CHECK-LABEL: @_Z3foov._Mebf16Msm4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 3
+//
+//
 // CHECK-LABEL: @_Z3foov.default(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 4
@@ -189,6 +201,14 @@ int bar() {
 // CHECK-NEXT:    ret i32 1
 //
 //
+// CHECK-LABEL: @_ZN7MyClass22unused_without_defaultEv._Mrdm(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
+// CHECK-NEXT:    ret i32 0
+//
+//
 // CHECK-LABEL: @_Z3barv(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[M:%.*]] = alloca [[STRUCT_MYCLASS:%.*]], align 1
@@ -250,26 +270,6 @@ int bar() {
 // CHECK-NEXT:    ret ptr @_Z3foov.default
 //
 //
-// CHECK-LABEL: @_Z3fooi._Mbf16Msme-f64f64(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[DOTADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4
-// CHECK-NEXT:    ret i32 1
-//
-//
-// CHECK-LABEL: @_Z3foov._Mebf16Msm4(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 3
-//
-//
-// CHECK-LABEL: @_ZN7MyClass22unused_without_defaultEv(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
-// CHECK-NEXT:    store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8
-// CHECK-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK-NEXT:    ret i32 0
-//
-//
 // CHECK-LABEL: @_ZN7MyClass23unused_with_default_defEv.resolver(
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
@@ -312,16 +312,16 @@ int bar() {
 // CHECK-NEXT:    ret ptr @_ZN7MyClass40unused_with_implicit_forward_default_defEv.default
 //
 //.
-// CHECK: attributes #[[ATTR0:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
-// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc" }
-// CHECK: attributes #[[ATTR2:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon" }
-// CHECK: attributes #[[ATTR3:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops" }
-// CHECK: attributes #[[ATTR4:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" }
-// CHECK: attributes #[[ATTR5:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve" }
-// CHECK: attributes #[[ATTR6:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" }
-// CHECK: attributes #[[ATTR7:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse" }
-// CHECK: attributes #[[ATTR8:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme-f64f64" }
-// CHECK: attributes #[[ATTR9:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+neon,+sm4" }
+// CHECK: attributes #[[ATTR0:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme-f64f64" }
+// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+// CHECK: attributes #[[ATTR2:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+neon,+sm4" }
+// CHECK: attributes #[[ATTR3:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc" }
+// CHECK: attributes #[[ATTR4:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon" }
+// CHECK: attributes #[[ATTR5:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops" }
+// CHECK: attributes #[[ATTR6:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" }
+// CHECK: attributes #[[ATTR7:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve" }
+// CHECK: attributes #[[ATTR8:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" }
+// CHECK: attributes #[[ATTR9:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse" }
 // CHECK: attributes #[[ATTR10:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm" }
 // CHECK: attributes #[[ATTR11:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
 //.
diff --git a/clang/test/Sema/aarch64-sme-func-attrs.c b/clang/test/Sema/aarch64-sme-func-attrs.c
index 47dbeca206a9..bfc8768c3f36 100644
--- a/clang/test/Sema/aarch64-sme-func-attrs.c
+++ b/clang/test/Sema/aarch64-sme-func-attrs.c
@@ -483,14 +483,16 @@ void just_fine(void) {}
 
 __arm_locally_streaming
 __attribute__((target_version("sme2")))
-void just_fine_locally_streaming(void) {}
+void incompatible_locally_streaming(void) {}
+// expected-error@-1 {{attribute 'target_version' multiversioning cannot be combined with attribute '__arm_locally_streaming'}}
+// expected-cpp-error@-2 {{attribute 'target_version' multiversioning cannot be combined with attribute '__arm_locally_streaming'}}
 __attribute__((target_version("default")))
-void just_fine_locally_streaming(void) {}
+void incompatible_locally_streaming(void) {}
 
 
 void fmv_caller() {
     cannot_work_version();
     cannot_work_clones();
     just_fine();
-    just_fine_locally_streaming();
+    incompatible_locally_streaming();
 }
diff --git a/clang/test/Sema/attr-target-version.c b/clang/test/Sema/attr-target-version.c
index e2940c434c2f..cd5be459456e 100644
--- a/clang/test/Sema/attr-target-version.c
+++ b/clang/test/Sema/attr-target-version.c
@@ -68,13 +68,15 @@ int __attribute__((target_version(""))) unsup1(void) { return 1; }
 void __attribute__((target_version("crc32"))) unsup2(void) {}
 
 void __attribute__((target_version("default+fp16"))) koo(void) {}
+//expected-error@-1 {{function multiversioning doesn't support feature 'default'}}
 void __attribute__((target_version("default+default+default"))) loo(void) {}
+//expected-error@-1 {{function multiversioning doesn't support feature 'default'}}
 void __attribute__((target_version("rdm+rng+crc"))) redef(void) {}
 //expected-error@+2 {{redefinition of 'redef'}}
 //expected-note@-2 {{previous definition is here}}
 void __attribute__((target_version("rdm+rng+crc"))) redef(void) {}
 
-int __attribute__((target_version("sm4"))) def(void);
+int def(void);
 void __attribute__((target_version("dit"))) nodef(void);
 void __attribute__((target_version("ls64"))) nodef(void);
 void __attribute__((target_version("aes"))) ovl(void);
@@ -83,7 +85,6 @@ int bar() {
   // expected-error@+2 {{reference to overloaded function could not be resolved; did you mean to call it?}}
   // expected-note@-3 {{possible target for call}}
   ovl++;
-  // expected-error@+1 {{no matching function for call to 'nodef'}}
   nodef();
   return def();
 }
@@ -92,8 +93,6 @@ int __attribute__((target_version("sha1"))) def(void) { return 1; }
 
 int __attribute__((target_version("sve"))) prot();
 // expected-error@-1 {{multiversioned function must have a prototype}}
-// expected-note@+1 {{function multiversioning caused by this declaration}}
-int __attribute__((target_version("fcma"))) prot();
 
 int __attribute__((target_version("pmull"))) rtype(int);
 // expected-error@+1 {{multiversioned function declaration has a different return type}}
@@ -104,6 +103,7 @@ int __attribute__((target_version("sha2"))) combine(void) { return 1; }
 int __attribute__((aarch64_vector_pcs, target_version("sha3"))) combine(void) { return 2; }
 
 int __attribute__((target_version("fp+aes+pmull+rcpc"))) unspec_args() { return -1; }
+// expected-error@-1 {{multiversioned function must have a prototype}}
 // expected-error@+1 {{multiversioned function must have a prototype}}
 int __attribute__((target_version("default"))) unspec_args() { return 0; }
 int cargs() { return unspec_args(); }
diff --git a/clang/test/SemaCXX/attr-target-version.cpp b/clang/test/SemaCXX/attr-target-version.cpp
index 0bd710c4e282..b3385f043590 100644
--- a/clang/test/SemaCXX/attr-target-version.cpp
+++ b/clang/test/SemaCXX/attr-target-version.cpp
@@ -9,7 +9,6 @@ void __attribute__((target_version("rcpc3"))) no_def(void);
 void __attribute__((target_version("mops"))) no_def(void);
 void __attribute__((target_version("rdma"))) no_def(void);
 
-// expected-error@+1 {{no matching function for call to 'no_def'}}
 void foo(void) { no_def(); }
 
 constexpr int __attribute__((target_version("sve2"))) diff_const(void) { return 1; }
@@ -41,6 +40,7 @@ inline int __attribute__((target_version("sme"))) diff_inline(void) { return 1;
 int __attribute__((target_version("fp16"))) diff_inline(void) { return 2; }
 
 inline int __attribute__((target_version("sme"))) diff_inline1(void) { return 1; }
+//expected-error@+1 {{multiversioned function declaration has a different inline specification}}
 int __attribute__((target_version("default"))) diff_inline1(void) { return 2; }
 
 int __attribute__((target_version("fcma"))) diff_type1(void) { return 1; }
@@ -59,8 +59,7 @@ int __attribute__((target_version("sve2-sha3"))) diff_type3(void) noexcept(true)
 template <typename T> int __attribute__((target_version("default"))) temp(T) { return 1; }
 
 template <typename T> int __attribute__((target_version("simd"))) temp1(T) { return 1; }
-// expected-error@+1 {{attribute 'target_version' multiversioned functions do not yet support function templates}}
-template <typename T> int __attribute__((target_version("sha3"))) temp1(T) { return 2; }
+// expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support function templates}}
 
 extern "C" {
 int __attribute__((target_version("aes"))) extc(void) { return 1; }
@@ -70,17 +69,23 @@ int __attribute__((target_version("lse"))) extc(void) { return 1; }
 
 auto __attribute__((target_version("default"))) ret1(void) { return 1; }
 auto __attribute__((target_version("dpb"))) ret2(void) { return 1; }
+// expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support deduced return types}}
 auto __attribute__((target_version("dpb2"))) ret3(void) -> int { return 1; }
 
 class Cls {
   __attribute__((target_version("rng"))) Cls();
+  // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support constructors}}
   __attribute__((target_version("sve-i8mm"))) ~Cls();
+  // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support destructors}}
 
   Cls &__attribute__((target_version("f32mm"))) operator=(const Cls &) = default;
+  // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support defaulted functions}}
   Cls &__attribute__((target_version("ssbs"))) operator=(Cls &&) = delete;
+  // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support deleted functions}}
 
   virtual void __attribute__((target_version("default"))) vfunc();
   virtual void __attribute__((target_version("sm4"))) vfunc1();
+  // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support virtual functions}}
 };
 
 __attribute__((target_version("sha3"))) void Decl();
-- 
GitLab


From 7434a6b96c68b6835a6de6e98199dcfc35981dd5 Mon Sep 17 00:00:00 2001
From: pvanhout <pierre.vanhoutryve@amd.com>
Date: Mon, 25 Mar 2024 10:50:08 +0100
Subject: [PATCH 102/404] [TableGen] Fix Linker Errors

Fix linker errors after landing fa3d789df15bd1f58fb8ba4ea3be909218cf7f03
---
 llvm/utils/TableGen/Basic/CMakeLists.txt  | 2 +-
 llvm/utils/TableGen/CMakeLists.txt        | 2 +-
 llvm/utils/TableGen/Common/CMakeLists.txt | 1 -
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/llvm/utils/TableGen/Basic/CMakeLists.txt b/llvm/utils/TableGen/Basic/CMakeLists.txt
index f2927d05c175..5a899e3b7c80 100644
--- a/llvm/utils/TableGen/Basic/CMakeLists.txt
+++ b/llvm/utils/TableGen/Basic/CMakeLists.txt
@@ -8,7 +8,7 @@ set(LLVM_LINK_COMPONENTS
   TableGen
   )
 
-add_llvm_library(LLVMTableGenBasic STATIC OBJECT EXCLUDE_FROM_ALL
+add_llvm_library(LLVMTableGenBasic OBJECT EXCLUDE_FROM_ALL
   CodeGenIntrinsics.cpp
   SDNodeProperties.cpp
 )
diff --git a/llvm/utils/TableGen/CMakeLists.txt b/llvm/utils/TableGen/CMakeLists.txt
index 14690329cabf..577aeded4be7 100644
--- a/llvm/utils/TableGen/CMakeLists.txt
+++ b/llvm/utils/TableGen/CMakeLists.txt
@@ -75,10 +75,10 @@ add_tablegen(llvm-tblgen LLVM
   X86MnemonicTables.cpp
   X86ModRMFilters.cpp
   X86RecognizableInstr.cpp
+  $<TARGET_OBJECTS:obj.LLVMTableGenBasic>
   $<TARGET_OBJECTS:obj.LLVMTableGenCommon>
 
   DEPENDS
   intrinsics_gen # via llvm-min-tablegen
   )
-target_link_libraries(llvm-tblgen PRIVATE LLVMTableGenCommon)
 set_target_properties(llvm-tblgen PROPERTIES FOLDER "Tablegenning")
diff --git a/llvm/utils/TableGen/Common/CMakeLists.txt b/llvm/utils/TableGen/Common/CMakeLists.txt
index 491d9bd2949d..0e985a083247 100644
--- a/llvm/utils/TableGen/Common/CMakeLists.txt
+++ b/llvm/utils/TableGen/Common/CMakeLists.txt
@@ -39,7 +39,6 @@ add_llvm_library(LLVMTableGenCommon STATIC OBJECT EXCLUDE_FROM_ALL
   vt_gen
   )
 set_target_properties(LLVMTableGenCommon PROPERTIES FOLDER "Tablegenning")
-target_link_libraries(LLVMTableGenCommon PUBLIC LLVMTableGenBasic)
 
 # Users may include its headers as "Common/*.h"
 target_include_directories(LLVMTableGenCommon
-- 
GitLab


From 336bdf1a255571f8d894e8befe4be7e9141f7541 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Storsj=C3=B6?= <martin@martin.st>
Date: Mon, 25 Mar 2024 11:52:29 +0200
Subject: [PATCH 103/404] [verify-uselistorder] Fix a typo, fix linking

This fixes a typo from 2ef612050844355906e4b67d892a00bbb58c41d6,
which broke the build with errors like:

    ld.lld: error: undefined symbol: LoadBitcodeIntoNewDbgInforFormat
    >>> referenced by verify-uselistorder.cpp
    >>>               tools/verify-uselistorder/CMakeFiles/verify-uselistorder.dir/verify-uselistorder.cpp.o:(main)
    >>> did you mean: LoadBitcodeIntoNewDbgInfoFormat
    >>> defined in: lib/libLLVMBitReader.a(BitcodeReader.cpp.o)
    collect2: error: ld returned 1 exit status
---
 llvm/tools/verify-uselistorder/verify-uselistorder.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp
index 8b299c394e4e..cb07dede1d13 100644
--- a/llvm/tools/verify-uselistorder/verify-uselistorder.cpp
+++ b/llvm/tools/verify-uselistorder/verify-uselistorder.cpp
@@ -68,7 +68,7 @@ static cl::opt<unsigned>
                 cl::desc("Number of times to shuffle and verify use-lists"),
                 cl::init(1), cl::cat(Cat));
 
-extern cl::opt<cl::boolOrDefault> LoadBitcodeIntoNewDbgInforFormat;
+extern cl::opt<cl::boolOrDefault> LoadBitcodeIntoNewDbgInfoFormat;
 
 namespace {
 
@@ -546,8 +546,8 @@ int main(int argc, char **argv) {
                               "llvm tool to verify use-list order\n");
 
   // Do not load bitcode into the new debug info format by default.
-  if (LoadBitcodeIntoNewDbgInforFormat == cl::boolOrDefault::BOU_UNSET)
-    LoadBitcodeIntoNewDbgInforFormat = cl::boolOrDefault::BOU_FALSE;
+  if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET)
+    LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_FALSE;
 
   LLVMContext Context;
   SMDiagnostic Err;
-- 
GitLab


From dbfc38ed6b3f2a9be0b1a86b2a074aad69eb58a6 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Mon, 25 Mar 2024 18:57:53 +0900
Subject: [PATCH 104/404] [mlir][bufferization] Add `BufferOriginAnalysis`
 (#86461)

This commit adds the `BufferOriginAnalysis`, which can be queried to
check if two buffer SSA values originate from the same allocation. This
new analysis is used in the buffer deallocation pass to fold away or
simplify `bufferization.dealloc` ops more aggressively.

The `BufferOriginAnalysis` is based on the `BufferViewFlowAnalysis`,
which collects buffer SSA value "same buffer" dependencies. E.g., given
IR such as:
```
%0 = memref.alloc()
%1 = memref.subview %0
%2 = memref.subview %1
```
The `BufferViewFlowAnalysis` will report the following "reverse"
dependencies (`resolveReverse`) for `%2`: {`%2`, `%1`, `%0`}. I.e., all
buffer SSA values in the reverse use-def chain that originate from the
same allocation as `%2`. The `BufferOriginAnalysis` is built on top of
that. It handles only simple cases at the moment and may conservatively
return "unknown" around certain IR with branches, memref globals and
function arguments.

This analysis enables additional simplifications during
`-buffer-deallocation-simplification`. In particular, "regular" scf.for
loop nests, that yield buffers (or reallocations thereof) in the same
order as they appear in the iter_args, are now handled much more
efficiently. Such IR patterns are generated by the sparse compiler.
---
 .../Bufferization/IR/BufferizationOps.td      |   1 +
 .../Transforms/BufferViewFlowAnalysis.h       |  36 ++++
 .../BufferDeallocationSimplification.cpp      |  80 ++++-----
 .../Transforms/BufferViewFlowAnalysis.cpp     | 160 ++++++++++++++++--
 .../dealloc-loops.mlir                        |  86 ++++++++++
 .../buffer-deallocation-simplification.mlir   |  14 +-
 6 files changed, 321 insertions(+), 56 deletions(-)
 create mode 100644 mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir

diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
index 9dc6afcaab31..4f609ddff9a4 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
@@ -10,6 +10,7 @@
 #define BUFFERIZATION_OPS
 
 include "mlir/Dialect/Bufferization/IR/AllocationOpInterface.td"
+include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td"
 include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td"
 include "mlir/Dialect/Bufferization/IR/BufferizationBase.td"
 include "mlir/Interfaces/DestinationStyleOpInterface.td"
diff --git a/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h b/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h
index 9e43265c5dfe..4015231c845d 100644
--- a/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h
+++ b/mlir/include/mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h
@@ -53,6 +53,7 @@ public:
   ///
   /// Results in resolve(B) returning {B, C}
   ValueSetT resolve(Value value) const;
+  ValueSetT resolveReverse(Value value) const;
 
   /// Removes the given values from all alias sets.
   void remove(const SetVector<Value> &aliasValues);
@@ -73,11 +74,46 @@ private:
 
   /// Maps values to all immediate dependencies this value can have.
   ValueMapT dependencies;
+  ValueMapT reverseDependencies;
 
   /// A set of all SSA values that may be terminal buffers.
   DenseSet<Value> terminals;
 };
 
+/// An is-same-buffer analysis that checks if two SSA values belong to the same
+/// buffer allocation or not.
+class BufferOriginAnalysis {
+public:
+  BufferOriginAnalysis(Operation *op);
+
+  /// Return "true" if `v1` and `v2` originate from the same buffer allocation.
+  /// Return "false" if `v1` and `v2` originate from different allocations.
+  /// Return "nullopt" if we do not know for sure.
+  ///
+  /// Example 1: isSameAllocation(%0, %1) == true
+  /// ```
+  /// %0 = memref.alloc()
+  /// %1 = memref.subview %0
+  /// ```
+  ///
+  /// Example 2: isSameAllocation(%0, %1) == false
+  /// ```
+  /// %0 = memref.alloc()
+  /// %1 = memref.alloc()
+  /// ```
+  ///
+  /// Example 3: isSameAllocation(%0, %2) == nullopt
+  /// ```
+  /// %0 = memref.alloc()
+  /// %1 = memref.alloc()
+  /// %2 = arith.select %c, %0, %1
+  /// ```
+  std::optional<bool> isSameAllocation(Value v1, Value v2);
+
+private:
+  BufferViewFlowAnalysis analysis;
+};
+
 } // namespace mlir
 
 #endif // MLIR_DIALECT_BUFFERIZATION_TRANSFORMS_BUFFERVIEWFLOWANALYSIS_H
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
index e30779868b47..954485cfede3 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
@@ -12,8 +12,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Analysis/AliasAnalysis.h"
 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
+#include "mlir/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.h"
 #include "mlir/Dialect/Bufferization/Transforms/Passes.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
@@ -34,6 +34,14 @@ using namespace mlir::bufferization;
 // Helpers
 //===----------------------------------------------------------------------===//
 
+/// Given a memref value, return the "base" value by skipping over all
+/// ViewLikeOpInterface ops (if any) in the reverse use-def chain.
+static Value getViewBase(Value value) {
+  while (auto viewLikeOp = value.getDefiningOp<ViewLikeOpInterface>())
+    value = viewLikeOp.getViewSource();
+  return value;
+}
+
 static LogicalResult updateDeallocIfChanged(DeallocOp deallocOp,
                                             ValueRange memrefs,
                                             ValueRange conditions,
@@ -49,14 +57,6 @@ static LogicalResult updateDeallocIfChanged(DeallocOp deallocOp,
   return success();
 }
 
-/// Given a memref value, return the "base" value by skipping over all
-/// ViewLikeOpInterface ops (if any) in the reverse use-def chain.
-static Value getViewBase(Value value) {
-  while (auto viewLikeOp = value.getDefiningOp<ViewLikeOpInterface>())
-    value = viewLikeOp.getViewSource();
-  return value;
-}
-
 /// Return "true" if the given values are guaranteed to be different (and
 /// non-aliasing) allocations based on the fact that one value is the result
 /// of an allocation and the other value is a block argument of a parent block.
@@ -80,12 +80,14 @@ static bool distinctAllocAndBlockArgument(Value v1, Value v2) {
 /// Checks if `memref` may potentially alias a MemRef in `otherList`. It is
 /// often a requirement of optimization patterns that there cannot be any
 /// aliasing memref in order to perform the desired simplification.
-static bool potentiallyAliasesMemref(AliasAnalysis &analysis,
+static bool potentiallyAliasesMemref(BufferOriginAnalysis &analysis,
                                      ValueRange otherList, Value memref) {
   for (auto other : otherList) {
     if (distinctAllocAndBlockArgument(other, memref))
       continue;
-    if (!analysis.alias(other, memref).isNo())
+    std::optional<bool> analysisResult =
+        analysis.isSameAllocation(other, memref);
+    if (!analysisResult.has_value() || analysisResult == true)
       return true;
   }
   return false;
@@ -129,8 +131,8 @@ namespace {
 struct RemoveDeallocMemrefsContainedInRetained
     : public OpRewritePattern<DeallocOp> {
   RemoveDeallocMemrefsContainedInRetained(MLIRContext *context,
-                                          AliasAnalysis &aliasAnalysis)
-      : OpRewritePattern<DeallocOp>(context), aliasAnalysis(aliasAnalysis) {}
+                                          BufferOriginAnalysis &analysis)
+      : OpRewritePattern<DeallocOp>(context), analysis(analysis) {}
 
   /// The passed 'memref' must not have a may-alias relation to any retained
   /// memref, and at least one must-alias relation. If there is no must-aliasing
@@ -147,10 +149,11 @@ struct RemoveDeallocMemrefsContainedInRetained
     // deallocated in some situations and can thus not be dropped).
     bool atLeastOneMustAlias = false;
     for (Value retained : deallocOp.getRetained()) {
-      AliasResult analysisResult = aliasAnalysis.alias(retained, memref);
-      if (analysisResult.isMay())
+      std::optional<bool> analysisResult =
+          analysis.isSameAllocation(retained, memref);
+      if (!analysisResult.has_value())
         return failure();
-      if (analysisResult.isMust() || analysisResult.isPartial())
+      if (analysisResult == true)
         atLeastOneMustAlias = true;
     }
     if (!atLeastOneMustAlias)
@@ -161,8 +164,9 @@ struct RemoveDeallocMemrefsContainedInRetained
     // we can remove that operand later on.
     for (auto [i, retained] : llvm::enumerate(deallocOp.getRetained())) {
       Value updatedCondition = deallocOp.getUpdatedConditions()[i];
-      AliasResult analysisResult = aliasAnalysis.alias(retained, memref);
-      if (analysisResult.isMust() || analysisResult.isPartial()) {
+      std::optional<bool> analysisResult =
+          analysis.isSameAllocation(retained, memref);
+      if (analysisResult == true) {
         auto disjunction = rewriter.create<arith::OrIOp>(
             deallocOp.getLoc(), updatedCondition, cond);
         rewriter.replaceAllUsesExcept(updatedCondition, disjunction.getResult(),
@@ -206,7 +210,7 @@ struct RemoveDeallocMemrefsContainedInRetained
   }
 
 private:
-  AliasAnalysis &aliasAnalysis;
+  BufferOriginAnalysis &analysis;
 };
 
 /// Remove memrefs from the `retained` list which are guaranteed to not alias
@@ -228,15 +232,15 @@ private:
 struct RemoveRetainedMemrefsGuaranteedToNotAlias
     : public OpRewritePattern<DeallocOp> {
   RemoveRetainedMemrefsGuaranteedToNotAlias(MLIRContext *context,
-                                            AliasAnalysis &aliasAnalysis)
-      : OpRewritePattern<DeallocOp>(context), aliasAnalysis(aliasAnalysis) {}
+                                            BufferOriginAnalysis &analysis)
+      : OpRewritePattern<DeallocOp>(context), analysis(analysis) {}
 
   LogicalResult matchAndRewrite(DeallocOp deallocOp,
                                 PatternRewriter &rewriter) const override {
     SmallVector<Value> newRetainedMemrefs, replacements;
 
     for (auto retainedMemref : deallocOp.getRetained()) {
-      if (potentiallyAliasesMemref(aliasAnalysis, deallocOp.getMemrefs(),
+      if (potentiallyAliasesMemref(analysis, deallocOp.getMemrefs(),
                                    retainedMemref)) {
         newRetainedMemrefs.push_back(retainedMemref);
         replacements.push_back({});
@@ -264,7 +268,7 @@ struct RemoveRetainedMemrefsGuaranteedToNotAlias
   }
 
 private:
-  AliasAnalysis &aliasAnalysis;
+  BufferOriginAnalysis &analysis;
 };
 
 /// Split off memrefs to separate dealloc operations to reduce the number of
@@ -297,8 +301,8 @@ private:
 struct SplitDeallocWhenNotAliasingAnyOther
     : public OpRewritePattern<DeallocOp> {
   SplitDeallocWhenNotAliasingAnyOther(MLIRContext *context,
-                                      AliasAnalysis &aliasAnalysis)
-      : OpRewritePattern<DeallocOp>(context), aliasAnalysis(aliasAnalysis) {}
+                                      BufferOriginAnalysis &analysis)
+      : OpRewritePattern<DeallocOp>(context), analysis(analysis) {}
 
   LogicalResult matchAndRewrite(DeallocOp deallocOp,
                                 PatternRewriter &rewriter) const override {
@@ -314,7 +318,7 @@ struct SplitDeallocWhenNotAliasingAnyOther
       SmallVector<Value> otherMemrefs(deallocOp.getMemrefs());
       otherMemrefs.erase(otherMemrefs.begin() + i);
       // Check if `memref` can split off into a separate bufferization.dealloc.
-      if (potentiallyAliasesMemref(aliasAnalysis, otherMemrefs, memref)) {
+      if (potentiallyAliasesMemref(analysis, otherMemrefs, memref)) {
         // `memref` alias with other memrefs, do not split off.
         remainingMemrefs.push_back(memref);
         remainingConditions.push_back(cond);
@@ -352,7 +356,7 @@ struct SplitDeallocWhenNotAliasingAnyOther
   }
 
 private:
-  AliasAnalysis &aliasAnalysis;
+  BufferOriginAnalysis &analysis;
 };
 
 /// Check for every retained memref if a must-aliasing memref exists in the
@@ -381,8 +385,8 @@ private:
 struct RetainedMemrefAliasingAlwaysDeallocatedMemref
     : public OpRewritePattern<DeallocOp> {
   RetainedMemrefAliasingAlwaysDeallocatedMemref(MLIRContext *context,
-                                                AliasAnalysis &aliasAnalysis)
-      : OpRewritePattern<DeallocOp>(context), aliasAnalysis(aliasAnalysis) {}
+                                                BufferOriginAnalysis &analysis)
+      : OpRewritePattern<DeallocOp>(context), analysis(analysis) {}
 
   LogicalResult matchAndRewrite(DeallocOp deallocOp,
                                 PatternRewriter &rewriter) const override {
@@ -396,8 +400,9 @@ struct RetainedMemrefAliasingAlwaysDeallocatedMemref
         if (!matchPattern(cond, m_One()))
           continue;
 
-        AliasResult analysisResult = aliasAnalysis.alias(retained, memref);
-        if (analysisResult.isMust() || analysisResult.isPartial()) {
+        std::optional<bool> analysisResult =
+            analysis.isSameAllocation(retained, memref);
+        if (analysisResult == true) {
           rewriter.replaceAllUsesWith(res, cond);
           aliasesWithConstTrueMemref[i] = true;
           canDropMemref = true;
@@ -411,10 +416,9 @@ struct RetainedMemrefAliasingAlwaysDeallocatedMemref
         if (!extractOp)
           continue;
 
-        AliasResult extractAnalysisResult =
-            aliasAnalysis.alias(retained, extractOp.getOperand());
-        if (extractAnalysisResult.isMust() ||
-            extractAnalysisResult.isPartial()) {
+        std::optional<bool> extractAnalysisResult =
+            analysis.isSameAllocation(retained, extractOp.getOperand());
+        if (extractAnalysisResult == true) {
           rewriter.replaceAllUsesWith(res, cond);
           aliasesWithConstTrueMemref[i] = true;
           canDropMemref = true;
@@ -434,7 +438,7 @@ struct RetainedMemrefAliasingAlwaysDeallocatedMemref
   }
 
 private:
-  AliasAnalysis &aliasAnalysis;
+  BufferOriginAnalysis &analysis;
 };
 
 } // namespace
@@ -452,13 +456,13 @@ struct BufferDeallocationSimplificationPass
     : public bufferization::impl::BufferDeallocationSimplificationBase<
           BufferDeallocationSimplificationPass> {
   void runOnOperation() override {
-    AliasAnalysis &aliasAnalysis = getAnalysis<AliasAnalysis>();
+    BufferOriginAnalysis analysis(getOperation());
     RewritePatternSet patterns(&getContext());
     patterns.add<RemoveDeallocMemrefsContainedInRetained,
                  RemoveRetainedMemrefsGuaranteedToNotAlias,
                  SplitDeallocWhenNotAliasingAnyOther,
                  RetainedMemrefAliasingAlwaysDeallocatedMemref>(&getContext(),
-                                                                aliasAnalysis);
+                                                                analysis);
     populateDeallocOpCanonicalizationPatterns(patterns, &getContext());
 
     if (failed(
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp
index 9a36057425f3..72f47b8b468e 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp
@@ -19,22 +19,23 @@
 using namespace mlir;
 using namespace mlir::bufferization;
 
+//===----------------------------------------------------------------------===//
+// BufferViewFlowAnalysis
+//===----------------------------------------------------------------------===//
+
 /// Constructs a new alias analysis using the op provided.
 BufferViewFlowAnalysis::BufferViewFlowAnalysis(Operation *op) { build(op); }
 
-/// Find all immediate and indirect dependent buffers this value could
-/// potentially have. Note that the resulting set will also contain the value
-/// provided as it is a dependent alias of itself.
-BufferViewFlowAnalysis::ValueSetT
-BufferViewFlowAnalysis::resolve(Value rootValue) const {
-  ValueSetT result;
+static BufferViewFlowAnalysis::ValueSetT
+resolveValues(const BufferViewFlowAnalysis::ValueMapT &map, Value value) {
+  BufferViewFlowAnalysis::ValueSetT result;
   SmallVector<Value, 8> queue;
-  queue.push_back(rootValue);
+  queue.push_back(value);
   while (!queue.empty()) {
     Value currentValue = queue.pop_back_val();
     if (result.insert(currentValue).second) {
-      auto it = dependencies.find(currentValue);
-      if (it != dependencies.end()) {
+      auto it = map.find(currentValue);
+      if (it != map.end()) {
         for (Value aliasValue : it->second)
           queue.push_back(aliasValue);
       }
@@ -43,6 +44,19 @@ BufferViewFlowAnalysis::resolve(Value rootValue) const {
   return result;
 }
 
+/// Find all immediate and indirect dependent buffers this value could
+/// potentially have. Note that the resulting set will also contain the value
+/// provided as it is a dependent alias of itself.
+BufferViewFlowAnalysis::ValueSetT
+BufferViewFlowAnalysis::resolve(Value rootValue) const {
+  return resolveValues(dependencies, rootValue);
+}
+
+BufferViewFlowAnalysis::ValueSetT
+BufferViewFlowAnalysis::resolveReverse(Value rootValue) const {
+  return resolveValues(reverseDependencies, rootValue);
+}
+
 /// Removes the given values from all alias sets.
 void BufferViewFlowAnalysis::remove(const SetVector<Value> &aliasValues) {
   for (auto &entry : dependencies)
@@ -69,8 +83,10 @@ void BufferViewFlowAnalysis::rename(Value from, Value to) {
 void BufferViewFlowAnalysis::build(Operation *op) {
   // Registers all dependencies of the given values.
   auto registerDependencies = [&](ValueRange values, ValueRange dependencies) {
-    for (auto [value, dep] : llvm::zip_equal(values, dependencies))
+    for (auto [value, dep] : llvm::zip_equal(values, dependencies)) {
       this->dependencies[value].insert(dep);
+      this->reverseDependencies[dep].insert(value);
+    }
   };
 
   // Mark all buffer results and buffer region entry block arguments of the
@@ -188,3 +204,127 @@ bool BufferViewFlowAnalysis::mayBeTerminalBuffer(Value value) const {
   assert(isa<BaseMemRefType>(value.getType()) && "expected memref");
   return terminals.contains(value);
 }
+
+//===----------------------------------------------------------------------===//
+// BufferOriginAnalysis
+//===----------------------------------------------------------------------===//
+
+/// Return "true" if the given value is the result of a memory allocation.
+static bool hasAllocateSideEffect(Value v) {
+  Operation *op = v.getDefiningOp();
+  if (!op)
+    return false;
+  return hasEffect<MemoryEffects::Allocate>(op, v);
+}
+
+/// Return "true" if the given value is a function block argument.
+static bool isFunctionArgument(Value v) {
+  auto bbArg = dyn_cast<BlockArgument>(v);
+  if (!bbArg)
+    return false;
+  Block *b = bbArg.getOwner();
+  auto funcOp = dyn_cast<FunctionOpInterface>(b->getParentOp());
+  if (!funcOp)
+    return false;
+  return bbArg.getOwner() == &funcOp.getFunctionBody().front();
+}
+
+/// Given a memref value, return the "base" value by skipping over all
+/// ViewLikeOpInterface ops (if any) in the reverse use-def chain.
+static Value getViewBase(Value value) {
+  while (auto viewLikeOp = value.getDefiningOp<ViewLikeOpInterface>())
+    value = viewLikeOp.getViewSource();
+  return value;
+}
+
+BufferOriginAnalysis::BufferOriginAnalysis(Operation *op) : analysis(op) {}
+
+std::optional<bool> BufferOriginAnalysis::isSameAllocation(Value v1, Value v2) {
+  assert(isa<BaseMemRefType>(v1.getType()) && "expected buffer");
+  assert(isa<BaseMemRefType>(v2.getType()) && "expected buffer");
+
+  // Skip over all view-like ops.
+  v1 = getViewBase(v1);
+  v2 = getViewBase(v2);
+
+  // Fast path: If both buffers are the same SSA value, we can be sure that
+  // they originate from the same allocation.
+  if (v1 == v2)
+    return true;
+
+  // Compute the SSA values from which the buffers `v1` and `v2` originate.
+  SmallPtrSet<Value, 16> origin1 = analysis.resolveReverse(v1);
+  SmallPtrSet<Value, 16> origin2 = analysis.resolveReverse(v2);
+
+  // Originating buffers are "terminal" if they could not be traced back any
+  // further by the `BufferViewFlowAnalysis`. Examples of terminal buffers:
+  // - function block arguments
+  // - values defined by allocation ops such as "memref.alloc"
+  // - values defined by ops that are unknown to the buffer view flow analysis
+  // - values that are marked as "terminal" in the `BufferViewFlowOpInterface`
+  SmallPtrSet<Value, 16> terminal1, terminal2;
+
+  // While gathering terminal buffers, keep track of whether all terminal
+  // buffers are newly allocated buffer or function entry arguments.
+  bool allAllocs1 = true, allAllocs2 = true;
+  bool allAllocsOrFuncEntryArgs1 = true, allAllocsOrFuncEntryArgs2 = true;
+
+  // Helper function that gathers terminal buffers among `origin`.
+  auto gatherTerminalBuffers = [this](const SmallPtrSet<Value, 16> &origin,
+                                      SmallPtrSet<Value, 16> &terminal,
+                                      bool &allAllocs,
+                                      bool &allAllocsOrFuncEntryArgs) {
+    for (Value v : origin) {
+      if (isa<BaseMemRefType>(v.getType()) && analysis.mayBeTerminalBuffer(v)) {
+        terminal.insert(v);
+        allAllocs &= hasAllocateSideEffect(v);
+        allAllocsOrFuncEntryArgs &=
+            isFunctionArgument(v) || hasAllocateSideEffect(v);
+      }
+    }
+    assert(!terminal.empty() && "expected non-empty terminal set");
+  };
+
+  // Gather terminal buffers for `v1` and `v2`.
+  gatherTerminalBuffers(origin1, terminal1, allAllocs1,
+                        allAllocsOrFuncEntryArgs1);
+  gatherTerminalBuffers(origin2, terminal2, allAllocs2,
+                        allAllocsOrFuncEntryArgs2);
+
+  // If both `v1` and `v2` have a single matching terminal buffer, they are
+  // guaranteed to originate from the same buffer allocation.
+  if (llvm::hasSingleElement(terminal1) && llvm::hasSingleElement(terminal2) &&
+      *terminal1.begin() == *terminal2.begin())
+    return true;
+
+  // At least one of the two values has multiple terminals.
+
+  // Check if there is overlap between the terminal buffers of `v1` and `v2`.
+  bool distinctTerminalSets = true;
+  for (Value v : terminal1)
+    distinctTerminalSets &= !terminal2.contains(v);
+  // If there is overlap between the terminal buffers of `v1` and `v2`, we
+  // cannot make an accurate decision without further analysis.
+  if (!distinctTerminalSets)
+    return std::nullopt;
+
+  // If `v1` originates from only allocs, and `v2` is guaranteed to originate
+  // from different allocations (that is guaranteed if `v2` originates from
+  // only distinct allocs or function entry arguments), we can be sure that
+  // `v1` and `v2` originate from different allocations. The same argument can
+  // be made when swapping `v1` and `v2`.
+  bool isolatedAlloc1 = allAllocs1 && (allAllocs2 || allAllocsOrFuncEntryArgs2);
+  bool isolatedAlloc2 = (allAllocs1 || allAllocsOrFuncEntryArgs1) && allAllocs2;
+  if (isolatedAlloc1 || isolatedAlloc2)
+    return false;
+
+  // Otherwise: We do not know whether `v1` and `v2` originate from the same
+  // allocation or not.
+  // TODO: Function arguments are currently handled conservatively. We assume
+  // that they could be the same allocation.
+  // TODO: Terminals other than allocations and function arguments are
+  // currently handled conservatively. We assume that they could be the same
+  // allocation. E.g., we currently return "nullopt" for values that originate
+  // from different "memref.get_global" ops (with different symbols).
+  return std::nullopt;
+}
diff --git a/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir b/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir
new file mode 100644
index 000000000000..53b28c3aab6f
--- /dev/null
+++ b/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-loops.mlir
@@ -0,0 +1,86 @@
+// RUN: mlir-opt %s -expand-realloc="emit-deallocs=false" -ownership-based-buffer-deallocation="private-function-dynamic-ownership=true" -canonicalize -buffer-deallocation-simplification | FileCheck %s
+
+// A function that reallocates two buffer inside of a loop. The simplification
+// pass should be able to figure out that the iter_args are always originating
+// from different allocations. IR like this one appears in the sparse compiler.
+
+// CHECK-LABEL: func private @loop_with_realloc(
+func.func private @loop_with_realloc(%lb: index, %ub: index, %step: index, %c: i1, %s1: index, %s2: index) -> (memref<?xf32>, memref<?xf32>) {
+  // CHECK-DAG: %[[false:.*]] = arith.constant false
+  // CHECK-DAG: %[[true:.*]] = arith.constant true
+
+  // CHECK: %[[m0:.*]] = memref.alloc
+  %m0 = memref.alloc(%s1) : memref<?xf32>
+  // CHECK: %[[m1:.*]] = memref.alloc
+  %m1 = memref.alloc(%s1) : memref<?xf32>
+
+  // CHECK: %[[r:.*]]:4 = scf.for {{.*}} iter_args(%[[arg0:.*]] = %[[m0]], %[[arg1:.*]] = %[[m1]], %[[o0:.*]] = %[[false]], %[[o1:.*]] = %[[false]])
+  %r0, %r1 = scf.for %iv = %lb to %ub step %step iter_args(%arg0 = %m0, %arg1 = %m1) -> (memref<?xf32>, memref<?xf32>) {
+    //      CHECK: %[[m2:.*]]:2 = scf.if %{{.*}} -> (memref<?xf32>, i1) {
+    // CHECK-NEXT:   memref.alloc
+    // CHECK-NEXT:   memref.subview
+    // CHECK-NEXT:   memref.copy
+    // CHECK-NEXT:   scf.yield %{{.*}}, %[[true]]
+    // CHECK-NEXT: } else {
+    // CHECK-NEXT:   memref.reinterpret_cast
+    // CHECK-NEXT:   scf.yield %{{.*}}, %[[false]]
+    // CHECK-NEXT: }
+    %m2 = memref.realloc %arg0(%s2) : memref<?xf32> to memref<?xf32>
+    //      CHECK: %[[m3:.*]]:2 = scf.if %{{.*}} -> (memref<?xf32>, i1) {
+    // CHECK-NEXT:   memref.alloc
+    // CHECK-NEXT:   memref.subview
+    // CHECK-NEXT:   memref.copy
+    // CHECK-NEXT:   scf.yield %{{.*}}, %[[true]]
+    // CHECK-NEXT: } else {
+    // CHECK-NEXT:   memref.reinterpret_cast
+    // CHECK-NEXT:   scf.yield %{{.*}}, %[[false]]
+    // CHECK-NEXT: }
+    %m3 = memref.realloc %arg1(%s2) : memref<?xf32> to memref<?xf32>
+
+    // CHECK: %[[base0:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[arg0]]
+    // CHECK: %[[base1:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[arg1]]
+    // CHECK: %[[d0:.*]] = bufferization.dealloc (%[[base0]] : memref<f32>) if (%[[o0]]) retain (%[[m2]]#0 : memref<?xf32>)
+    // CHECK: %[[d1:.*]] = bufferization.dealloc (%[[base1]] : memref<f32>) if (%[[o1]]) retain (%[[m3]]#0 : memref<?xf32>)
+    // CHECK-DAG: %[[o2:.*]] = arith.ori %[[d0]], %[[m2]]#1
+    // CHECK-DAG: %[[o3:.*]] = arith.ori %[[d1]], %[[m3]]#1
+    // CHECK: scf.yield %[[m2]]#0, %[[m3]]#0, %[[o2]], %[[o3]]
+    scf.yield %m2, %m3 : memref<?xf32>, memref<?xf32>
+  }
+
+  // CHECK: %[[d2:.*]] = bufferization.dealloc (%[[m0]] : memref<?xf32>) if (%[[true]]) retain (%[[r]]#0 : memref<?xf32>)
+  // CHECK: %[[d3:.*]] = bufferization.dealloc (%[[m1]] : memref<?xf32>) if (%[[true]]) retain (%[[r]]#1 : memref<?xf32>)
+  // CHECK-DAG: %[[or0:.*]] = arith.ori %[[d2]], %[[r]]#2
+  // CHECK-DAG: %[[or1:.*]] = arith.ori %[[d3]], %[[r]]#3
+  // CHECK: return %[[r]]#0, %[[r]]#1, %[[or0]], %[[or1]]
+  return %r0, %r1 : memref<?xf32>, memref<?xf32>
+}
+
+// -----
+
+// The yielded values of the loop are swapped. Therefore, the
+// bufferization.dealloc before the func.return can now longer be split,
+// because %r0 could originate from either %m0 and %m1 (same for %r1).
+
+// CHECK-LABEL: func private @swapping_loop_with_realloc(
+func.func private @swapping_loop_with_realloc(%lb: index, %ub: index, %step: index, %c: i1, %s1: index, %s2: index) -> (memref<?xf32>, memref<?xf32>) {
+  // CHECK-DAG: %[[false:.*]] = arith.constant false
+  // CHECK-DAG: %[[true:.*]] = arith.constant true
+
+  // CHECK: %[[m0:.*]] = memref.alloc
+  %m0 = memref.alloc(%s1) : memref<?xf32>
+  // CHECK: %[[m1:.*]] = memref.alloc
+  %m1 = memref.alloc(%s1) : memref<?xf32>
+
+  // CHECK: %[[r:.*]]:4 = scf.for {{.*}} iter_args(%[[arg0:.*]] = %[[m0]], %[[arg1:.*]] = %[[m1]], %[[o0:.*]] = %[[false]], %[[o1:.*]] = %[[false]])
+  %r0, %r1 = scf.for %iv = %lb to %ub step %step iter_args(%arg0 = %m0, %arg1 = %m1) -> (memref<?xf32>, memref<?xf32>) {
+    %m2 = memref.realloc %arg0(%s2) : memref<?xf32> to memref<?xf32>
+    %m3 = memref.realloc %arg1(%s2) : memref<?xf32> to memref<?xf32>
+    scf.yield %m3, %m2 : memref<?xf32>, memref<?xf32>
+  }
+
+  // CHECK: %[[base0:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[r]]#0
+  // CHECK: %[[base1:.*]], %{{.*}}, %{{.*}}, %{{.*}} = memref.extract_strided_metadata %[[r]]#1
+  // CHECK: %[[d:.*]]:2 = bufferization.dealloc (%[[m0]], %[[m1]], %[[base0]], %[[base1]] : {{.*}}) if (%[[true]], %[[true]], %[[r]]#2, %[[r]]#3) retain (%[[r]]#0, %[[r]]#1 : {{.*}})
+  // CHECK: return %[[r]]#0, %[[r]]#1, %[[d]]#0, %[[d]]#1
+  return %r0, %r1 : memref<?xf32>, memref<?xf32>
+}
diff --git a/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir b/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir
index eee69acbe821..b40a17cf800b 100644
--- a/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir
+++ b/mlir/test/Dialect/Bufferization/Transforms/buffer-deallocation-simplification.mlir
@@ -92,15 +92,13 @@ func.func @dealloc_split_when_no_other_aliasing(%arg0: i1, %arg1: memref<2xi32>,
 //  CHECK-NEXT:   [[ALLOC0:%.+]] = memref.alloc(
 //  CHECK-NEXT:   [[ALLOC1:%.+]] = memref.alloc(
 //  CHECK-NEXT:   [[V0:%.+]] = arith.select{{.*}}[[ALLOC0]], [[ALLOC1]] :
-// COM: there is only one value in the retained list because the
-// COM: RemoveRetainedMemrefsGuaranteedToNotAlias pattern also applies here and
-// COM: removes %arg1 from the list. In the second dealloc, this does not apply
-// COM: because function arguments are assumed potentially alias (even if the
-// COM: types don't exactly match).
+// COM: there is only one value in the retained lists because the
+// COM: RemoveRetainedMemrefsGuaranteedToNotAlias pattern also applies here:
+// COM: - %alloc is guaranteed to not alias with %arg1.
+// COM: - %arg2 is guaranteed to not alias with %0.
 //  CHECK-NEXT:   [[V1:%.+]] = bufferization.dealloc ([[ALLOC0]] : memref<2xi32>) if ([[ARG0]]) retain ([[V0]] : memref<2xi32>)
-//  CHECK-NEXT:   [[V2:%.+]]:2 = bufferization.dealloc ([[ARG2]] : memref<2xi32>) if ([[ARG3]]) retain ([[ARG1]], [[V0]] : memref<2xi32>, memref<2xi32>)
-//  CHECK-NEXT:   [[V3:%.+]] = arith.ori [[V1]], [[V2]]#1
-//  CHECK-NEXT:   return [[V2]]#0, [[V3]] :
+//  CHECK-NEXT:   [[V2:%.+]] = bufferization.dealloc ([[ARG2]] : memref<2xi32>) if ([[ARG3]]) retain ([[ARG1]] : memref<2xi32>)
+//  CHECK-NEXT:   return [[V2]], [[V1]] :
 
 // -----
 
-- 
GitLab


From 94a550dab26c4b30a187dd8e3ce431e0f915923b Mon Sep 17 00:00:00 2001
From: Mariusz Sikora <mariusz.sikora@amd.com>
Date: Mon, 25 Mar 2024 11:00:59 +0100
Subject: [PATCH 105/404] [AMDGPU][NFC] Rename Feature GFX11FullVGPRs to
 1_5xVGPRs (#86468)

---
 llvm/lib/Target/AMDGPU/AMDGPU.td                | 14 ++++++++------
 llvm/lib/Target/AMDGPU/GCNSubtarget.h           |  4 ++--
 llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp |  4 ++--
 3 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index c877658cd38e..37dcfef3b2a3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -831,10 +831,12 @@ def FeatureNoDataDepHazard : SubtargetFeature<"no-data-dep-hazard",
   "Does not need SW waitstates"
 >;
 
-def FeatureGFX11FullVGPRs : SubtargetFeature<"gfx11-full-vgprs",
-  "HasGFX11FullVGPRs",
+// Allocate 1536 VGPRs for wave32 and 768 VGPRs for wave64
+// with allocation granularity 24 for wave32 and 12 for wave64
+def Feature1_5xVGPRs : SubtargetFeature<"allocate1_5xvgprs",
+  "Has1_5xVGPRs",
   "true",
-  "GFX11 with 50% more physical VGPRs and 50% larger allocation granule than GFX10"
+  "Has 50% more physical VGPRs and 50% larger allocation granule"
 >;
 
 
@@ -1491,12 +1493,12 @@ def FeatureISAVersion11_0_Common : FeatureSet<
 
 def FeatureISAVersion11_0_0 : FeatureSet<
   !listconcat(FeatureISAVersion11_0_Common.Features,
-    [FeatureGFX11FullVGPRs,
+    [Feature1_5xVGPRs,
      FeatureUserSGPRInit16Bug])>;
 
 def FeatureISAVersion11_0_1 : FeatureSet<
   !listconcat(FeatureISAVersion11_0_Common.Features,
-    [FeatureGFX11FullVGPRs])>;
+    [Feature1_5xVGPRs])>;
 
 def FeatureISAVersion11_0_2 : FeatureSet<
   !listconcat(FeatureISAVersion11_0_Common.Features,
@@ -1517,7 +1519,7 @@ def FeatureISAVersion11_5_1 : FeatureSet<
     [FeatureSALUFloatInsts,
      FeatureDPPSrc1SGPR,
      FeatureVGPRSingleUseHintInsts,
-     FeatureGFX11FullVGPRs])>;
+     Feature1_5xVGPRs])>;
 
 def FeatureISAVersion12 : FeatureSet<
   [FeatureGFX12,
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index ca51da659c33..4da10beabe31 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -223,7 +223,7 @@ protected:
   bool HasImageStoreD16Bug = false;
   bool HasImageGather4D16Bug = false;
   bool HasMSAALoadDstSelBug = false;
-  bool HasGFX11FullVGPRs = false;
+  bool Has1_5xVGPRs = false;
   bool HasMADIntraFwdBug = false;
   bool HasVOPDInsts = false;
   bool HasVALUTransUseHazard = false;
@@ -1202,7 +1202,7 @@ public:
   /// target.
   bool hasNullExportTarget() const { return !GFX11Insts; }
 
-  bool hasGFX11FullVGPRs() const { return HasGFX11FullVGPRs; }
+  bool has1_5xVGPRs() const { return Has1_5xVGPRs; }
 
   bool hasVOPDInsts() const { return HasVOPDInsts; }
 
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index a90dc32d396f..7bb84d78442b 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -1087,7 +1087,7 @@ unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI,
       *EnableWavefrontSize32 :
       STI->getFeatureBits().test(FeatureWavefrontSize32);
 
-  if (STI->getFeatureBits().test(FeatureGFX11FullVGPRs))
+  if (STI->getFeatureBits().test(Feature1_5xVGPRs))
     return IsWave32 ? 24 : 12;
 
   if (hasGFX10_3Insts(*STI))
@@ -1114,7 +1114,7 @@ unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) {
   if (!isGFX10Plus(*STI))
     return 256;
   bool IsWave32 = STI->getFeatureBits().test(FeatureWavefrontSize32);
-  if (STI->getFeatureBits().test(FeatureGFX11FullVGPRs))
+  if (STI->getFeatureBits().test(Feature1_5xVGPRs))
     return IsWave32 ? 1536 : 768;
   return IsWave32 ? 1024 : 512;
 }
-- 
GitLab


From 37785fedabd8fa752129ef5bac3462311af91c35 Mon Sep 17 00:00:00 2001
From: Discookie <viktor.cseh@ericsson.com>
Date: Mon, 25 Mar 2024 10:08:56 +0000
Subject: [PATCH 106/404] [clang][analyzer] Bring cplusplus.ArrayDelete out of
 alpha (#83985)

The checker finds a type of undefined behavior, where if the type of a
pointer to an object-array is different from the objects' underlying
type, calling `delete[]` is undefined, as the size of the two objects
might be different.

The checker has been in alpha for a while now, it is a simple checker
that causes no crashes, and considering the severity of the issue, it
has a low result-count on open-source projects (in my last test-run on
my usual projects, it had 0 results).

This commit cleans up the documentation and adds docs for the limitation
related to tracking through references, in addition to moving it to
`cplusplus`.

---------

Co-authored-by: Balazs Benics <benicsbalazs@gmail.com>
Co-authored-by: whisperity <whisperity@gmail.com>
---
 clang/docs/analyzer/checkers.rst              | 69 ++++++++++++-------
 .../clang/StaticAnalyzer/Checkers/Checkers.td | 10 +--
 .../Checkers/CXXDeleteChecker.cpp             |  4 +-
 clang/test/Analysis/ArrayDelete.cpp           |  2 +-
 clang/www/analyzer/alpha_checks.html          | 20 ------
 clang/www/analyzer/available_checks.html      | 27 ++++++++
 6 files changed, 80 insertions(+), 52 deletions(-)

diff --git a/clang/docs/analyzer/checkers.rst b/clang/docs/analyzer/checkers.rst
index fe2115149142..66da1c7b35f2 100644
--- a/clang/docs/analyzer/checkers.rst
+++ b/clang/docs/analyzer/checkers.rst
@@ -340,6 +340,51 @@ cplusplus
 
 C++ Checkers.
 
+.. _cplusplus-ArrayDelete:
+
+cplusplus.ArrayDelete (C++)
+"""""""""""""""""""""""""""
+
+Reports destructions of arrays of polymorphic objects that are destructed as
+their base class. If the dynamic type of the array is different from its static
+type, calling `delete[]` is undefined.
+
+This checker corresponds to the SEI CERT rule `EXP51-CPP: Do not delete an array through a pointer of the incorrect type <https://wiki.sei.cmu.edu/confluence/display/cplusplus/EXP51-CPP.+Do+not+delete+an+array+through+a+pointer+of+the+incorrect+type>`_.
+
+.. code-block:: cpp
+
+ class Base {
+ public:
+   virtual ~Base() {}
+ };
+ class Derived : public Base {};
+
+ Base *create() {
+   Base *x = new Derived[10]; // note: Casting from 'Derived' to 'Base' here
+   return x;
+ }
+
+ void foo() {
+   Base *x = create();
+   delete[] x; // warn: Deleting an array of 'Derived' objects as their base class 'Base' is undefined
+ }
+
+**Limitations**
+
+The checker does not emit note tags when casting to and from reference types,
+even though the pointer values are tracked across references.
+
+.. code-block:: cpp
+
+ void foo() {
+   Derived *d = new Derived[10];
+   Derived &dref = *d;
+
+   Base &bref = static_cast<Base&>(dref); // no note
+   Base *b = &bref;
+   delete[] b; // warn: Deleting an array of 'Derived' objects as their base class 'Base' is undefined
+ }
+
 .. _cplusplus-InnerPointer:
 
 cplusplus.InnerPointer (C++)
@@ -2139,30 +2184,6 @@ Either the comparison is useless or there is division by zero.
 alpha.cplusplus
 ^^^^^^^^^^^^^^^
 
-.. _alpha-cplusplus-ArrayDelete:
-
-alpha.cplusplus.ArrayDelete (C++)
-"""""""""""""""""""""""""""""""""
-Reports destructions of arrays of polymorphic objects that are destructed as their base class.
-This checker corresponds to the CERT rule `EXP51-CPP: Do not delete an array through a pointer of the incorrect type <https://wiki.sei.cmu.edu/confluence/display/cplusplus/EXP51-CPP.+Do+not+delete+an+array+through+a+pointer+of+the+incorrect+type>`_.
-
-.. code-block:: cpp
-
- class Base {
-   virtual ~Base() {}
- };
- class Derived : public Base {}
-
- Base *create() {
-   Base *x = new Derived[10]; // note: Casting from 'Derived' to 'Base' here
-   return x;
- }
-
- void foo() {
-   Base *x = create();
-   delete[] x; // warn: Deleting an array of 'Derived' objects as their base class 'Base' is undefined
- }
-
 .. _alpha-cplusplus-DeleteWithNonVirtualDtor:
 
 alpha.cplusplus.DeleteWithNonVirtualDtor (C++)
diff --git a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
index 686e5e99f4a6..bf46766d44b3 100644
--- a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
+++ b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
@@ -622,6 +622,11 @@ def BlockInCriticalSectionChecker : Checker<"BlockInCriticalSection">,
 
 let ParentPackage = Cplusplus in {
 
+def ArrayDeleteChecker : Checker<"ArrayDelete">,
+  HelpText<"Reports destructions of arrays of polymorphic objects that are "
+           "destructed as their base class.">,
+  Documentation<HasDocumentation>;
+
 def InnerPointerChecker : Checker<"InnerPointer">,
   HelpText<"Check for inner pointers of C++ containers used after "
            "re/deallocation">,
@@ -777,11 +782,6 @@ def ContainerModeling : Checker<"ContainerModeling">,
   Documentation<NotDocumented>,
   Hidden;
 
-def CXXArrayDeleteChecker : Checker<"ArrayDelete">,
-  HelpText<"Reports destructions of arrays of polymorphic objects that are "
-           "destructed as their base class.">,
-  Documentation<HasDocumentation>;
-
 def DeleteWithNonVirtualDtorChecker : Checker<"DeleteWithNonVirtualDtor">,
   HelpText<"Reports destructions of polymorphic objects with a non-virtual "
            "destructor in their base class">,
diff --git a/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp
index b4dee1e300e8..1b1226a7f1a7 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CXXDeleteChecker.cpp
@@ -220,11 +220,11 @@ CXXDeleteChecker::PtrCastVisitor::VisitNode(const ExplodedNode *N,
                                                     /*addPosRange=*/true);
 }
 
-void ento::registerCXXArrayDeleteChecker(CheckerManager &mgr) {
+void ento::registerArrayDeleteChecker(CheckerManager &mgr) {
   mgr.registerChecker<CXXArrayDeleteChecker>();
 }
 
-bool ento::shouldRegisterCXXArrayDeleteChecker(const CheckerManager &mgr) {
+bool ento::shouldRegisterArrayDeleteChecker(const CheckerManager &mgr) {
   return true;
 }
 
diff --git a/clang/test/Analysis/ArrayDelete.cpp b/clang/test/Analysis/ArrayDelete.cpp
index 3b8d49552376..6887e0a35fb8 100644
--- a/clang/test/Analysis/ArrayDelete.cpp
+++ b/clang/test/Analysis/ArrayDelete.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=alpha.cplusplus.ArrayDelete -std=c++11 -verify -analyzer-output=text %s
+// RUN: %clang_cc1 -analyze -analyzer-checker=cplusplus.ArrayDelete -std=c++11 -verify -analyzer-output=text %s
 
 struct Base {
     virtual ~Base() = default;
diff --git a/clang/www/analyzer/alpha_checks.html b/clang/www/analyzer/alpha_checks.html
index 7bbe4a20288f..f040d1957b0f 100644
--- a/clang/www/analyzer/alpha_checks.html
+++ b/clang/www/analyzer/alpha_checks.html
@@ -307,26 +307,6 @@ void test(int x) {
 <tbody>
 
 
-<tr><td><a id="alpha.cplusplus.ArrayDelete"><div class="namedescr expandable"><span class="name">
-alpha.cplusplus.ArrayDelete</span><span class="lang">
-(C++)</span><div class="descr">
-Reports destructions of arrays of polymorphic objects that are destructed as
-their base class
-</div></div></a></td>
-<td><div class="exampleContainer expandable">
-<div class="example"><pre>
-Base *create() {
-  Base *x = new Derived[10]; // note: Casting from 'Derived' to 'Base' here
-  return x;
-}
-
-void sink(Base *x) {
-  delete[] x; // warn: Deleting an array of 'Derived' objects as their base class 'Base' undefined
-}
-
-</pre></div></div></td></tr>
-
-
 <tr><td><a id="alpha.cplusplus.DeleteWithNonVirtualDtor"><div class="namedescr expandable"><span class="name">
 alpha.cplusplus.DeleteWithNonVirtualDtor</span><span class="lang">
 (C++)</span><div class="descr">
diff --git a/clang/www/analyzer/available_checks.html b/clang/www/analyzer/available_checks.html
index 501a2b306dfa..c23865e57e87 100644
--- a/clang/www/analyzer/available_checks.html
+++ b/clang/www/analyzer/available_checks.html
@@ -369,6 +369,33 @@ int test() {
 <thead><tr><td>Name, Description</td><td>Example</td></tr></thead>
 
 <tbody>
+
+<tr><td><a id="cplusplus.ArrayDelete"><div class="namedescr expandable"><span class="name">
+cplusplus.ArrayDelete</span><span class="lang">
+(C++)</span><div class="descr">
+Reports destructions of arrays of polymorphic objects that are destructed as
+their base class. If the type of an array-pointer is different from the type of
+its underlying objects, calling <code>delete[]</code> is undefined.
+</div></div></a></td>
+<td><div class="exampleContainer expandable">
+<div class="example"><pre>
+class Base {
+public:
+  virtual ~Base() {}
+};
+class Derived : public Base {};
+
+Base *create() {
+  Base *x = new Derived[10]; // note: Casting from 'Derived' to 'Base' here
+  return x;
+}
+
+void sink(Base *x) {
+  delete[] x; // warn: Deleting an array of 'Derived' objects as their base class 'Base' undefined
+}
+</pre></div></div></td></tr>
+
+
 <tr><td><a id="cplusplus.NewDelete"><div class="namedescr expandable"><span class="name">
 cplusplus.NewDelete</span><span class="lang">
 (C++)</span><div class="descr">
-- 
GitLab


From f0eb9083402c0130f240a7ef948b3f7b3bdf221c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nathan=20Gau=C3=ABr?= <brioche@google.com>
Date: Mon, 25 Mar 2024 11:30:47 +0100
Subject: [PATCH 107/404] [SPIR-V] Add WaveGetLaneIndex() intrinsic support
 (#85979)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add support to generate valid SPIR-V for the WaveGetLaneIndex() HLSL
builtin.

To implement this, I had to fix a few small issues in the backend, like
the i8* pointer type being emitted, even if we have the type information
elsewhere.

Signed-off-by: Nathan Gauër <brioche@google.com>
---
 llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp       | 33 ++++++---
 llvm/lib/Target/SPIRV/SPIRVBuiltins.td        |  2 +
 llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp   | 23 +++++--
 llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 13 ++--
 llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp |  2 +-
 llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp   |  4 +-
 llvm/lib/Target/SPIRV/SPIRVUtils.cpp          |  9 ++-
 .../SPIRV/hlsl-intrinsics/WaveGetLaneIndex.ll | 68 +++++++++++++++++++
 .../CodeGen/SPIRV/scfg-add-pre-headers.ll     |  9 ++-
 .../spirv-private-array-initialization.ll     |  2 -
 10 files changed, 131 insertions(+), 34 deletions(-)
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveGetLaneIndex.ll

diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 0478fc33cedc..950f9df28dd3 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -370,12 +370,10 @@ static Register buildLoadInst(SPIRVType *BaseType, Register PtrRegister,
 
 /// Helper function for building a load instruction for loading a builtin global
 /// variable of \p BuiltinValue value.
-static Register buildBuiltinVariableLoad(MachineIRBuilder &MIRBuilder,
-                                         SPIRVType *VariableType,
-                                         SPIRVGlobalRegistry *GR,
-                                         SPIRV::BuiltIn::BuiltIn BuiltinValue,
-                                         LLT LLType,
-                                         Register Reg = Register(0)) {
+static Register buildBuiltinVariableLoad(
+    MachineIRBuilder &MIRBuilder, SPIRVType *VariableType,
+    SPIRVGlobalRegistry *GR, SPIRV::BuiltIn::BuiltIn BuiltinValue, LLT LLType,
+    Register Reg = Register(0), bool isConst = true, bool hasLinkageTy = true) {
   Register NewRegister =
       MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::IDRegClass);
   MIRBuilder.getMRI()->setType(NewRegister,
@@ -387,8 +385,9 @@ static Register buildBuiltinVariableLoad(MachineIRBuilder &MIRBuilder,
   // Set up the global OpVariable with the necessary builtin decorations.
   Register Variable = GR->buildGlobalVariable(
       NewRegister, PtrType, getLinkStringForBuiltIn(BuiltinValue), nullptr,
-      SPIRV::StorageClass::Input, nullptr, true, true,
-      SPIRV::LinkageType::Import, MIRBuilder, false);
+      SPIRV::StorageClass::Input, nullptr, /* isConst= */ isConst,
+      /* HasLinkageTy */ hasLinkageTy, SPIRV::LinkageType::Import, MIRBuilder,
+      false);
 
   // Load the value from the global variable.
   Register LoadedRegister =
@@ -1341,6 +1340,22 @@ static bool generateDotOrFMulInst(const SPIRV::IncomingCall *Call,
   return true;
 }
 
+static bool generateWaveInst(const SPIRV::IncomingCall *Call,
+                             MachineIRBuilder &MIRBuilder,
+                             SPIRVGlobalRegistry *GR) {
+  const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
+  SPIRV::BuiltIn::BuiltIn Value =
+      SPIRV::lookupGetBuiltin(Builtin->Name, Builtin->Set)->Value;
+
+  // For now, we only support a single Wave intrinsic with a single return type.
+  assert(Call->ReturnType->getOpcode() == SPIRV::OpTypeInt);
+  LLT LLType = LLT::scalar(GR->getScalarOrVectorBitWidth(Call->ReturnType));
+
+  return buildBuiltinVariableLoad(
+      MIRBuilder, Call->ReturnType, GR, Value, LLType, Call->ReturnRegister,
+      /* isConst= */ false, /* hasLinkageTy= */ false);
+}
+
 static bool generateGetQueryInst(const SPIRV::IncomingCall *Call,
                                  MachineIRBuilder &MIRBuilder,
                                  SPIRVGlobalRegistry *GR) {
@@ -2229,6 +2244,8 @@ std::optional<bool> lowerBuiltin(const StringRef DemangledCall,
     return generateBarrierInst(Call.get(), MIRBuilder, GR);
   case SPIRV::Dot:
     return generateDotOrFMulInst(Call.get(), MIRBuilder, GR);
+  case SPIRV::Wave:
+    return generateWaveInst(Call.get(), MIRBuilder, GR);
   case SPIRV::GetQuery:
     return generateGetQueryInst(Call.get(), MIRBuilder, GR);
   case SPIRV::ImageSizeQuery:
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
index ee4f13d89c3c..660000fb548d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
@@ -41,6 +41,7 @@ def Variable : BuiltinGroup;
 def Atomic : BuiltinGroup;
 def Barrier : BuiltinGroup;
 def Dot : BuiltinGroup;
+def Wave : BuiltinGroup;
 def GetQuery : BuiltinGroup;
 def ImageSizeQuery : BuiltinGroup;
 def ImageMiscQuery : BuiltinGroup;
@@ -1143,6 +1144,7 @@ defm : DemangledGetBuiltin<"get_global_size", OpenCL_std, GetQuery, GlobalSize>;
 defm : DemangledGetBuiltin<"get_group_id", OpenCL_std, GetQuery, WorkgroupId>;
 defm : DemangledGetBuiltin<"get_enqueued_local_size", OpenCL_std, GetQuery, EnqueuedWorkgroupSize>;
 defm : DemangledGetBuiltin<"get_num_groups", OpenCL_std, GetQuery, NumWorkgroups>;
+defm : DemangledGetBuiltin<"__hlsl_wave_get_lane_index", GLSL_std_450, Wave, SubgroupLocalInvocationId>;
 
 //===----------------------------------------------------------------------===//
 // Class defining an image query builtin record used for lowering the OpenCL
diff --git a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
index 6f23f055b8c2..afdca01561b0 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
@@ -493,9 +493,15 @@ bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
   Register ResVReg =
       Info.OrigRet.Regs.empty() ? Register(0) : Info.OrigRet.Regs[0];
   const auto *ST = static_cast<const SPIRVSubtarget *>(&MF.getSubtarget());
-  // TODO: check that it's OCL builtin, then apply OpenCL_std.
-  if (!DemangledName.empty() && CF && CF->isDeclaration() &&
-      ST->canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
+
+  bool isFunctionDecl = CF && CF->isDeclaration();
+  bool canUseOpenCL = ST->canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std);
+  bool canUseGLSL = ST->canUseExtInstSet(SPIRV::InstructionSet::GLSL_std_450);
+  assert(canUseGLSL != canUseOpenCL &&
+         "Scenario where both sets are enabled is not supported.");
+
+  if (isFunctionDecl && !DemangledName.empty() &&
+      (canUseGLSL || canUseOpenCL)) {
     SmallVector<Register, 8> ArgVRegs;
     for (auto Arg : Info.OrigArgs) {
       assert(Arg.Regs.size() == 1 && "Call arg has multiple VRegs");
@@ -504,12 +510,15 @@ bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
       if (!GR->getSPIRVTypeForVReg(Arg.Regs[0]))
         GR->assignSPIRVTypeToVReg(SPIRVTy, Arg.Regs[0], MF);
     }
-    if (auto Res = SPIRV::lowerBuiltin(
-            DemangledName, SPIRV::InstructionSet::OpenCL_std, MIRBuilder,
-            ResVReg, OrigRetTy, ArgVRegs, GR))
+    auto instructionSet = canUseOpenCL ? SPIRV::InstructionSet::OpenCL_std
+                                       : SPIRV::InstructionSet::GLSL_std_450;
+    if (auto Res =
+            SPIRV::lowerBuiltin(DemangledName, instructionSet, MIRBuilder,
+                                ResVReg, OrigRetTy, ArgVRegs, GR))
       return *Res;
   }
-  if (CF && CF->isDeclaration() && !GR->find(CF, &MF).isValid()) {
+
+  if (isFunctionDecl && !GR->find(CF, &MF).isValid()) {
     // Emit the type info and forward function declaration to the first MBB
     // to ensure VReg definition dependencies are valid across all MBBs.
     MachineIRBuilder FirstBlockBuilder;
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index db66ed4f0e0f..9592f3e81b40 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -723,11 +723,14 @@ SPIRVType *SPIRVGlobalRegistry::createSPIRVType(
     AddrSpace = PType->getAddressSpace();
   else
     report_fatal_error("Unable to convert LLVM type to SPIRVType", true);
-  SPIRVType *SpvElementType;
-  // At the moment, all opaque pointers correspond to i8 element type.
-  // TODO: change the implementation once opaque pointers are supported
-  // in the SPIR-V specification.
-  SpvElementType = getOrCreateSPIRVIntegerType(8, MIRBuilder);
+
+  SPIRVType *SpvElementType = nullptr;
+  if (auto PType = dyn_cast<TypedPointerType>(Ty))
+    SpvElementType = getOrCreateSPIRVType(PType->getElementType(), MIRBuilder,
+                                          AccQual, EmitIR);
+  else
+    SpvElementType = getOrCreateSPIRVIntegerType(8, MIRBuilder);
+
   // Get access to information about available extensions
   const SPIRVSubtarget *ST =
       static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index 00d0cbd76373..40c3e5f9c6bd 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -658,7 +658,7 @@ void RequirementHandler::initAvailableCapabilitiesForVulkan(
 
   // Provided by all supported Vulkan versions.
   addAvailableCaps({Capability::Int16, Capability::Int64, Capability::Float16,
-                    Capability::Float64});
+                    Capability::Float64, Capability::GroupNonUniform});
 }
 
 } // namespace SPIRV
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index 1f0d8d8cd43a..41807da6afcb 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -186,7 +186,9 @@ static SPIRVType *propagateSPIRVType(MachineInstr *MI, SPIRVGlobalRegistry *GR,
       }
       case TargetOpcode::G_GLOBAL_VALUE: {
         MIB.setInsertPt(*MI->getParent(), MI);
-        Type *Ty = MI->getOperand(1).getGlobal()->getType();
+        const auto *Global = MI->getOperand(1).getGlobal();
+        auto *Ty = TypedPointerType::get(Global->getValueType(),
+                                         Global->getType()->getAddressSpace());
         SpirvTy = GR->getOrCreateSPIRVType(Ty, MIB);
         break;
       }
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index fc7502479fdc..c87c1293c622 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -306,10 +306,12 @@ static bool isNonMangledOCLBuiltin(StringRef Name) {
 std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) {
   bool IsNonMangledOCL = isNonMangledOCLBuiltin(Name);
   bool IsNonMangledSPIRV = Name.starts_with("__spirv_");
+  bool IsNonMangledHLSL = Name.starts_with("__hlsl_");
   bool IsMangled = Name.starts_with("_Z");
 
-  if (!IsNonMangledOCL && !IsNonMangledSPIRV && !IsMangled)
-    return std::string();
+  // Otherwise use simple demangling to return the function name.
+  if (IsNonMangledOCL || IsNonMangledSPIRV || IsNonMangledHLSL || !IsMangled)
+    return Name.str();
 
   // Try to use the itanium demangler.
   if (char *DemangledName = itaniumDemangle(Name.data())) {
@@ -317,9 +319,6 @@ std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) {
     free(DemangledName);
     return Result;
   }
-  // Otherwise use simple demangling to return the function name.
-  if (IsNonMangledOCL || IsNonMangledSPIRV)
-    return Name.str();
 
   // Autocheck C++, maybe need to do explicit check of the source language.
   // OpenCL C++ built-ins are declared in cl namespace.
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveGetLaneIndex.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveGetLaneIndex.ll
new file mode 100644
index 000000000000..ec35690ac154
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveGetLaneIndex.ll
@@ -0,0 +1,68 @@
+; RUN: llc -O0 -mtriple=spirv-vulkan-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-vulkan-unknown %s -o - -filetype=obj | spirv-val %}
+
+; This file generated from the following command:
+; clang -cc1 -triple spirv-vulkan-compute -x hlsl -emit-llvm -finclude-default-header -o - - <<EOF
+; [numthreads(1, 1, 1)]
+; void main() {
+;   int idx = WaveGetLaneIndex();
+; }
+; EOF
+
+; CHECK-DAG:                         OpCapability Shader
+; CHECK-DAG:                         OpCapability GroupNonUniform
+; CHECK-DAG:                         OpDecorate %[[#var:]] BuiltIn SubgroupLocalInvocationId
+; CHECK-DAG:            %[[#int:]] = OpTypeInt 32 0
+; CHECK-DAG:           %[[#ptri:]] = OpTypePointer Input %[[#int]]
+; CHECK-DAG:           %[[#ptrf:]] = OpTypePointer Function %[[#int]]
+; CHECK-DAG:             %[[#var]] = OpVariable %[[#ptri]] Input
+
+; CHECK-NOT:                         OpDecorate %[[#var]] LinkageAttributes
+
+
+; ModuleID = '-'
+source_filename = "-"
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+target triple = "spirv-unknown-vulkan-compute"
+
+; Function Attrs: convergent noinline norecurse nounwind optnone
+define internal spir_func void @main() #0 {
+entry:
+  %0 = call token @llvm.experimental.convergence.entry()
+  %idx = alloca i32, align 4
+; CHECK:     %[[#idx:]] = OpVariable %[[#ptrf]] Function
+
+  %1 = call i32 @__hlsl_wave_get_lane_index() [ "convergencectrl"(token %0) ]
+; CHECK:    %[[#tmp:]] = OpLoad %[[#int]] %[[#var]]
+
+  store i32 %1, ptr %idx, align 4
+; CHECK:                 OpStore %[[#idx]] %[[#tmp]]
+
+  ret void
+}
+
+; Function Attrs: norecurse
+define void @main.1() #1 {
+entry:
+  call void @main()
+  ret void
+}
+
+; Function Attrs: convergent
+declare i32 @__hlsl_wave_get_lane_index() #2
+
+; Function Attrs: convergent nocallback nofree nosync nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.entry() #3
+
+attributes #0 = { convergent noinline norecurse nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { norecurse "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #2 = { convergent }
+attributes #3 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"dx.disable_optimizations", i32 1}
+!2 = !{!"clang version 19.0.0git (/usr/local/google/home/nathangauer/projects/llvm-project/clang bc6fd04b73a195981ee77823cf1382d04ab96c44)"}
+
diff --git a/llvm/test/CodeGen/SPIRV/scfg-add-pre-headers.ll b/llvm/test/CodeGen/SPIRV/scfg-add-pre-headers.ll
index 329399bab3e5..2ea5c767730e 100644
--- a/llvm/test/CodeGen/SPIRV/scfg-add-pre-headers.ll
+++ b/llvm/test/CodeGen/SPIRV/scfg-add-pre-headers.ll
@@ -1,5 +1,6 @@
 ; RUN: llc -mtriple=spirv-unknown-unknown -O0 %s -o - | FileCheck %s
 
+; CHECK-DAG:                  OpDecorate %[[#SubgroupLocalInvocationId:]] BuiltIn SubgroupLocalInvocationId
 ; CHECK-DAG:    %[[#bool:]] = OpTypeBool
 ; CHECK-DAG:    %[[#uint:]] = OpTypeInt 32 0
 ; CHECK-DAG:  %[[#uint_0:]] = OpConstant %[[#uint]] 0
@@ -37,10 +38,10 @@ l1_continue:
 ; CHECK-NEXT:                      OpBranch %[[#l1_header]]
 
 l1_end:
-  %call = call spir_func i32 @_Z3absi(i32 0) [ "convergencectrl"(token %tl1) ]
+  %call = call i32 @__hlsl_wave_get_lane_index() [ "convergencectrl"(token %tl1) ]
   br label %end
 ; CHECK-DAG:   %[[#l1_end]] = OpLabel
-; CHECK-DAG:         %[[#]] = OpFunctionCall
+; CHECK-DAG:         %[[#]] = OpLoad %[[#]] %[[#SubgroupLocalInvocationId]]
 ; CHECK-NEXT:                 OpBranch %[[#end:]]
 
 l2:
@@ -76,6 +77,4 @@ declare token @llvm.experimental.convergence.entry()
 declare token @llvm.experimental.convergence.control()
 declare token @llvm.experimental.convergence.loop()
 
-; This intrinsic is not convergent. This is only because the backend doesn't
-; support convergent operations yet.
-declare spir_func i32 @_Z3absi(i32) convergent
+declare i32 @__hlsl_wave_get_lane_index() convergent
diff --git a/llvm/test/CodeGen/SPIRV/transcoding/spirv-private-array-initialization.ll b/llvm/test/CodeGen/SPIRV/transcoding/spirv-private-array-initialization.ll
index 3551030843d0..e0172ec3c1bd 100644
--- a/llvm/test/CodeGen/SPIRV/transcoding/spirv-private-array-initialization.ll
+++ b/llvm/test/CodeGen/SPIRV/transcoding/spirv-private-array-initialization.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
 ;
-; CHECK-SPIRV-DAG: %[[#i8:]] = OpTypeInt 8 0
 ; CHECK-SPIRV-DAG: %[[#i32:]] = OpTypeInt 32 0
 ; CHECK-SPIRV-DAG: %[[#one:]] = OpConstant %[[#i32]] 1
 ; CHECK-SPIRV-DAG: %[[#two:]] = OpConstant %[[#i32]] 2
@@ -13,7 +12,6 @@
 ; CHECK-SPIRV:     %[[#test_arr2:]] = OpVariable %[[#const_i32x3_ptr]] UniformConstant %[[#test_arr_init]]
 ; CHECK-SPIRV:     %[[#test_arr:]] = OpVariable %[[#const_i32x3_ptr]] UniformConstant %[[#test_arr_init]]
 
-; CHECK-SPIRV-DAG: %[[#const_i8_ptr:]] = OpTypePointer UniformConstant %[[#i8]]
 ; CHECK-SPIRV-DAG: %[[#i32x3_ptr:]] = OpTypePointer Function %[[#i32x3]]
 
 ; CHECK-SPIRV:     %[[#arr:]] = OpVariable %[[#i32x3_ptr]] Function
-- 
GitLab


From a0e9a8da45402eea0db71976ab78d195a7287222 Mon Sep 17 00:00:00 2001
From: jeanPerier <jperier@nvidia.com>
Date: Mon, 25 Mar 2024 11:31:51 +0100
Subject: [PATCH 108/404] [flang][NFC] speedup BoxedProcedure for derived types
 with many components (#86144)

This patch speeds up the compilation time of the example in
https://github.com/llvm/llvm-project/issues/76478#issuecomment-2011023289
from 2 minutes with my builds to about 2 seconds.

MLIR timers showed more than 98% of the time was spend in BoxedProcedure
trying to figure out if a type needs to be converted.

This is because walking the fir.type members is very expansive for types
containing many components and/or components with many sub-components.

Increase the caching time of visited types from "the type being visited"
to "the whole pass". Use DenseMap since it is not ok anymore to assume
this container will only have a few elements.
---
 .../lib/Optimizer/CodeGen/BoxedProcedure.cpp  | 32 +++++++++++++------
 flang/test/Fir/boxproc-2.fir                  |  7 ++++
 2 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp b/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
index 746c275f37ea..dccb642d5f79 100644
--- a/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
+++ b/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
@@ -69,17 +69,32 @@ public:
       return false;
     }
     if (auto recTy = ty.dyn_cast<RecordType>()) {
-      if (llvm::is_contained(visitedTypes, recTy))
-        return false;
+      auto visited = visitedTypes.find(ty);
+      if (visited != visitedTypes.end())
+        return visited->second;
+      [[maybe_unused]] auto newIt = visitedTypes.try_emplace(ty, false);
+      assert(newIt.second && "expected ty to not be in the map");
+      bool wasAlreadyVisitingRecordType = needConversionIsVisitingRecordType;
+      needConversionIsVisitingRecordType = true;
       bool result = false;
-      visitedTypes.push_back(recTy);
       for (auto t : recTy.getTypeList()) {
         if (needsConversion(t.second)) {
           result = true;
           break;
         }
       }
-      visitedTypes.pop_back();
+      // Only keep the result cached if the fir.type visited was a "top-level
+      // type". Nested types with a recursive reference to the "top-level type"
+      // may incorrectly have been resolved as not needed conversions because it
+      // had not been determined yet if the "top-level type" needed conversion.
+      // This is not an issue to determine the "top-level type" need of
+      // conversion, but the result should not be kept and later used in other
+      // contexts.
+      needConversionIsVisitingRecordType = wasAlreadyVisitingRecordType;
+      if (needConversionIsVisitingRecordType)
+        visitedTypes.erase(ty);
+      else
+        visitedTypes.find(ty)->second = result;
       return result;
     }
     if (auto boxTy = ty.dyn_cast<BaseBoxType>())
@@ -140,9 +155,7 @@ public:
       if (rec.isFinalized())
         return rec;
       auto it = convertedTypes.try_emplace(ty, rec);
-      if (!it.second) {
-        llvm::errs() << "failed\n" << ty << "\n";
-      }
+      assert(it.second && "expected ty to not be in the map");
       std::vector<RecordType::TypePair> ps = ty.getLenParamList();
       std::vector<RecordType::TypePair> cs;
       for (auto t : ty.getTypeList()) {
@@ -171,11 +184,12 @@ public:
   void setLocation(mlir::Location location) { loc = location; }
 
 private:
-  llvm::SmallVector<mlir::Type> visitedTypes;
-  // Map to deal with recursive derived types (avoid infinite loops).
+  // Maps to deal with recursive derived types (avoid infinite loops).
   // Caching is also beneficial for apps with big types (dozens of
   // components and or parent types), so the lifetime of the cache
   // is the whole pass.
+  llvm::DenseMap<mlir::Type, bool> visitedTypes;
+  bool needConversionIsVisitingRecordType = false;
   llvm::DenseMap<mlir::Type, mlir::Type> convertedTypes;
   mlir::Location loc;
 };
diff --git a/flang/test/Fir/boxproc-2.fir b/flang/test/Fir/boxproc-2.fir
index 3279bb9bed8a..84132f89afeb 100644
--- a/flang/test/Fir/boxproc-2.fir
+++ b/flang/test/Fir/boxproc-2.fir
@@ -108,3 +108,10 @@ func.func @very_nested_type(%arg0: !fir.type<t0{t01:!fir.type<t01{t02:!fir.type<
 }
 // CHECK-LABEL: func.func @very_nested_type(
 // CHECK-SAME: !fir.type<t0UnboxProc{t01:!fir.type<t01UnboxProc{t02:!fir.type<t02UnboxProc{t3:!fir.type<t3UnboxProc{t4:!fir.type<t4UnboxProc{t5:!fir.type<t5UnboxProc{t6:!fir.type<t6UnboxProc{t7:!fir.type<t7UnboxProc{t8:!fir.type<t8UnboxProc{t9:!fir.type<t9UnboxProc{t10:!fir.type<t10UnboxProc{t11:!fir.type<t11UnboxProc{t12:!fir.type<t12UnboxProc{t13:!fir.type<t13UnboxProc{t14:!fir.type<t14UnboxProc{t15:!fir.type<t15UnboxProc{t16:!fir.type<t16UnboxProc{t17:!fir.type<t17UnboxProc{t18:!fir.type<t18UnboxProc{t19:!fir.type<t19UnboxProc{b:() -> ()}>}>}>}>}>}>}>}>}>}>}>}>}>}>}>}>}>}>}>}>)
+
+func.func @test_indirect_type_recursion() {
+    %0 = fir.zero_bits !fir.ptr<!fir.type<t001{x:!fir.ptr<!fir.type<t002{x:!fir.ptr<!fir.type<t001>>}>>,p:!fir.boxproc<() -> ()>}>>
+  return
+}
+// CHECK-LABEL: func.func @test_indirect_type_recursion(
+// CHECK: fir.zero_bits !fir.ptr<!fir.type<t001UnboxProc{x:!fir.ptr<!fir.type<t002UnboxProc{x:!fir.ptr<!fir.type<t001UnboxProc>>}>>,p:() -> ()}>>
-- 
GitLab


From f5a067bb9002afeedfd6d94722af1cef698979e3 Mon Sep 17 00:00:00 2001
From: AtariDreams <83477269+AtariDreams@users.noreply.github.com>
Date: Mon, 25 Mar 2024 06:35:28 -0400
Subject: [PATCH 109/404] [SelectionDAG]: Deduce KnownNeverZero from SMIN and
 SMAX (#85722)

---
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |  40 ++-
 llvm/test/CodeGen/X86/known-never-zero.ll     | 303 ++++++++++++------
 2 files changed, 250 insertions(+), 93 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 0ab5142ab816..e8d1ac1d3a91 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -5362,10 +5362,44 @@ bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const {
     return isKnownNeverZero(Op.getOperand(1), Depth + 1) ||
            isKnownNeverZero(Op.getOperand(0), Depth + 1);
 
-    // TODO for smin/smax: If either operand is known negative/positive
+    // For smin/smax: If either operand is known negative/positive
     // respectively we don't need the other to be known at all.
-  case ISD::SMAX:
-  case ISD::SMIN:
+  case ISD::SMAX: {
+    KnownBits Op1 = computeKnownBits(Op.getOperand(1), Depth + 1);
+    if (Op1.isStrictlyPositive())
+      return true;
+
+    KnownBits Op0 = computeKnownBits(Op.getOperand(0), Depth + 1);
+    if (Op0.isStrictlyPositive())
+      return true;
+
+    if (Op1.isNonZero() && Op0.isNonZero())
+      return true;
+
+    if (KnownBits::smax(Op0, Op1).isNonZero())
+      return true;
+
+    return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
+           isKnownNeverZero(Op.getOperand(0), Depth + 1);
+  }
+  case ISD::SMIN: {
+    KnownBits Op1 = computeKnownBits(Op.getOperand(1), Depth + 1);
+    if (Op1.isNegative())
+      return true;
+
+    KnownBits Op0 = computeKnownBits(Op.getOperand(0), Depth + 1);
+    if (Op0.isNegative())
+      return true;
+
+    if (Op1.isNonZero() && Op0.isNonZero())
+      return true;
+
+    if (KnownBits::smin(Op0, Op1).isNonZero())
+      return true;
+
+    return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
+           isKnownNeverZero(Op.getOperand(0), Depth + 1);
+  }
   case ISD::UMIN:
     return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
            isKnownNeverZero(Op.getOperand(0), Depth + 1);
diff --git a/llvm/test/CodeGen/X86/known-never-zero.ll b/llvm/test/CodeGen/X86/known-never-zero.ll
index 423516bc3271..39d02f9112f4 100644
--- a/llvm/test/CodeGen/X86/known-never-zero.ll
+++ b/llvm/test/CodeGen/X86/known-never-zero.ll
@@ -10,10 +10,13 @@ declare i32 @llvm.uadd.sat.i32(i32, i32)
 declare i32 @llvm.umax.i32(i32, i32)
 declare i32 @llvm.umin.i32(i32, i32)
 declare i32 @llvm.smin.i32(i32, i32)
+declare <4 x i32> @llvm.smin.v4i32(<4 x i32>, <4 x i32>)
 declare i32 @llvm.smax.i32(i32, i32)
+declare <4 x i32> @llvm.smax.v4i32(<4 x i32>, <4 x i32>)
 declare i32 @llvm.bswap.i32(i32)
 declare i32 @llvm.bitreverse.i32(i32)
 declare i32 @llvm.ctpop.i32(i32)
+declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)
 declare i32 @llvm.abs.i32(i32, i1)
 declare i32 @llvm.fshl.i32(i32, i32, i32)
 declare i32 @llvm.fshr.i32(i32, i32, i32)
@@ -442,6 +445,62 @@ define i32 @smin_known_nonzero(i32 %xx, i32 %yy) {
   ret i32 %r
 }
 
+define i32 @smin_known_zero(i32 %x, i32 %y) {
+; X86-LABEL: smin_known_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmpl $-54, %eax
+; X86-NEXT:    movl $-54, %ecx
+; X86-NEXT:    cmovll %eax, %ecx
+; X86-NEXT:    rep bsfl %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: smin_known_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    cmpl $-54, %edi
+; X64-NEXT:    movl $-54, %eax
+; X64-NEXT:    cmovll %edi, %eax
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
+  %z = call i32 @llvm.smin.i32(i32 %x, i32 -54)
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
+define <4 x i32> @smin_known_zero_vec(<4 x i32> %x, <4 x i32> %y) {
+; X86-LABEL: smin_known_zero_vec:
+; X86:       # %bb.0:
+; X86-NEXT:    movdqa {{.*#+}} xmm1 = [4294967242,4294967273,4294967284,4294967295]
+; X86-NEXT:    movdqa %xmm1, %xmm2
+; X86-NEXT:    pcmpgtd %xmm0, %xmm2
+; X86-NEXT:    pand %xmm2, %xmm0
+; X86-NEXT:    pandn %xmm1, %xmm2
+; X86-NEXT:    por %xmm2, %xmm0
+; X86-NEXT:    pcmpeqd %xmm1, %xmm1
+; X86-NEXT:    paddd %xmm0, %xmm1
+; X86-NEXT:    pand %xmm1, %xmm0
+; X86-NEXT:    pxor %xmm1, %xmm1
+; X86-NEXT:    pcmpeqd %xmm1, %xmm0
+; X86-NEXT:    psrld $31, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: smin_known_zero_vec:
+; X64:       # %bb.0:
+; X64-NEXT:    vpminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; X64-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; X64-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; X64-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vpsrld $31, %xmm0, %xmm0
+; X64-NEXT:    retq
+  %z = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %x, <4 x i32> <i32 -54, i32 -23, i32 -12, i32 -1>)
+  %r = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %z)
+  %3 = icmp eq <4 x i32> %r, <i32 1, i32 1, i32 1, i32 1>
+  %ret = zext <4 x i1> %3 to <4 x i32>
+  ret <4 x i32> %ret
+}
+
 define i32 @smin_maybe_zero(i32 %x, i32 %y) {
 ; X86-LABEL: smin_maybe_zero:
 ; X86:       # %bb.0:
@@ -450,11 +509,11 @@ define i32 @smin_maybe_zero(i32 %x, i32 %y) {
 ; X86-NEXT:    movl $54, %eax
 ; X86-NEXT:    cmovll %ecx, %eax
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    je .LBB15_1
+; X86-NEXT:    je .LBB17_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB15_1:
+; X86-NEXT:  .LBB17_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
@@ -464,11 +523,11 @@ define i32 @smin_maybe_zero(i32 %x, i32 %y) {
 ; X64-NEXT:    movl $54, %eax
 ; X64-NEXT:    cmovll %edi, %eax
 ; X64-NEXT:    testl %eax, %eax
-; X64-NEXT:    je .LBB15_1
+; X64-NEXT:    je .LBB17_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    rep bsfl %eax, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB15_1:
+; X64-NEXT:  .LBB17_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %z = call i32 @llvm.smin.i32(i32 %x, i32 54)
@@ -514,9 +573,7 @@ define i32 @smax_maybe_zero(i32 %x, i32 %y) {
 ; X86-NEXT:    cmpl $55, %eax
 ; X86-NEXT:    movl $54, %ecx
 ; X86-NEXT:    cmovgel %eax, %ecx
-; X86-NEXT:    bsfl %ecx, %ecx
-; X86-NEXT:    movl $32, %eax
-; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    rep bsfl %ecx, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: smax_maybe_zero:
@@ -524,15 +581,81 @@ define i32 @smax_maybe_zero(i32 %x, i32 %y) {
 ; X64-NEXT:    cmpl $55, %edi
 ; X64-NEXT:    movl $54, %eax
 ; X64-NEXT:    cmovgel %edi, %eax
-; X64-NEXT:    bsfl %eax, %ecx
-; X64-NEXT:    movl $32, %eax
-; X64-NEXT:    cmovnel %ecx, %eax
+; X64-NEXT:    rep bsfl %eax, %eax
 ; X64-NEXT:    retq
   %z = call i32 @llvm.smax.i32(i32 %x, i32 54)
   %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
   ret i32 %r
 }
 
+define <4 x i32> @smax_known_zero_vec(<4 x i32> %x, <4 x i32> %y) {
+; X86-LABEL: smax_known_zero_vec:
+; X86:       # %bb.0:
+; X86-NEXT:    movdqa {{.*#+}} xmm1 = [54,23,12,1]
+; X86-NEXT:    movdqa %xmm0, %xmm2
+; X86-NEXT:    pcmpgtd %xmm1, %xmm2
+; X86-NEXT:    pand %xmm2, %xmm0
+; X86-NEXT:    pandn %xmm1, %xmm2
+; X86-NEXT:    por %xmm2, %xmm0
+; X86-NEXT:    pcmpeqd %xmm1, %xmm1
+; X86-NEXT:    paddd %xmm0, %xmm1
+; X86-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-NEXT:    pxor %xmm1, %xmm0
+; X86-NEXT:    pcmpgtd %xmm1, %xmm0
+; X86-NEXT:    psrld $31, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: smax_known_zero_vec:
+; X64:       # %bb.0:
+; X64-NEXT:    vpmaxsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; X64-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; X64-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vpminud %xmm1, %xmm0, %xmm1
+; X64-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-NEXT:    retq
+  %z = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %x, <4 x i32> <i32 54, i32 23, i32 12, i32 1>)
+  %r = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %z)
+  %3 = icmp eq <4 x i32> %r, <i32 1, i32 1, i32 1, i32 1>
+  %ret = zext <4 x i1> %3 to <4 x i32>
+  ret <4 x i32> %ret
+}
+
+define i32 @smax_known_zero(i32 %x, i32 %y) {
+; X86-LABEL: smax_known_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    testl %ecx, %ecx
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovnsl %ecx, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    je .LBB21_1
+; X86-NEXT:  # %bb.2: # %cond.false
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB21_1:
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: smax_known_zero:
+; X64:       # %bb.0:
+; X64-NEXT:    testl %edi, %edi
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovnsl %edi, %eax
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    je .LBB21_1
+; X64-NEXT:  # %bb.2: # %cond.false
+; X64-NEXT:    rep bsfl %eax, %eax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB21_1:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    retq
+  %z = call i32 @llvm.smax.i32(i32 %x, i32 -1)
+  %r = call i32 @llvm.cttz.i32(i32 %z, i1 false)
+  ret i32 %r
+}
+
 define i32 @rotr_known_nonzero(i32 %xx, i32 %y) {
 ; X86-LABEL: rotr_known_nonzero:
 ; X86:       # %bb.0:
@@ -541,11 +664,11 @@ define i32 @rotr_known_nonzero(i32 %xx, i32 %y) {
 ; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    rorl %cl, %eax
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    je .LBB18_1
+; X86-NEXT:    je .LBB22_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB18_1:
+; X86-NEXT:  .LBB22_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
@@ -556,11 +679,11 @@ define i32 @rotr_known_nonzero(i32 %xx, i32 %y) {
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    rorl %cl, %edi
 ; X64-NEXT:    testl %edi, %edi
-; X64-NEXT:    je .LBB18_1
+; X64-NEXT:    je .LBB22_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    rep bsfl %edi, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB18_1:
+; X64-NEXT:  .LBB22_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %x = or i32 %xx, 256
@@ -579,11 +702,11 @@ define i32 @rotr_maybe_zero(i32 %x, i32 %y) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    rorl %cl, %eax
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    je .LBB19_1
+; X86-NEXT:    je .LBB23_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB19_1:
+; X86-NEXT:  .LBB23_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
@@ -593,11 +716,11 @@ define i32 @rotr_maybe_zero(i32 %x, i32 %y) {
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    rorl %cl, %edi
 ; X64-NEXT:    testl %edi, %edi
-; X64-NEXT:    je .LBB19_1
+; X64-NEXT:    je .LBB23_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    rep bsfl %edi, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB19_1:
+; X64-NEXT:  .LBB23_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %shr = lshr i32 %x, %y
@@ -639,11 +762,11 @@ define i32 @rotr_with_fshr_maybe_zero(i32 %x, i32 %y) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    rorl %cl, %eax
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    je .LBB21_1
+; X86-NEXT:    je .LBB25_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB21_1:
+; X86-NEXT:  .LBB25_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
@@ -653,11 +776,11 @@ define i32 @rotr_with_fshr_maybe_zero(i32 %x, i32 %y) {
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    rorl %cl, %edi
 ; X64-NEXT:    testl %edi, %edi
-; X64-NEXT:    je .LBB21_1
+; X64-NEXT:    je .LBB25_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    rep bsfl %edi, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB21_1:
+; X64-NEXT:  .LBB25_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %z = call i32 @llvm.fshr.i32(i32 %x, i32 %x, i32 %y)
@@ -673,11 +796,11 @@ define i32 @rotl_known_nonzero(i32 %xx, i32 %y) {
 ; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    roll %cl, %eax
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    je .LBB22_1
+; X86-NEXT:    je .LBB26_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB22_1:
+; X86-NEXT:  .LBB26_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
@@ -688,11 +811,11 @@ define i32 @rotl_known_nonzero(i32 %xx, i32 %y) {
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    roll %cl, %edi
 ; X64-NEXT:    testl %edi, %edi
-; X64-NEXT:    je .LBB22_1
+; X64-NEXT:    je .LBB26_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    rep bsfl %edi, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB22_1:
+; X64-NEXT:  .LBB26_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %x = or i32 %xx, 256
@@ -711,11 +834,11 @@ define i32 @rotl_maybe_zero(i32 %x, i32 %y) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    roll %cl, %eax
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    je .LBB23_1
+; X86-NEXT:    je .LBB27_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB23_1:
+; X86-NEXT:  .LBB27_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
@@ -725,11 +848,11 @@ define i32 @rotl_maybe_zero(i32 %x, i32 %y) {
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    roll %cl, %edi
 ; X64-NEXT:    testl %edi, %edi
-; X64-NEXT:    je .LBB23_1
+; X64-NEXT:    je .LBB27_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    rep bsfl %edi, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB23_1:
+; X64-NEXT:  .LBB27_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %shl = shl i32 %x, %y
@@ -771,11 +894,11 @@ define i32 @rotl_with_fshl_maybe_zero(i32 %x, i32 %y) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    roll %cl, %eax
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    je .LBB25_1
+; X86-NEXT:    je .LBB29_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB25_1:
+; X86-NEXT:  .LBB29_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
@@ -785,11 +908,11 @@ define i32 @rotl_with_fshl_maybe_zero(i32 %x, i32 %y) {
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    roll %cl, %edi
 ; X64-NEXT:    testl %edi, %edi
-; X64-NEXT:    je .LBB25_1
+; X64-NEXT:    je .LBB29_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    rep bsfl %edi, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB25_1:
+; X64-NEXT:  .LBB29_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %z = call i32 @llvm.fshl.i32(i32 %x, i32 %x, i32 %y)
@@ -850,11 +973,11 @@ define i32 @sra_maybe_zero(i32 %x, i32 %y) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    sarl %cl, %eax
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    je .LBB28_1
+; X86-NEXT:    je .LBB32_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB28_1:
+; X86-NEXT:  .LBB32_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
@@ -864,11 +987,11 @@ define i32 @sra_maybe_zero(i32 %x, i32 %y) {
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    sarl %cl, %esi
 ; X64-NEXT:    testl %esi, %esi
-; X64-NEXT:    je .LBB28_1
+; X64-NEXT:    je .LBB32_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    rep bsfl %esi, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB28_1:
+; X64-NEXT:  .LBB32_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %z = ashr exact i32 %y, %x
@@ -929,11 +1052,11 @@ define i32 @srl_maybe_zero(i32 %x, i32 %y) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    shrl %cl, %eax
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    je .LBB31_1
+; X86-NEXT:    je .LBB35_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB31_1:
+; X86-NEXT:  .LBB35_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
@@ -943,11 +1066,11 @@ define i32 @srl_maybe_zero(i32 %x, i32 %y) {
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shrl %cl, %esi
 ; X64-NEXT:    testl %esi, %esi
-; X64-NEXT:    je .LBB31_1
+; X64-NEXT:    je .LBB35_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    rep bsfl %esi, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB31_1:
+; X64-NEXT:  .LBB35_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %z = lshr exact i32 %y, %x
@@ -986,11 +1109,11 @@ define i32 @udiv_maybe_zero(i32 %x, i32 %y) {
 ; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    divl {{[0-9]+}}(%esp)
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    je .LBB33_1
+; X86-NEXT:    je .LBB37_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB33_1:
+; X86-NEXT:  .LBB37_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
@@ -1000,11 +1123,11 @@ define i32 @udiv_maybe_zero(i32 %x, i32 %y) {
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    divl %esi
 ; X64-NEXT:    testl %eax, %eax
-; X64-NEXT:    je .LBB33_1
+; X64-NEXT:    je .LBB37_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    rep bsfl %eax, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB33_1:
+; X64-NEXT:  .LBB37_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %z = udiv exact i32 %x, %y
@@ -1043,11 +1166,11 @@ define i32 @sdiv_maybe_zero(i32 %x, i32 %y) {
 ; X86-NEXT:    cltd
 ; X86-NEXT:    idivl {{[0-9]+}}(%esp)
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    je .LBB35_1
+; X86-NEXT:    je .LBB39_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB35_1:
+; X86-NEXT:  .LBB39_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
@@ -1057,11 +1180,11 @@ define i32 @sdiv_maybe_zero(i32 %x, i32 %y) {
 ; X64-NEXT:    cltd
 ; X64-NEXT:    idivl %esi
 ; X64-NEXT:    testl %eax, %eax
-; X64-NEXT:    je .LBB35_1
+; X64-NEXT:    je .LBB39_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    rep bsfl %eax, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB35_1:
+; X64-NEXT:  .LBB39_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %z = sdiv exact i32 %x, %y
@@ -1096,11 +1219,11 @@ define i32 @add_maybe_zero(i32 %xx, i32 %y) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    orl $1, %eax
 ; X86-NEXT:    addl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    je .LBB37_1
+; X86-NEXT:    je .LBB41_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB37_1:
+; X86-NEXT:  .LBB41_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
@@ -1108,11 +1231,11 @@ define i32 @add_maybe_zero(i32 %xx, i32 %y) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    orl $1, %edi
 ; X64-NEXT:    addl %esi, %edi
-; X64-NEXT:    je .LBB37_1
+; X64-NEXT:    je .LBB41_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    rep bsfl %edi, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB37_1:
+; X64-NEXT:  .LBB41_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %x = or i32 %xx, 1
@@ -1179,11 +1302,11 @@ define i32 @sub_maybe_zero(i32 %x) {
 ; X86-NEXT:    movl %ecx, %eax
 ; X86-NEXT:    orl $64, %eax
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    je .LBB40_1
+; X86-NEXT:    je .LBB44_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB40_1:
+; X86-NEXT:  .LBB44_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
@@ -1192,11 +1315,11 @@ define i32 @sub_maybe_zero(i32 %x) {
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    orl $64, %eax
 ; X64-NEXT:    subl %edi, %eax
-; X64-NEXT:    je .LBB40_1
+; X64-NEXT:    je .LBB44_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    rep bsfl %eax, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB40_1:
+; X64-NEXT:  .LBB44_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %y = or i32 %x, 64
@@ -1210,22 +1333,22 @@ define i32 @sub_maybe_zero2(i32 %x) {
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    negl %eax
-; X86-NEXT:    je .LBB41_1
+; X86-NEXT:    je .LBB45_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB41_1:
+; X86-NEXT:  .LBB45_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: sub_maybe_zero2:
 ; X64:       # %bb.0:
 ; X64-NEXT:    negl %edi
-; X64-NEXT:    je .LBB41_1
+; X64-NEXT:    je .LBB45_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    rep bsfl %edi, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB41_1:
+; X64-NEXT:  .LBB45_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %z = sub i32 0, %x
@@ -1240,11 +1363,11 @@ define i32 @mul_known_nonzero_nsw(i32 %x, i32 %yy) {
 ; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    imull {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    je .LBB42_1
+; X86-NEXT:    je .LBB46_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB42_1:
+; X86-NEXT:  .LBB46_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
@@ -1253,11 +1376,11 @@ define i32 @mul_known_nonzero_nsw(i32 %x, i32 %yy) {
 ; X64-NEXT:    orl $256, %esi # imm = 0x100
 ; X64-NEXT:    imull %edi, %esi
 ; X64-NEXT:    testl %esi, %esi
-; X64-NEXT:    je .LBB42_1
+; X64-NEXT:    je .LBB46_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    rep bsfl %esi, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB42_1:
+; X64-NEXT:  .LBB46_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %y = or i32 %yy, 256
@@ -1273,11 +1396,11 @@ define i32 @mul_known_nonzero_nuw(i32 %x, i32 %yy) {
 ; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    imull {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    je .LBB43_1
+; X86-NEXT:    je .LBB47_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB43_1:
+; X86-NEXT:  .LBB47_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
@@ -1286,11 +1409,11 @@ define i32 @mul_known_nonzero_nuw(i32 %x, i32 %yy) {
 ; X64-NEXT:    orl $256, %esi # imm = 0x100
 ; X64-NEXT:    imull %edi, %esi
 ; X64-NEXT:    testl %esi, %esi
-; X64-NEXT:    je .LBB43_1
+; X64-NEXT:    je .LBB47_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    rep bsfl %esi, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB43_1:
+; X64-NEXT:  .LBB47_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %y = or i32 %yy, 256
@@ -1305,11 +1428,11 @@ define i32 @mul_maybe_zero(i32 %x, i32 %y) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    imull {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    je .LBB44_1
+; X86-NEXT:    je .LBB48_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB44_1:
+; X86-NEXT:  .LBB48_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
@@ -1317,11 +1440,11 @@ define i32 @mul_maybe_zero(i32 %x, i32 %y) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    imull %esi, %edi
 ; X64-NEXT:    testl %edi, %edi
-; X64-NEXT:    je .LBB44_1
+; X64-NEXT:    je .LBB48_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    rep bsfl %edi, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB44_1:
+; X64-NEXT:  .LBB48_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %z = mul nuw nsw i32 %y, %x
@@ -1368,11 +1491,11 @@ define i32 @bitcast_maybe_zero(<2 x i16> %x) {
 ; X86:       # %bb.0:
 ; X86-NEXT:    movd %xmm0, %eax
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    je .LBB46_1
+; X86-NEXT:    je .LBB50_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB46_1:
+; X86-NEXT:  .LBB50_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
@@ -1380,11 +1503,11 @@ define i32 @bitcast_maybe_zero(<2 x i16> %x) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovd %xmm0, %eax
 ; X64-NEXT:    testl %eax, %eax
-; X64-NEXT:    je .LBB46_1
+; X64-NEXT:    je .LBB50_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    rep bsfl %eax, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB46_1:
+; X64-NEXT:  .LBB50_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %z = bitcast <2 x i16> %x to i32
@@ -1398,11 +1521,11 @@ define i32 @bitcast_from_float(float %x) {
 ; X86-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-NEXT:    movd %xmm0, %eax
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    je .LBB47_1
+; X86-NEXT:    je .LBB51_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB47_1:
+; X86-NEXT:  .LBB51_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
@@ -1410,11 +1533,11 @@ define i32 @bitcast_from_float(float %x) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovd %xmm0, %eax
 ; X64-NEXT:    testl %eax, %eax
-; X64-NEXT:    je .LBB47_1
+; X64-NEXT:    je .LBB51_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    rep bsfl %eax, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB47_1:
+; X64-NEXT:  .LBB51_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %z = bitcast float %x to i32
@@ -1452,24 +1575,24 @@ define i32 @zext_maybe_zero(i16 %x) {
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    testw %ax, %ax
-; X86-NEXT:    je .LBB49_1
+; X86-NEXT:    je .LBB53_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    movzwl %ax, %eax
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB49_1:
+; X86-NEXT:  .LBB53_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: zext_maybe_zero:
 ; X64:       # %bb.0:
 ; X64-NEXT:    testw %di, %di
-; X64-NEXT:    je .LBB49_1
+; X64-NEXT:    je .LBB53_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    movzwl %di, %eax
 ; X64-NEXT:    rep bsfl %eax, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB49_1:
+; X64-NEXT:  .LBB53_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %z = zext i16 %x to i32
@@ -1507,23 +1630,23 @@ define i32 @sext_maybe_zero(i16 %x) {
 ; X86:       # %bb.0:
 ; X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    je .LBB51_1
+; X86-NEXT:    je .LBB55_1
 ; X86-NEXT:  # %bb.2: # %cond.false
 ; X86-NEXT:    rep bsfl %eax, %eax
 ; X86-NEXT:    retl
-; X86-NEXT:  .LBB51_1:
+; X86-NEXT:  .LBB55_1:
 ; X86-NEXT:    movl $32, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: sext_maybe_zero:
 ; X64:       # %bb.0:
 ; X64-NEXT:    testw %di, %di
-; X64-NEXT:    je .LBB51_1
+; X64-NEXT:    je .LBB55_1
 ; X64-NEXT:  # %bb.2: # %cond.false
 ; X64-NEXT:    movswl %di, %eax
 ; X64-NEXT:    rep bsfl %eax, %eax
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB51_1:
+; X64-NEXT:  .LBB55_1:
 ; X64-NEXT:    movl $32, %eax
 ; X64-NEXT:    retq
   %z = sext i16 %x to i32
-- 
GitLab


From 13078cbc3eeb0ae91c370ce0f604f7165b26e0c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= <stefan.graenitz@gmail.com>
Date: Mon, 25 Mar 2024 11:36:20 +0100
Subject: [PATCH 110/404] [clang-repl] Skip cross-JIT tests if specified target
 is not available (#84461)

---
 .../Interpreter/InterpreterExtensionsTest.cpp | 28 +++++++++++++++----
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp
index 8bc429d9ec2d..1ba865a79ed7 100644
--- a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp
+++ b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp
@@ -19,6 +19,7 @@
 
 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
+#include "llvm/MC/TargetRegistry.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/Threading.h"
@@ -44,14 +45,23 @@ static bool HostSupportsJit() {
   return false;
 }
 
+// Some tests require a arm-registered-target
+static bool IsARMTargetRegistered() {
+  llvm::Triple TT;
+  TT.setArch(llvm::Triple::arm);
+  TT.setVendor(llvm::Triple::UnknownVendor);
+  TT.setOS(llvm::Triple::UnknownOS);
+
+  std::string UnusedErr;
+  return llvm::TargetRegistry::lookupTarget(TT.str(), UnusedErr);
+}
+
 struct LLVMInitRAII {
   LLVMInitRAII() {
-    llvm::InitializeNativeTarget();
-    llvm::InitializeNativeTargetAsmPrinter();
-    LLVMInitializeARMTarget();
-    LLVMInitializeARMTargetInfo();
-    LLVMInitializeARMTargetMC();
-    LLVMInitializeARMAsmPrinter();
+    llvm::InitializeAllTargets();
+    llvm::InitializeAllTargetInfos();
+    llvm::InitializeAllTargetMCs();
+    llvm::InitializeAllAsmPrinters();
   }
   ~LLVMInitRAII() { llvm::llvm_shutdown(); }
 } LLVMInit;
@@ -190,6 +200,9 @@ TEST(InterpreterExtensionsTest, DISABLED_DefaultCrossJIT) {
 #else
 TEST(InterpreterExtensionsTest, DefaultCrossJIT) {
 #endif
+  if (!IsARMTargetRegistered())
+    GTEST_SKIP();
+
   IncrementalCompilerBuilder CB;
   CB.SetTargetTriple("armv6-none-eabi");
   auto CI = cantFail(CB.CreateCpp());
@@ -204,6 +217,9 @@ TEST(InterpreterExtensionsTest, DISABLED_CustomCrossJIT) {
 #else
 TEST(InterpreterExtensionsTest, CustomCrossJIT) {
 #endif
+  if (!IsARMTargetRegistered())
+    GTEST_SKIP();
+
   std::string TargetTriple = "armv6-none-eabi";
 
   IncrementalCompilerBuilder CB;
-- 
GitLab


From aeeb7d566c9047d4e64b5fc7ea8930b93e80def9 Mon Sep 17 00:00:00 2001
From: Tobias Gysi <tobias.gysi@nextsilicon.com>
Date: Mon, 25 Mar 2024 12:19:22 +0100
Subject: [PATCH 111/404] [MLIR][LLVM] Make subprogram flags optional (#86433)

This revision makes the subprogramFlags field in the DISubprogrammAttr
optional. This is necessary since the DISubprogram attached to a
declaration may have none of the subprogram flags set.
---
 mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td | 2 +-
 mlir/lib/Target/LLVMIR/DebugImporter.cpp         | 8 ++++----
 mlir/test/Dialect/LLVMIR/debuginfo.mlir          | 8 +++-----
 mlir/test/Target/LLVMIR/Import/debug-info.ll     | 3 ++-
 mlir/test/Target/LLVMIR/llvmir-debug.mlir        | 4 ++--
 5 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
index 1b1824a28e99..91bd3702f93b 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
@@ -546,7 +546,7 @@ def LLVM_DISubprogramAttr : LLVM_Attr<"DISubprogram", "di_subprogram",
     "DIFileAttr":$file,
     OptionalParameter<"unsigned">:$line,
     OptionalParameter<"unsigned">:$scopeLine,
-    "DISubprogramFlags":$subprogramFlags,
+    OptionalParameter<"DISubprogramFlags">:$subprogramFlags,
     OptionalParameter<"DISubroutineTypeAttr">:$type
   );
   let builders = [
diff --git a/mlir/lib/Target/LLVMIR/DebugImporter.cpp b/mlir/lib/Target/LLVMIR/DebugImporter.cpp
index 4bdc03a3e282..779ad26fc847 100644
--- a/mlir/lib/Target/LLVMIR/DebugImporter.cpp
+++ b/mlir/lib/Target/LLVMIR/DebugImporter.cpp
@@ -179,12 +179,13 @@ DISubprogramAttr DebugImporter::translateImpl(llvm::DISubprogram *node) {
   mlir::DistinctAttr id;
   if (node->isDistinct())
     id = getOrCreateDistinctID(node);
-  std::optional<DISubprogramFlags> subprogramFlags =
-      symbolizeDISubprogramFlags(node->getSubprogram()->getSPFlags());
   // Return nullptr if the scope or type is invalid.
   DIScopeAttr scope = translate(node->getScope());
   if (node->getScope() && !scope)
     return nullptr;
+  std::optional<DISubprogramFlags> subprogramFlags =
+      symbolizeDISubprogramFlags(node->getSubprogram()->getSPFlags());
+  assert(subprogramFlags && "expected valid subprogram flags");
   DISubroutineTypeAttr type = translate(node->getType());
   if (node->getType() && !type)
     return nullptr;
@@ -192,8 +193,7 @@ DISubprogramAttr DebugImporter::translateImpl(llvm::DISubprogram *node) {
                                getStringAttrOrNull(node->getRawName()),
                                getStringAttrOrNull(node->getRawLinkageName()),
                                translate(node->getFile()), node->getLine(),
-                               node->getScopeLine(), subprogramFlags.value(),
-                               type);
+                               node->getScopeLine(), *subprogramFlags, type);
 }
 
 DISubrangeAttr DebugImporter::translateImpl(llvm::DISubrange *node) {
diff --git a/mlir/test/Dialect/LLVMIR/debuginfo.mlir b/mlir/test/Dialect/LLVMIR/debuginfo.mlir
index 4c2de0aa4c22..94bb2bb06229 100644
--- a/mlir/test/Dialect/LLVMIR/debuginfo.mlir
+++ b/mlir/test/Dialect/LLVMIR/debuginfo.mlir
@@ -96,11 +96,10 @@
   file = #file, line = 3, scopeLine = 3, subprogramFlags = "Definition|Optimized", type = #spType0
 >
 
-// CHECK-DAG: #[[SP1:.*]] = #llvm.di_subprogram<compileUnit = #[[CU]], scope = #[[COMP2]], name = "value", file = #[[FILE]], subprogramFlags = Definition, type = #[[SPTYPE1]]>
+// CHECK-DAG: #[[SP1:.*]] = #llvm.di_subprogram<scope = #[[COMP2]], file = #[[FILE]], type = #[[SPTYPE1]]>
 #sp1 = #llvm.di_subprogram<
-  // Omit the optional linkageName parameter.
-  compileUnit = #cu, scope = #comp2, name = "value",
-  file = #file, subprogramFlags = "Definition", type = #spType1
+  // Omit the optional parameters.
+  scope = #comp2, file = #file, type = #spType1
 >
 
 // CHECK-DAG: #[[MODULE:.*]] = #llvm.di_module<file = #[[FILE]], scope = #[[FILE]], name = "module", configMacros = "bar", includePath = "/", apinotes = "/", line = 42, isDecl = true>
@@ -112,7 +111,6 @@
 
 // CHECK-DAG: #[[SP2:.*]] = #llvm.di_subprogram<compileUnit = #[[CU]], scope = #[[MODULE]], name = "value", file = #[[FILE]], subprogramFlags = Definition, type = #[[SPTYPE2]]>
 #sp2 = #llvm.di_subprogram<
-  // Omit the optional linkageName parameter.
   compileUnit = #cu, scope = #module, name = "value",
   file = #file, subprogramFlags = "Definition", type = #spType2
 >
diff --git a/mlir/test/Target/LLVMIR/Import/debug-info.ll b/mlir/test/Target/LLVMIR/Import/debug-info.ll
index a7947eb0d444..959a5a1cd971 100644
--- a/mlir/test/Target/LLVMIR/Import/debug-info.ll
+++ b/mlir/test/Target/LLVMIR/Import/debug-info.ll
@@ -601,8 +601,9 @@ declare !dbg !1 void @declaration()
 
 ; CHECK: #di_subprogram = #llvm.di_subprogram<
 ; CHECK-NOT: id = distinct
+; CHECK-NOT: subprogramFlags =
 
 !llvm.module.flags = !{!0}
 !0 = !{i32 2, !"Debug Info Version", i32 3}
-!1 = !DISubprogram(name: "declaration", scope: !2, file: !2, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized)
+!1 = !DISubprogram(name: "declaration", scope: !2, file: !2, flags: DIFlagPrototyped, spFlags: 0)
 !2 = !DIFile(filename: "debug-info.ll", directory: "/")
diff --git a/mlir/test/Target/LLVMIR/llvmir-debug.mlir b/mlir/test/Target/LLVMIR/llvmir-debug.mlir
index c34f9187d4df..785a525caab8 100644
--- a/mlir/test/Target/LLVMIR/llvmir-debug.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-debug.mlir
@@ -172,14 +172,14 @@ llvm.func @empty_types() {
 
 #di_file = #llvm.di_file<"foo.mlir" in "/test/">
 #di_subprogram = #llvm.di_subprogram<
-  scope = #di_file, name = "func_decl_with_subprogram", file = #di_file, subprogramFlags = "Optimized"
+  scope = #di_file, name = "func_decl_with_subprogram", file = #di_file
 >
 
 // CHECK-LABEL: declare !dbg
 // CHECK-SAME: ![[SUBPROGRAM:.*]] i32 @func_decl_with_subprogram(
 llvm.func @func_decl_with_subprogram() -> (i32) loc(fused<#di_subprogram>["foo.mlir":2:1])
 
-// CHECK: ![[SUBPROGRAM]] = !DISubprogram(name: "func_decl_with_subprogram", scope: ![[FILE:.*]], file: ![[FILE]], spFlags: DISPFlagOptimized)
+// CHECK: ![[SUBPROGRAM]] = !DISubprogram(name: "func_decl_with_subprogram", scope: ![[FILE:.*]], file: ![[FILE]], spFlags: 0)
 // CHECK: ![[FILE]] = !DIFile(filename: "foo.mlir", directory: "/test/")
 
 // -----
-- 
GitLab


From 4e29c6acd3683973e86d46bc124c78e338f1510a Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Mon, 25 Mar 2024 11:20:56 +0000
Subject: [PATCH 112/404] [AArch64] Correct Neoverse V1 SVE 16-bit sdot/udot
 schedule pipelines. (#86142)

Fixes #86102
---
 .../Target/AArch64/AArch64SchedNeoverseV1.td   |  2 +-
 .../AArch64/Neoverse/V1-sve-instructions.s     | 18 +++++++++---------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td
index c7dfd64b2fb2..f7e6545f0dd3 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td
@@ -1446,7 +1446,7 @@ def : InstRW<[V1Write_3c_1V01], (instregex "^[SU]DOT_ZZZI?_S$")>;
 def : InstRW<[V1Write_3c_1V], (instrs SUDOT_ZZZI, USDOT_ZZZ, USDOT_ZZZI)>;
 
 // Dot product, 16 bit
-def : InstRW<[V1Write_4c_1V01], (instregex "^[SU]DOT_ZZZI?_D$")>;
+def : InstRW<[V1Write_4c_1V0], (instregex "^[SU]DOT_ZZZI?_D$")>;
 
 // Duplicate, immediate and indexed form
 def : InstRW<[V1Write_2c_1V01], (instregex "^DUP_ZI_[BHSD]$",
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-sve-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-sve-instructions.s
index efa81b0ffcd4..f120f5feaf7c 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-sve-instructions.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-sve-instructions.s
@@ -4093,8 +4093,8 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  1      12    7.00                        sdiv	z0.s, p7/m, z0.s, z31.s
 # CHECK-NEXT:  1      20    7.00                        sdivr	z0.d, p7/m, z0.d, z31.d
 # CHECK-NEXT:  1      12    7.00                        sdivr	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  1      4     0.50                        sdot	z0.d, z1.h, z15.h[1]
-# CHECK-NEXT:  1      4     0.50                        sdot	z0.d, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        sdot	z0.d, z1.h, z15.h[1]
+# CHECK-NEXT:  1      4     1.00                        sdot	z0.d, z1.h, z31.h
 # CHECK-NEXT:  1      3     0.50                        sdot	z0.s, z1.b, z31.b
 # CHECK-NEXT:  1      3     0.50                        sdot	z0.s, z1.b, z7.b[3]
 # CHECK-NEXT:  1      2     0.50                        sel	z23.b, p11, z13.b, z8.b
@@ -4569,8 +4569,8 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  1      12    7.00                        udiv	z0.s, p7/m, z0.s, z31.s
 # CHECK-NEXT:  1      20    7.00                        udivr	z0.d, p7/m, z0.d, z31.d
 # CHECK-NEXT:  1      12    7.00                        udivr	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  1      4     0.50                        udot	z0.d, z1.h, z15.h[1]
-# CHECK-NEXT:  1      4     0.50                        udot	z0.d, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        udot	z0.d, z1.h, z15.h[1]
+# CHECK-NEXT:  1      4     1.00                        udot	z0.d, z1.h, z31.h
 # CHECK-NEXT:  1      3     0.50                        udot	z0.s, z1.b, z31.b
 # CHECK-NEXT:  1      3     0.50                        udot	z0.s, z1.b, z7.b[3]
 # CHECK-NEXT:  1      2     0.50                        umax	z0.b, z0.b, #0
@@ -4839,7 +4839,7 @@ zip2	z31.s, z31.s, z31.s
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]
-# CHECK-NEXT:  -      -      -      -     88.67  500.67 500.67 797.50 2.50   92.50  92.50  1250.00 923.00 178.50 181.50
+# CHECK-NEXT:  -      -      -      -     88.67  500.67 500.67 797.50 2.50   92.50  92.50  1252.00 921.00 178.50 181.50
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0.0]  [0.1]  [1.0]  [1.1]  [2]    [3.0]  [3.1]  [4]    [5]    [6.0]  [6.1]  [7]    [8]    [9]    [10]   Instructions:
@@ -6521,8 +6521,8 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     sdiv	z0.s, p7/m, z0.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     sdivr	z0.d, p7/m, z0.d, z31.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     sdivr	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sdot	z0.d, z1.h, z15.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sdot	z0.d, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sdot	z0.d, z1.h, z15.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     sdot	z0.d, z1.h, z31.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sdot	z0.s, z1.b, z31.b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sdot	z0.s, z1.b, z7.b[3]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     sel	z23.b, p11, z13.b, z8.b
@@ -6997,8 +6997,8 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     udiv	z0.s, p7/m, z0.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     udivr	z0.d, p7/m, z0.d, z31.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     7.00    -      -      -     udivr	z0.s, p7/m, z0.s, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     udot	z0.d, z1.h, z15.h[1]
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     udot	z0.d, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     udot	z0.d, z1.h, z15.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     udot	z0.d, z1.h, z31.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     udot	z0.s, z1.b, z31.b
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     udot	z0.s, z1.b, z7.b[3]
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     umax	z0.b, z0.b, #0
-- 
GitLab


From 1081d3a0a7aeee795b11947d850cecc9c63a187a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 25 Mar 2024 11:27:17 +0000
Subject: [PATCH 113/404] [VPlan] Mark CanonicalIVIncrementForPart as only
 using part 0 of IV.

CanonicalIVIncrementForPart uses VPIteration(0, 0) of the IV (first
operand), mark it as only using part 0.

This avoids generating redundant IV increments per part.
---
 llvm/lib/Transforms/Vectorize/VPlan.h             |  1 +
 .../LoopVectorize/AArch64/masked-call.ll          | 15 +++++----------
 .../Transforms/LoopVectorize/AArch64/pr73894.ll   |  3 +--
 3 files changed, 7 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 22173954f7ce..ba4992a7dcc9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1265,6 +1265,7 @@ public:
     default:
       return false;
     case VPInstruction::BranchOnCount:
+    case VPInstruction::CanonicalIVIncrementForPart:
       return true;
     };
     llvm_unreachable("switch should return");
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
index 1e79c3e1e8dc..b91579106261 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
@@ -99,7 +99,7 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP8]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; TFA_INTERLEAVE:       vector.body:
-; TFA_INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT4:%.*]], [[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], [[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = getelementptr i64, ptr [[B:%.*]], i64 [[INDEX]]
@@ -117,7 +117,6 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP13]], ptr [[TMP15]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP14]], ptr [[TMP18]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
 ; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT:%.*]] = add i64 [[INDEX]], [[TMP6]]
-; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT4]] = add i64 [[INDEX]], [[TMP6]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
 ; TFA_INTERLEAVE-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP21:%.*]] = add i64 [[INDEX_NEXT]], [[TMP20]]
@@ -254,7 +253,7 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP8]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; TFA_INTERLEAVE:       vector.body:
-; TFA_INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT5:%.*]], [[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT6:%.*]], [[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
@@ -284,7 +283,6 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr [[TMP25]], i32 8, <vscale x 2 x i1> [[TMP23]])
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI4]], ptr [[TMP28]], i32 8, <vscale x 2 x i1> [[TMP24]])
 ; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT:%.*]] = add i64 [[INDEX]], [[TMP6]]
-; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT5]] = add i64 [[INDEX]], [[TMP6]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
 ; TFA_INTERLEAVE-NEXT:    [[TMP30:%.*]] = mul i64 [[TMP29]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP31:%.*]] = add i64 [[INDEX_NEXT]], [[TMP30]]
@@ -437,7 +435,7 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP8]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; TFA_INTERLEAVE:       vector.body:
-; TFA_INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT5:%.*]], [[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT6:%.*]], [[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
@@ -469,7 +467,6 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr [[TMP27]], i32 8, <vscale x 2 x i1> [[TMP25]])
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI4]], ptr [[TMP30]], i32 8, <vscale x 2 x i1> [[TMP26]])
 ; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT:%.*]] = add i64 [[INDEX]], [[TMP6]]
-; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT5]] = add i64 [[INDEX]], [[TMP6]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP31:%.*]] = call i64 @llvm.vscale.i64()
 ; TFA_INTERLEAVE-NEXT:    [[TMP32:%.*]] = mul i64 [[TMP31]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP33:%.*]] = add i64 [[INDEX_NEXT]], [[TMP32]]
@@ -771,7 +768,7 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP8]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; TFA_INTERLEAVE:       vector.body:
-; TFA_INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT4:%.*]], [[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], [[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = getelementptr i64, ptr [[B:%.*]], i64 [[INDEX]]
@@ -789,7 +786,6 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP13]], ptr [[TMP15]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP14]], ptr [[TMP18]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
 ; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT:%.*]] = add i64 [[INDEX]], [[TMP6]]
-; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT4]] = add i64 [[INDEX]], [[TMP6]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
 ; TFA_INTERLEAVE-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP21:%.*]] = add i64 [[INDEX_NEXT]], [[TMP20]]
@@ -970,7 +966,7 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
 ; TFA_INTERLEAVE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
 ; TFA_INTERLEAVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; TFA_INTERLEAVE:       vector.body:
-; TFA_INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT4:%.*]], [[VECTOR_BODY]] ]
+; TFA_INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], [[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, [[ENTRY]] ], [ [[TMP26:%.*]], [[VECTOR_BODY]] ]
@@ -997,7 +993,6 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
 ; TFA_INTERLEAVE-NEXT:    [[TMP25:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> [[TMP14]], <vscale x 2 x double> shufflevector (<vscale x 2 x double> insertelement (<vscale x 2 x double> poison, double -0.000000e+00, i64 0), <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer)
 ; TFA_INTERLEAVE-NEXT:    [[TMP26]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[TMP24]], <vscale x 2 x double> [[TMP25]])
 ; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT:%.*]] = add i64 [[INDEX]], [[TMP6]]
-; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT4]] = add i64 [[INDEX]], [[TMP6]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
 ; TFA_INTERLEAVE-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP29:%.*]] = add i64 [[INDEX_NEXT]], [[TMP28]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
index 1970ac966535..809d2e8f7ea1 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
@@ -20,7 +20,7 @@ define i32 @pr70988() {
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ult i64 1, [[UMAX]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT6:%.*]], [[PRED_LOAD_CONTINUE5:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE5:%.*]] ]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_LOAD_CONTINUE5]] ]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK2:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY1]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT7:%.*]], [[PRED_LOAD_CONTINUE5]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[PRED_LOAD_CONTINUE5]] ]
@@ -50,7 +50,6 @@ define i32 @pr70988() {
 ; CHECK-NEXT:    [[TMP17]] = select i1 [[ACTIVE_LANE_MASK]], i32 [[TMP15]], i32 [[VEC_PHI]]
 ; CHECK-NEXT:    [[TMP18]] = select i1 [[ACTIVE_LANE_MASK2]], i32 [[TMP16]], i32 [[VEC_PHI3]]
 ; CHECK-NEXT:    [[INDEX_NEXT:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT:    [[INDEX_NEXT6]] = add i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], 1
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX_NEXT]], [[UMAX]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT7]] = icmp ult i64 [[TMP19]], [[UMAX]]
-- 
GitLab


From e1d4ddb0c68f69001d7de43d9c678f4d73c1ecba Mon Sep 17 00:00:00 2001
From: NagyDonat <donat.nagy@ericsson.com>
Date: Mon, 25 Mar 2024 12:43:51 +0100
Subject: [PATCH 114/404] Reapply "[analyzer] Accept C library functions from
 the `std` namespace" again (#85791)

This reapplies 80ab8234ac309418637488b97e0a62d8377b2ecf again, after
fixing a name collision warning in the unit tests (see the revert commit
13ccaf9b9d4400bb128b35ff4ac733e4afc3ad1c for details).

In addition to the previously applied changes, this commit also clarifies the
code in MallocChecker that distinguishes POSIX "getline()" and C++ standard
library "std::getline()" (which are two completely different functions). Note
that "std::getline()" was (accidentally) handled correctly even without this
clarification; but it's better to explicitly handle and test this corner case.

---------

Co-authored-by: Balazs Benics <benicsbalazs@gmail.com>
---
 .../Core/PathSensitive/CallDescription.h      |  8 +-
 .../StaticAnalyzer/Checkers/MallocChecker.cpp | 27 ++++--
 .../StaticAnalyzer/Core/CheckerContext.cpp    |  8 +-
 .../Inputs/system-header-simulator-cxx.h      | 11 ++-
 clang/test/Analysis/getline-cpp.cpp           | 15 ++++
 clang/unittests/StaticAnalyzer/CMakeLists.txt |  1 +
 .../StaticAnalyzer/IsCLibraryFunctionTest.cpp | 84 +++++++++++++++++++
 .../clang/unittests/StaticAnalyzer/BUILD.gn   |  1 +
 8 files changed, 139 insertions(+), 16 deletions(-)
 create mode 100644 clang/test/Analysis/getline-cpp.cpp
 create mode 100644 clang/unittests/StaticAnalyzer/IsCLibraryFunctionTest.cpp

diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h
index 3432d2648633..b4e1636130ca 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h
@@ -41,12 +41,8 @@ public:
     ///  - We also accept calls where the number of arguments or parameters is
     ///    greater than the specified value.
     /// For the exact heuristics, see CheckerContext::isCLibraryFunction().
-    /// Note that functions whose declaration context is not a TU (e.g.
-    /// methods, functions in namespaces) are not accepted as C library
-    /// functions.
-    /// FIXME: If I understand it correctly, this discards calls where C++ code
-    /// refers a C library function through the namespace `std::` via headers
-    /// like <cstdlib>.
+    /// (This mode only matches functions that are declared either directly
+    /// within a TU or in the namespace `std`.)
     CLibrary,
 
     /// Matches "simple" functions that are not methods. (Static methods are
diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
index c2d96f592609..88fb42b6625a 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
@@ -394,8 +394,10 @@ private:
                                      const CallEvent &Call, CheckerContext &C)>;
 
   const CallDescriptionMap<CheckFn> PreFnMap{
-      {{{"getline"}, 3}, &MallocChecker::preGetdelim},
-      {{{"getdelim"}, 4}, &MallocChecker::preGetdelim},
+      // NOTE: the following CallDescription also matches the C++ standard
+      // library function std::getline(); the callback will filter it out.
+      {{CDM::CLibrary, {"getline"}, 3}, &MallocChecker::preGetdelim},
+      {{CDM::CLibrary, {"getdelim"}, 4}, &MallocChecker::preGetdelim},
   };
 
   const CallDescriptionMap<CheckFn> FreeingMemFnMap{
@@ -446,8 +448,11 @@ private:
        std::bind(&MallocChecker::checkRealloc, _1, _2, _3, false)},
       {{{"g_realloc_n"}, 3}, &MallocChecker::checkReallocN},
       {{{"g_try_realloc_n"}, 3}, &MallocChecker::checkReallocN},
-      {{{"getline"}, 3}, &MallocChecker::checkGetdelim},
-      {{{"getdelim"}, 4}, &MallocChecker::checkGetdelim},
+
+      // NOTE: the following CallDescription also matches the C++ standard
+      // library function std::getline(); the callback will filter it out.
+      {{CDM::CLibrary, {"getline"}, 3}, &MallocChecker::checkGetdelim},
+      {{CDM::CLibrary, {"getdelim"}, 4}, &MallocChecker::checkGetdelim},
   };
 
   bool isMemCall(const CallEvent &Call) const;
@@ -1435,9 +1440,17 @@ void MallocChecker::checkGMallocN0(const CallEvent &Call,
   C.addTransition(State);
 }
 
+static bool isFromStdNamespace(const CallEvent &Call) {
+  const Decl *FD = Call.getDecl();
+  assert(FD && "a CallDescription cannot match a call without a Decl");
+  return FD->isInStdNamespace();
+}
+
 void MallocChecker::preGetdelim(const CallEvent &Call,
                                 CheckerContext &C) const {
-  if (!Call.isGlobalCFunction())
+  // Discard calls to the C++ standard library function std::getline(), which
+  // is completely unrelated to the POSIX getline() that we're checking.
+  if (isFromStdNamespace(Call))
     return;
 
   ProgramStateRef State = C.getState();
@@ -1458,7 +1471,9 @@ void MallocChecker::preGetdelim(const CallEvent &Call,
 
 void MallocChecker::checkGetdelim(const CallEvent &Call,
                                   CheckerContext &C) const {
-  if (!Call.isGlobalCFunction())
+  // Discard calls to the C++ standard library function std::getline(), which
+  // is completely unrelated to the POSIX getline() that we're checking.
+  if (isFromStdNamespace(Call))
     return;
 
   ProgramStateRef State = C.getState();
diff --git a/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp b/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp
index d6d4cec9dd3d..1a9bff529e9b 100644
--- a/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp
@@ -87,9 +87,11 @@ bool CheckerContext::isCLibraryFunction(const FunctionDecl *FD,
   if (!II)
     return false;
 
-  // Look through 'extern "C"' and anything similar invented in the future.
-  // If this function is not in TU directly, it is not a C library function.
-  if (!FD->getDeclContext()->getRedeclContext()->isTranslationUnit())
+  // C library functions are either declared directly within a TU (the common
+  // case) or they are accessed through the namespace `std` (when they are used
+  // in C++ via headers like <cstdlib>).
+  const DeclContext *DC = FD->getDeclContext()->getRedeclContext();
+  if (!(DC->isTranslationUnit() || DC->isStdNamespace()))
     return false;
 
   // If this function is not externally visible, it is not a C library function.
diff --git a/clang/test/Analysis/Inputs/system-header-simulator-cxx.h b/clang/test/Analysis/Inputs/system-header-simulator-cxx.h
index 3ef7af2ea6c6..85db68d41a6c 100644
--- a/clang/test/Analysis/Inputs/system-header-simulator-cxx.h
+++ b/clang/test/Analysis/Inputs/system-header-simulator-cxx.h
@@ -1106,11 +1106,20 @@ using ostream = basic_ostream<char>;
 extern std::ostream cout;
 
 ostream &operator<<(ostream &, const string &);
-
 #if __cplusplus >= 202002L
 template <class T>
 ostream &operator<<(ostream &, const std::unique_ptr<T> &);
 #endif
+
+template <class CharT>
+class basic_istream;
+
+using istream = basic_istream<char>;
+
+extern std::istream cin;
+
+istream &getline(istream &, string &, char);
+istream &getline(istream &, string &);
 } // namespace std
 
 #ifdef TEST_INLINABLE_ALLOCATORS
diff --git a/clang/test/Analysis/getline-cpp.cpp b/clang/test/Analysis/getline-cpp.cpp
new file mode 100644
index 000000000000..ef9d3186009c
--- /dev/null
+++ b/clang/test/Analysis/getline-cpp.cpp
@@ -0,0 +1,15 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix,debug.ExprInspection -verify %s
+
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix,alpha.unix,debug.ExprInspection -verify %s
+//
+// expected-no-diagnostics
+
+#include "Inputs/system-header-simulator-cxx.h"
+
+void test_std_getline() {
+  std::string userid, comment;
+  // MallocChecker should not confuse the POSIX function getline() and the
+  // unrelated C++ standard library function std::getline.
+  std::getline(std::cin, userid, ' '); // no-crash
+  std::getline(std::cin, comment); // no-crash
+}
diff --git a/clang/unittests/StaticAnalyzer/CMakeLists.txt b/clang/unittests/StaticAnalyzer/CMakeLists.txt
index 519be36fe0fa..ff34d5747cc8 100644
--- a/clang/unittests/StaticAnalyzer/CMakeLists.txt
+++ b/clang/unittests/StaticAnalyzer/CMakeLists.txt
@@ -11,6 +11,7 @@ add_clang_unittest(StaticAnalysisTests
   CallEventTest.cpp
   ConflictingEvalCallsTest.cpp
   FalsePositiveRefutationBRVisitorTest.cpp
+  IsCLibraryFunctionTest.cpp
   MemRegionDescriptiveNameTest.cpp
   NoStateChangeFuncVisitorTest.cpp
   ParamRegionTest.cpp
diff --git a/clang/unittests/StaticAnalyzer/IsCLibraryFunctionTest.cpp b/clang/unittests/StaticAnalyzer/IsCLibraryFunctionTest.cpp
new file mode 100644
index 000000000000..d6dfcaac6f3b
--- /dev/null
+++ b/clang/unittests/StaticAnalyzer/IsCLibraryFunctionTest.cpp
@@ -0,0 +1,84 @@
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/Analysis/AnalysisDeclContext.h"
+#include "clang/Frontend/ASTUnit.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
+#include "clang/Tooling/Tooling.h"
+#include "gtest/gtest.h"
+
+#include <memory>
+
+using namespace clang;
+using namespace ento;
+using namespace ast_matchers;
+
+class IsCLibraryFunctionTest : public testing::Test {
+  std::unique_ptr<ASTUnit> ASTUnitP;
+  const FunctionDecl *Result = nullptr;
+
+public:
+  const FunctionDecl *getFunctionDecl() const { return Result; }
+
+  testing::AssertionResult buildAST(StringRef Code) {
+    ASTUnitP = tooling::buildASTFromCode(Code);
+    if (!ASTUnitP)
+      return testing::AssertionFailure() << "AST construction failed";
+
+    ASTContext &Context = ASTUnitP->getASTContext();
+    if (Context.getDiagnostics().hasErrorOccurred())
+      return testing::AssertionFailure() << "Compilation error";
+
+    auto Matches = ast_matchers::match(functionDecl().bind("fn"), Context);
+    if (Matches.empty())
+      return testing::AssertionFailure() << "No function declaration found";
+
+    if (Matches.size() > 1)
+      return testing::AssertionFailure()
+             << "Multiple function declarations found";
+
+    Result = Matches[0].getNodeAs<FunctionDecl>("fn");
+    return testing::AssertionSuccess();
+  }
+};
+
+TEST_F(IsCLibraryFunctionTest, AcceptsGlobal) {
+  ASSERT_TRUE(buildAST(R"cpp(void fun();)cpp"));
+  EXPECT_TRUE(CheckerContext::isCLibraryFunction(getFunctionDecl()));
+}
+
+TEST_F(IsCLibraryFunctionTest, AcceptsExternCGlobal) {
+  ASSERT_TRUE(buildAST(R"cpp(extern "C" { void fun(); })cpp"));
+  EXPECT_TRUE(CheckerContext::isCLibraryFunction(getFunctionDecl()));
+}
+
+TEST_F(IsCLibraryFunctionTest, RejectsNoInlineNoExternalLinkage) {
+  // Functions that are neither inlined nor externally visible cannot be C
+  // library functions.
+  ASSERT_TRUE(buildAST(R"cpp(static void fun();)cpp"));
+  EXPECT_FALSE(CheckerContext::isCLibraryFunction(getFunctionDecl()));
+}
+
+TEST_F(IsCLibraryFunctionTest, RejectsAnonymousNamespace) {
+  ASSERT_TRUE(buildAST(R"cpp(namespace { void fun(); })cpp"));
+  EXPECT_FALSE(CheckerContext::isCLibraryFunction(getFunctionDecl()));
+}
+
+TEST_F(IsCLibraryFunctionTest, AcceptsStdNamespace) {
+  ASSERT_TRUE(buildAST(R"cpp(namespace std { void fun(); })cpp"));
+  EXPECT_TRUE(CheckerContext::isCLibraryFunction(getFunctionDecl()));
+}
+
+TEST_F(IsCLibraryFunctionTest, RejectsOtherNamespaces) {
+  ASSERT_TRUE(buildAST(R"cpp(namespace stdx { void fun(); })cpp"));
+  EXPECT_FALSE(CheckerContext::isCLibraryFunction(getFunctionDecl()));
+}
+
+TEST_F(IsCLibraryFunctionTest, RejectsClassStatic) {
+  ASSERT_TRUE(buildAST(R"cpp(class A { static void fun(); };)cpp"));
+  EXPECT_FALSE(CheckerContext::isCLibraryFunction(getFunctionDecl()));
+}
+
+TEST_F(IsCLibraryFunctionTest, RejectsClassMember) {
+  ASSERT_TRUE(buildAST(R"cpp(class A { void fun(); };)cpp"));
+  EXPECT_FALSE(CheckerContext::isCLibraryFunction(getFunctionDecl()));
+}
diff --git a/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn
index 9230ac79a48a..adaf6c476dfe 100644
--- a/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn
@@ -19,6 +19,7 @@ unittest("StaticAnalysisTests") {
     "CallEventTest.cpp",
     "ConflictingEvalCallsTest.cpp",
     "FalsePositiveRefutationBRVisitorTest.cpp",
+    "IsCLibraryFunctionTest.cpp",
     "MemRegionDescriptiveNameTest.cpp",
     "NoStateChangeFuncVisitorTest.cpp",
     "ParamRegionTest.cpp",
-- 
GitLab


From 6b8bb29c26765ab085cd0986168407f5586f620e Mon Sep 17 00:00:00 2001
From: Christian Sigg <csigg@google.com>
Date: Mon, 25 Mar 2024 12:43:50 +0100
Subject: [PATCH 115/404] [llvm][bazel] Update BUILD file after
 fa3d789df15bd1f58fb8ba4ea3be909218cf7f03.

---
 .../llvm-project-overlay/llvm/BUILD.bazel     | 27 +++++++++++--------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index f5a2d264b690..65173f986585 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -598,14 +598,14 @@ cc_binary(
     name = "llvm-min-tblgen",
     srcs = [
         "utils/TableGen/Attributes.cpp",
-        "utils/TableGen/CodeGenIntrinsics.cpp",
-        "utils/TableGen/CodeGenIntrinsics.h",
+        "utils/TableGen/Basic/CodeGenIntrinsics.cpp",
+        "utils/TableGen/Basic/CodeGenIntrinsics.h",
+        "utils/TableGen/Basic/SDNodeProperties.cpp",
+        "utils/TableGen/Basic/SDNodeProperties.h",
+        "utils/TableGen/Basic/SequenceToOffsetTable.h",
         "utils/TableGen/DirectiveEmitter.cpp",
         "utils/TableGen/IntrinsicEmitter.cpp",
         "utils/TableGen/RISCVTargetDefEmitter.cpp",
-        "utils/TableGen/SDNodeProperties.cpp",
-        "utils/TableGen/SDNodeProperties.h",
-        "utils/TableGen/SequenceToOffsetTable.h",
         "utils/TableGen/TableGen.cpp",
         "utils/TableGen/VTEmitter.cpp",
     ],
@@ -621,7 +621,7 @@ cc_binary(
 cc_library(
     name = "TableGenGlobalISel",
     srcs = [
-        "utils/TableGen/GlobalISel/CodeExpander.cpp",
+        "utils/TableGen/Common/GlobalISel/CodeExpander.cpp",
     ],
     hdrs = glob([
         # We have to include these headers here as well as in the `hdrs` below
@@ -629,8 +629,8 @@ cc_library(
         # them, even though consumers of this library use inclusion relative to
         # `utils/TableGen` with the `strip_includes_prefix` of this library.
         # This mixture appears to be incompatible with header modules.
-        "utils/TableGen/GlobalISel/CodeExpander.h",
-        "utils/TableGen/GlobalISel/CodeExpansions.h",
+        "utils/TableGen/Common/GlobalISel/CodeExpander.h",
+        "utils/TableGen/Common/GlobalISel/CodeExpansions.h",
     ]),
     copts = llvm_copts,
     features = ["-header_modules"],
@@ -654,8 +654,12 @@ cc_binary(
         [
             "utils/TableGen/*.cpp",
             "utils/TableGen/*.h",
-            "utils/TableGen/GlobalISel/*.cpp",
-            "utils/TableGen/GlobalISel/*.h",
+            "utils/TableGen/Basic/*.cpp",
+            "utils/TableGen/Basic/*.h",
+            "utils/TableGen/Common/*.cpp",
+            "utils/TableGen/Common/*.h",
+            "utils/TableGen/Common/GlobalISel/*.cpp",
+            "utils/TableGen/Common/GlobalISel/*.h",
 
             # Some tablegen sources include headers from MC, so these have to be
             # listed here. MC uses headers produced by tablegen, so it cannot be a
@@ -663,9 +667,10 @@ cc_binary(
             "include/llvm/MC/*.h",
             "include/llvm/TargetParser/SubtargetFeature.h",
         ],
-        exclude = ["utils/TableGen/GlobalISel/CodeExpander.cpp"],
+        exclude = ["utils/TableGen/Common/GlobalISel/CodeExpander.cpp"],
     ),
     copts = llvm_copts,
+    includes = ["utils/TableGen"],
     stamp = 0,
     deps = [
         ":CodeGenTypes",
-- 
GitLab


From edcf65d40c4160db0a888e801560a671b10179d2 Mon Sep 17 00:00:00 2001
From: Leandro Lupori <leandro.lupori@linaro.org>
Date: Mon, 25 Mar 2024 08:48:11 -0300
Subject: [PATCH 116/404] [flang][OpenMP] Allow loop iteration variables in DSA
 clauses (#86194)

Iteration variables of non-associated loops may be listed in DSA
clauses.

Fixes https://github.com/llvm/llvm-project/issues/78938
---
 flang/lib/Semantics/resolve-directives.cpp |  9 +++++++++
 flang/test/Semantics/OpenMP/do20.f90       | 18 ++++++++++++++++++
 2 files changed, 27 insertions(+)
 create mode 100644 flang/test/Semantics/OpenMP/do20.f90

diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 6d58013b87d2..27af192f606b 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -1649,6 +1649,15 @@ void OmpAttributeVisitor::ResolveSeqLoopIndexInParallelOrTaskConstruct(
       break;
     }
   }
+  // If this symbol already has a data-sharing attribute then there is nothing
+  // to do here.
+  if (const Symbol * symbol{iv.symbol}) {
+    for (auto symMap : targetIt->objectWithDSA) {
+      if (symMap.first->name() == symbol->name()) {
+        return;
+      }
+    }
+  }
   // If this symbol is already Private or Firstprivate in the enclosing
   // OpenMP parallel or task then there is nothing to do here.
   if (auto *symbol{targetIt->scope.FindSymbol(iv.source)}) {
diff --git a/flang/test/Semantics/OpenMP/do20.f90 b/flang/test/Semantics/OpenMP/do20.f90
new file mode 100644
index 000000000000..915d01e69edd
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/do20.f90
@@ -0,0 +1,18 @@
+! RUN: %python %S/../test_symbols.py %s %flang_fc1 -fopenmp
+
+! OpenMP 5.2 5.1.1
+! Iteration variables of non-associated loops may be listed in DSA clauses.
+
+!DEF: /shared_iv (Subroutine)Subprogram
+subroutine shared_iv
+  !DEF: /shared_iv/i ObjectEntity INTEGER(4)
+  integer i
+
+  !$omp parallel shared(i)
+    !$omp single
+      !REF: /shared_iv/i
+      do i = 0, 1
+      end do
+    !$omp end single
+  !$omp end parallel
+end subroutine
-- 
GitLab


From 684ca6a25ae0a4b8d015ed321f7e99cb7153951e Mon Sep 17 00:00:00 2001
From: Dmitri Gribenko <gribozavr@gmail.com>
Date: Mon, 25 Mar 2024 13:35:08 +0100
Subject: [PATCH 117/404] [bazel] Format defs.bzl

---
 .../llvm-project-overlay/clang-tools-extra/clang-tidy/defs.bzl  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/defs.bzl b/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/defs.bzl
index 5abe4b08f5d9..387d104a773a 100644
--- a/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/defs.bzl
+++ b/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/defs.bzl
@@ -17,7 +17,7 @@ _common_library_deps = [
 
 def clang_tidy_library(name, **kwargs):
     kwargs["srcs"] = kwargs.get("srcs", native.glob([paths.join(name, "*.cpp")], allow_empty = True))
-    kwargs["hdrs"] = kwargs.get("hdrs", native.glob([paths.join(name, "*.h")], allow_empty=True))
+    kwargs["hdrs"] = kwargs.get("hdrs", native.glob([paths.join(name, "*.h")], allow_empty = True))
     kwargs["deps"] = kwargs.get("deps", []) + _common_library_deps
     cc_library(
         name = name,
-- 
GitLab


From 46a737cef2a709d5c121ab616fc80088ce2eed46 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Mon, 25 Mar 2024 08:02:53 -0500
Subject: [PATCH 118/404] [LLVM] Fix linking object library in tablegen

Summary:
The patch https://github.com/llvm/llvm-project/pull/80847 introduced
some changes to how we build tablegen. Some files weren't build included
properly on my build. We don't need to fish out the objects directly
from an object library build, we can simply add it to the link libraries
and CMake will include them for you. This fixed the issue of missings
defs for me and is more canonical.
---
 llvm/utils/TableGen/Common/CMakeLists.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/llvm/utils/TableGen/Common/CMakeLists.txt b/llvm/utils/TableGen/Common/CMakeLists.txt
index 0e985a083247..0440f027f286 100644
--- a/llvm/utils/TableGen/Common/CMakeLists.txt
+++ b/llvm/utils/TableGen/Common/CMakeLists.txt
@@ -33,7 +33,9 @@ add_llvm_library(LLVMTableGenCommon STATIC OBJECT EXCLUDE_FROM_ALL
   SubtargetFeatureInfo.cpp
   Types.cpp
   VarLenCodeEmitterGen.cpp
-  $<TARGET_OBJECTS:obj.LLVMTableGenBasic>
+
+  LINK_LIBS
+  LLVMTableGenBasic
 
   DEPENDS
   vt_gen
-- 
GitLab


From 4bb9f918ff4e2a932ce1e3da5f14103778b04136 Mon Sep 17 00:00:00 2001
From: Jianbang Yang <yangjianbang112@gmail.com>
Date: Mon, 25 Mar 2024 21:08:18 +0800
Subject: [PATCH 119/404] [mlir][tensor] fix out-of-bound index in tensor.dim
 (#85901)

fix a crash when fold tensor.dim with out-of-bound index.

Fixes: https://github.com/llvm/llvm-project/issues/70183
---
 .../Bufferization/IR/BufferizationOps.cpp     |  3 +++
 mlir/test/Dialect/Tensor/canonicalize.mlir    | 23 +++++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
index 2b226c7a1207..a656c812a59f 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
@@ -333,6 +333,9 @@ struct FoldDimOfAllocTensorOp : public OpRewritePattern<tensor::DimOp> {
     auto allocTensorOp = dimOp.getSource().getDefiningOp<AllocTensorOp>();
     if (!allocTensorOp || !maybeConstantIndex)
       return failure();
+    if (*maybeConstantIndex < 0 ||
+        *maybeConstantIndex >= allocTensorOp.getType().getRank())
+      return failure();
     if (!allocTensorOp.getType().isDynamicDim(*maybeConstantIndex))
       return failure();
     rewriter.replaceOp(
diff --git a/mlir/test/Dialect/Tensor/canonicalize.mlir b/mlir/test/Dialect/Tensor/canonicalize.mlir
index e5374f031be5..9ab54fe9c133 100644
--- a/mlir/test/Dialect/Tensor/canonicalize.mlir
+++ b/mlir/test/Dialect/Tensor/canonicalize.mlir
@@ -2367,3 +2367,26 @@ func.func @dim_of_reshape_undominated(%arg0: tensor<*xf32>, %arg1: tensor<?xinde
     %dim = tensor.dim %reshape, %0 : tensor<*xf32>
     return %dim : index
   }
+
+// -----
+
+// Test case: This test fails to fold because the index of tensor.dim is out_of_bounds
+// CHECK-LABEL: func @dim_out_of_bounds(
+//       CHECK: %[[IDX:.*]] = index.constant 28
+//  CHECK-NEXT: bufferization.alloc_tensor
+//  CHECK-NEXT: %[[DIM:.*]] = tensor.dim %{{.*}}, %[[IDX]]
+//  CHECK-NEXT: memref.alloc
+//  CHECK-NEXT: memref.cast
+//  CHECK-NEXT: affine.vector_load %{{.*}}[{{.*}}, {{.*}}, symbol(%[[DIM]])]
+//  CHECK-NEXT: return
+func.func @dim_out_of_bounds() -> vector<7xi32> {
+    %c1 = arith.constant 1 : index
+    %idx28 = index.constant 28
+    %c29 = arith.constant 29 : index
+    %3 = bufferization.alloc_tensor(%c29) : tensor<?xi16>
+    %dim = tensor.dim %3, %idx28 : tensor<?xi16>
+    %alloc_21 = memref.alloc(%c29) : memref<?x26x2xi32>
+    %16 = affine.vector_load %alloc_21[%c1, %c1, %dim] : memref<?x26x2xi32>, vector<7xi32>
+    return %16 : vector<7xi32>
+}
+
-- 
GitLab


From e6f63a942a45e3545332cd9a43982a69a4d5667b Mon Sep 17 00:00:00 2001
From: martinboehme <mboehme@google.com>
Date: Mon, 25 Mar 2024 14:08:25 +0100
Subject: [PATCH 120/404] [clang][dataflow] Bail out if input is Objective-C++.
 (#86479)

We only ever intended to support C++, but the condition we were testing
allowed
Objective-C++ code by mistake.
---
 .../lib/Analysis/FlowSensitive/AdornedCFG.cpp |  2 +-
 .../FlowSensitive/DeterminismTest.cpp         |  4 ++-
 .../Analysis/FlowSensitive/TransferTest.cpp   | 33 +++++++++++++++----
 3 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/clang/lib/Analysis/FlowSensitive/AdornedCFG.cpp b/clang/lib/Analysis/FlowSensitive/AdornedCFG.cpp
index daa73bed1bd9..255543021a99 100644
--- a/clang/lib/Analysis/FlowSensitive/AdornedCFG.cpp
+++ b/clang/lib/Analysis/FlowSensitive/AdornedCFG.cpp
@@ -144,7 +144,7 @@ llvm::Expected<AdornedCFG> AdornedCFG::build(const Decl &D, Stmt &S,
 
   // The shape of certain elements of the AST can vary depending on the
   // language. We currently only support C++.
-  if (!C.getLangOpts().CPlusPlus)
+  if (!C.getLangOpts().CPlusPlus || C.getLangOpts().ObjC)
     return llvm::createStringError(
         std::make_error_code(std::errc::invalid_argument),
         "Can only analyze C++");
diff --git a/clang/unittests/Analysis/FlowSensitive/DeterminismTest.cpp b/clang/unittests/Analysis/FlowSensitive/DeterminismTest.cpp
index e794bd4943f2..a2cbfb1ff582 100644
--- a/clang/unittests/Analysis/FlowSensitive/DeterminismTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/DeterminismTest.cpp
@@ -30,7 +30,9 @@ namespace clang::dataflow {
 // flow-condition at function exit.
 std::string analyzeAndPrintExitCondition(llvm::StringRef Code) {
   DataflowAnalysisContext DACtx(std::make_unique<WatchedLiteralsSolver>());
-  clang::TestAST AST(Code);
+  TestInputs Inputs(Code);
+  Inputs.Language = TestLanguage::Lang_CXX17;
+  clang::TestAST AST(Inputs);
   const auto *Target =
       cast<FunctionDecl>(test::findValueDecl(AST.context(), "target"));
   Environment InitEnv(DACtx, *Target);
diff --git a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
index 1d3b268976a7..ca055a462a28 100644
--- a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
@@ -17,6 +17,7 @@
 #include "clang/Analysis/FlowSensitive/StorageLocation.h"
 #include "clang/Analysis/FlowSensitive/Value.h"
 #include "clang/Basic/LangStandard.h"
+#include "clang/Testing/TestAST.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Testing/Support/Error.h"
@@ -135,12 +136,32 @@ const Formula &getFormula(const ValueDecl &D, const Environment &Env) {
 }
 
 TEST(TransferTest, CNotSupported) {
-  std::string Code = R"(
-    void target() {}
-  )";
-  ASSERT_THAT_ERROR(checkDataflowWithNoopAnalysis(
-                        Code, [](const auto &, auto &) {}, {BuiltinOptions{}},
-                        LangStandard::lang_c89),
+  TestInputs Inputs("void target() {}");
+  Inputs.Language = TestLanguage::Lang_C89;
+  clang::TestAST AST(Inputs);
+  const auto *Target =
+      cast<FunctionDecl>(test::findValueDecl(AST.context(), "target"));
+  ASSERT_THAT_ERROR(AdornedCFG::build(*Target).takeError(),
+                    llvm::FailedWithMessage("Can only analyze C++"));
+}
+
+TEST(TransferTest, ObjectiveCNotSupported) {
+  TestInputs Inputs("void target() {}");
+  Inputs.Language = TestLanguage::Lang_OBJC;
+  clang::TestAST AST(Inputs);
+  const auto *Target =
+      cast<FunctionDecl>(test::findValueDecl(AST.context(), "target"));
+  ASSERT_THAT_ERROR(AdornedCFG::build(*Target).takeError(),
+                    llvm::FailedWithMessage("Can only analyze C++"));
+}
+
+TEST(TransferTest, ObjectiveCXXNotSupported) {
+  TestInputs Inputs("void target() {}");
+  Inputs.Language = TestLanguage::Lang_OBJCXX;
+  clang::TestAST AST(Inputs);
+  const auto *Target =
+      cast<FunctionDecl>(test::findValueDecl(AST.context(), "target"));
+  ASSERT_THAT_ERROR(AdornedCFG::build(*Target).takeError(),
                     llvm::FailedWithMessage("Can only analyze C++"));
 }
 
-- 
GitLab


From e1a003dbbd016f57b349c87981bf210f8ec3e1e0 Mon Sep 17 00:00:00 2001
From: alx32 <103613512+alx32@users.noreply.github.com>
Date: Mon, 25 Mar 2024 06:55:11 -0700
Subject: [PATCH 121/404] [lld-macho][NFC] Refactor ObjCSelRefsSection =>
 ObjCSelRefsHelper (#86456)

In a previous PR: https://github.com/llvm/llvm-project/pull/83878, the
intent was to make no functional changes, just refactor out the code for
reuse.
However, by creating `ObjCSelRefsSection` as a `SyntheticSection` - this
slightly changed the functionality of the application as the
`SyntheticSection` constructor registers the `SyntheticSection` as a
functional one - with an associated `SyntheticInputSection`.

With this change we remove this unintended consequence by making the
code not use a `SyntheticSection` as base, but just by having it be a
static helper.
---
 lld/MachO/Driver.cpp            |  1 +
 lld/MachO/SyntheticSections.cpp | 19 ++++++++++---------
 lld/MachO/SyntheticSections.h   | 21 ++++++---------------
 lld/MachO/Writer.cpp            |  3 +--
 4 files changed, 18 insertions(+), 26 deletions(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 919a14b8bcf0..14c111ce9685 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -1394,6 +1394,7 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
     syntheticSections.clear();
     thunkMap.clear();
     unprocessedLCLinkerOptions.clear();
+    ObjCSelRefsHelper::cleanup();
 
     firstTLVDataSection = nullptr;
     tar = nullptr;
diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp
index 1b3694528de1..0afbbd478bb9 100644
--- a/lld/MachO/SyntheticSections.cpp
+++ b/lld/MachO/SyntheticSections.cpp
@@ -806,10 +806,9 @@ void StubHelperSection::setUp() {
   dyldPrivate->used = true;
 }
 
-ObjCSelRefsSection::ObjCSelRefsSection()
-    : SyntheticSection(segment_names::data, section_names::objcSelrefs) {}
-
-void ObjCSelRefsSection::initialize() {
+llvm::DenseMap<llvm::CachedHashStringRef, ConcatInputSection *>
+    ObjCSelRefsHelper::methnameToSelref;
+void ObjCSelRefsHelper::initialize() {
   // Do not fold selrefs without ICF.
   if (config->icfLevel == ICFLevel::none)
     return;
@@ -836,7 +835,9 @@ void ObjCSelRefsSection::initialize() {
   }
 }
 
-ConcatInputSection *ObjCSelRefsSection::makeSelRef(StringRef methname) {
+void ObjCSelRefsHelper::cleanup() { methnameToSelref.clear(); }
+
+ConcatInputSection *ObjCSelRefsHelper::makeSelRef(StringRef methname) {
   auto methnameOffset =
       in.objcMethnameSection->getStringOffset(methname).outSecOff;
 
@@ -861,7 +862,7 @@ ConcatInputSection *ObjCSelRefsSection::makeSelRef(StringRef methname) {
   return objcSelref;
 }
 
-ConcatInputSection *ObjCSelRefsSection::getSelRef(StringRef methname) {
+ConcatInputSection *ObjCSelRefsHelper::getSelRef(StringRef methname) {
   auto it = methnameToSelref.find(CachedHashStringRef(methname));
   if (it == methnameToSelref.end())
     return nullptr;
@@ -890,8 +891,8 @@ StringRef ObjCStubsSection::getMethname(Symbol *sym) {
 void ObjCStubsSection::addEntry(Symbol *sym) {
   StringRef methname = getMethname(sym);
   // We create a selref entry for each unique methname.
-  if (!in.objcSelRefs->getSelRef(methname))
-    in.objcSelRefs->makeSelRef(methname);
+  if (!ObjCSelRefsHelper::getSelRef(methname))
+    ObjCSelRefsHelper::makeSelRef(methname);
 
   auto stubSize = config->objcStubsMode == ObjCStubsMode::fast
                       ? target->objcStubsFastSize
@@ -940,7 +941,7 @@ void ObjCStubsSection::writeTo(uint8_t *buf) const {
     Defined *sym = symbols[i];
 
     auto methname = getMethname(sym);
-    InputSection *selRef = in.objcSelRefs->getSelRef(methname);
+    InputSection *selRef = ObjCSelRefsHelper::getSelRef(methname);
     assert(selRef != nullptr && "no selref for methname");
     auto selrefAddr = selRef->getVA(0);
     target->writeObjCMsgSendStub(buf + stubOffset, sym, in.objcStubs->addr,
diff --git a/lld/MachO/SyntheticSections.h b/lld/MachO/SyntheticSections.h
index 6d85f0aea8e0..4586a4a0bf43 100644
--- a/lld/MachO/SyntheticSections.h
+++ b/lld/MachO/SyntheticSections.h
@@ -315,24 +315,16 @@ public:
   Defined *dyldPrivate = nullptr;
 };
 
-class ObjCSelRefsSection final : public SyntheticSection {
+class ObjCSelRefsHelper {
 public:
-  ObjCSelRefsSection();
-  void initialize();
-
-  // This SyntheticSection does not do directly write data to the output, it is
-  // just a placeholder for easily creating SyntheticInputSection's which will
-  // be inserted into inputSections and handeled by the default writing
-  // mechanism.
-  uint64_t getSize() const override { return 0; }
-  bool isNeeded() const override { return false; }
-  void writeTo(uint8_t *buf) const override {}
+  static void initialize();
+  static void cleanup();
 
-  ConcatInputSection *getSelRef(StringRef methname);
-  ConcatInputSection *makeSelRef(StringRef methname);
+  static ConcatInputSection *getSelRef(StringRef methname);
+  static ConcatInputSection *makeSelRef(StringRef methname);
 
 private:
-  llvm::DenseMap<llvm::CachedHashStringRef, ConcatInputSection *>
+  static llvm::DenseMap<llvm::CachedHashStringRef, ConcatInputSection *>
       methnameToSelref;
 };
 
@@ -813,7 +805,6 @@ struct InStruct {
   LazyPointerSection *lazyPointers = nullptr;
   StubsSection *stubs = nullptr;
   StubHelperSection *stubHelper = nullptr;
-  ObjCSelRefsSection *objcSelRefs = nullptr;
   ObjCStubsSection *objcStubs = nullptr;
   UnwindInfoSection *unwindInfo = nullptr;
   ObjCImageInfoSection *objCImageInfo = nullptr;
diff --git a/lld/MachO/Writer.cpp b/lld/MachO/Writer.cpp
index 8f335188e12c..a18b5268fd42 100644
--- a/lld/MachO/Writer.cpp
+++ b/lld/MachO/Writer.cpp
@@ -720,7 +720,7 @@ static void addNonWeakDefinition(const Defined *defined) {
 
 void Writer::scanSymbols() {
   TimeTraceScope timeScope("Scan symbols");
-  in.objcSelRefs->initialize();
+  ObjCSelRefsHelper::initialize();
   for (Symbol *sym : symtab->getSymbols()) {
     if (auto *defined = dyn_cast<Defined>(sym)) {
       if (!defined->isLive())
@@ -1359,7 +1359,6 @@ void macho::createSyntheticSections() {
   in.got = make<GotSection>();
   in.tlvPointers = make<TlvPointerSection>();
   in.stubs = make<StubsSection>();
-  in.objcSelRefs = make<ObjCSelRefsSection>();
   in.objcStubs = make<ObjCStubsSection>();
   in.unwindInfo = makeUnwindInfoSection();
   in.objCImageInfo = make<ObjCImageInfoSection>();
-- 
GitLab


From b33166472c17b51b0b70a72424c2387e95f11b2d Mon Sep 17 00:00:00 2001
From: Jie Fu <jiefu@tencent.com>
Date: Mon, 25 Mar 2024 21:59:01 +0800
Subject: [PATCH 122/404] [flang] Fix -Wunused-variable in BoxedProcedure.cpp
 (NFC)

llvm-project/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp:157:12:
error: unused variable 'it' [-Werror,-Wunused-variable]
      auto it = convertedTypes.try_emplace(ty, rec);
           ^
1 error generated.
---
 flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp b/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
index dccb642d5f79..48173033ecbe 100644
--- a/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
+++ b/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
@@ -154,7 +154,7 @@ public:
                                  ty.getName().str() + boxprocSuffix.str());
       if (rec.isFinalized())
         return rec;
-      auto it = convertedTypes.try_emplace(ty, rec);
+      [[maybe_unused]] auto it = convertedTypes.try_emplace(ty, rec);
       assert(it.second && "expected ty to not be in the map");
       std::vector<RecordType::TypePair> ps = ty.getLenParamList();
       std::vector<RecordType::TypePair> cs;
-- 
GitLab


From f7e7064992ced7795e66ab90f9a7c3195b6fa230 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Mon, 25 Mar 2024 14:09:52 +0000
Subject: [PATCH 123/404] [mlir][SVE] Fix memory leaks in integration tests
 (#86488)

Buffers are no longer deallocated by One-Shot Bufferize - this is now
done by a separate buffer deallocation pass.

In order to see the leaks in SVE integration tests, use the following
CMake flags (enables the address sanitizer and SVE integration tests):

    -DLLVM_USE_SANITIZER="Address"
    -DMLIR_INCLUDE_INTEGRATION_TESTS=On
    -DMLIR_RUN_ARM_SVE_TESTS=On

Follow-up for #85366
---
 mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/fill-1d.mlir | 2 +-
 mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/matmul.mlir  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/fill-1d.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/fill-1d.mlir
index a5b2ad1605cd..44b555c8c3ad 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/fill-1d.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/fill-1d.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -transform-interpreter -test-transform-dialect-erase-schedule -lower-vector-mask -one-shot-bufferize -test-lower-to-llvm | \
+// RUN: mlir-opt %s -transform-interpreter -test-transform-dialect-erase-schedule -lower-vector-mask -one-shot-bufferize -buffer-deallocation-pipeline -test-lower-to-llvm | \
 // RUN: %mcr_aarch64_cmd -e=entry -entry-point-result=void --march=aarch64 --mattr="+sve" -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils | \
 // RUN: FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/matmul.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/matmul.mlir
index 51a0c8f7c945..68e474fe5cef 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/matmul.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/matmul.mlir
@@ -1,6 +1,6 @@
 // DEFINE: %{compile} =  mlir-opt %s \
 // DEFINE:    -transform-interpreter -test-transform-dialect-erase-schedule \
-// DEFINE:    -one-shot-bufferize -func-bufferize -cse -canonicalize -convert-vector-to-scf -arm-sve-legalize-vector-storage \
+// DEFINE:    -one-shot-bufferize="bufferize-function-boundaries" -buffer-deallocation-pipeline -cse -canonicalize -convert-vector-to-scf -arm-sve-legalize-vector-storage \
 // DEFINE:    -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm -o %t
 // DEFINE: %{entry_point} = matmul_f32
 // DEFINE: %{run} = %mcr_aarch64_cmd %t -e %{entry_point} -entry-point-result=void --march=aarch64 --mattr="+sve"\
-- 
GitLab


From 865294b2e67d91d6e6a123db1b71a175e015a210 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Mon, 25 Mar 2024 10:10:35 -0400
Subject: [PATCH 124/404] [CodeGen][MISched] Add misched post-regalloc
 bidirectional scheduling (#77138)

This PR is stacked on #76186.

This PR keeps the default strategy as top-down since that is what
existing targets expect. It can be enabled using
`-misched-postra-direction=bidirectional`.

It is up to targets to decide whether they would like to enable this
option for themselves.
---
 llvm/include/llvm/CodeGen/MachineScheduler.h  |   9 ++
 llvm/lib/CodeGen/MachineScheduler.cpp         | 113 ++++++++++++++++--
 .../RISCV/misched-postra-direction.mir        |  20 +++-
 3 files changed, 128 insertions(+), 14 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/MachineScheduler.h b/llvm/include/llvm/CodeGen/MachineScheduler.h
index 25703dd6b61f..9cca6b3a571c 100644
--- a/llvm/include/llvm/CodeGen/MachineScheduler.h
+++ b/llvm/include/llvm/CodeGen/MachineScheduler.h
@@ -1296,6 +1296,11 @@ protected:
   SchedBoundary Bot;
   MachineSchedPolicy RegionPolicy;
 
+  /// Candidate last picked from Top boundary.
+  SchedCandidate TopCand;
+  /// Candidate last picked from Bot boundary.
+  SchedCandidate BotCand;
+
 public:
   PostGenericScheduler(const MachineSchedContext *C)
       : GenericSchedulerBase(C), Top(SchedBoundary::TopQID, "TopQ"),
@@ -1316,6 +1321,8 @@ public:
 
   SUnit *pickNode(bool &IsTopNode) override;
 
+  SUnit *pickNodeBidirectional(bool &IsTopNode);
+
   void scheduleTree(unsigned SubtreeID) override {
     llvm_unreachable("PostRA scheduler does not support subtree analysis.");
   }
@@ -1326,12 +1333,14 @@ public:
     if (SU->isScheduled)
       return;
     Top.releaseNode(SU, SU->TopReadyCycle, false);
+    TopCand.SU = nullptr;
   }
 
   void releaseBottomNode(SUnit *SU) override {
     if (SU->isScheduled)
       return;
     Bot.releaseNode(SU, SU->BotReadyCycle, false);
+    BotCand.SU = nullptr;
   }
 
 protected:
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 0d5bf3299387..cbeb02a72552 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -85,6 +85,7 @@ namespace MISchedPostRASched {
 enum Direction {
   TopDown,
   BottomUp,
+  Bidirectional,
 };
 } // end namespace MISchedPostRASched
 cl::opt<MISchedPostRASched::Direction> PostRADirection(
@@ -93,10 +94,13 @@ cl::opt<MISchedPostRASched::Direction> PostRADirection(
     // Default to top-down because it was implemented first and existing targets
     // expect that behavior by default.
     cl::init(MISchedPostRASched::TopDown),
-    cl::values(clEnumValN(MISchedPostRASched::TopDown, "topdown",
-                          "Force top-down post reg-alloc list scheduling"),
-               clEnumValN(MISchedPostRASched::BottomUp, "bottomup",
-                          "Force bottom-up post reg-alloc list scheduling")));
+    cl::values(
+        clEnumValN(MISchedPostRASched::TopDown, "topdown",
+                   "Force top-down post reg-alloc list scheduling"),
+        clEnumValN(MISchedPostRASched::BottomUp, "bottomup",
+                   "Force bottom-up post reg-alloc list scheduling"),
+        clEnumValN(MISchedPostRASched::Bidirectional, "bidirectional",
+                   "Force bidirectional post reg-alloc list scheduling")));
 cl::opt<bool>
 DumpCriticalPathLength("misched-dcpl", cl::Hidden,
                        cl::desc("Print critical path length to stdout"));
@@ -500,8 +504,10 @@ bool PostMachineScheduler::runOnMachineFunction(MachineFunction &mf) {
   ScheduleDAGMI::DumpDirection D;
   if (PostRADirection == MISchedPostRASched::TopDown)
     D = ScheduleDAGMI::DumpDirection::TopDown;
-  else
+  else if (PostRADirection == MISchedPostRASched::BottomUp)
     D = ScheduleDAGMI::DumpDirection::BottomUp;
+  else
+    D = ScheduleDAGMI::DumpDirection::Bidirectional;
   Scheduler->setDumpDirection(D);
   scheduleRegions(*Scheduler, true);
 
@@ -3715,7 +3721,7 @@ SUnit *GenericScheduler::pickNodeBidirectional(bool &IsTopNode) {
       TCand.reset(CandPolicy());
       pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TCand);
       assert(TCand.SU == TopCand.SU &&
-           "Last pick result should correspond to re-picking right now");
+             "Last pick result should correspond to re-picking right now");
     }
 #endif
   }
@@ -3891,6 +3897,9 @@ void PostGenericScheduler::initPolicy(MachineBasicBlock::iterator Begin,
   } else if (PostRADirection == MISchedPostRASched::BottomUp) {
     RegionPolicy.OnlyTopDown = false;
     RegionPolicy.OnlyBottomUp = true;
+  } else if (PostRADirection == MISchedPostRASched::Bidirectional) {
+    RegionPolicy.OnlyBottomUp = false;
+    RegionPolicy.OnlyTopDown = false;
   }
 }
 
@@ -3970,6 +3979,87 @@ void PostGenericScheduler::pickNodeFromQueue(SchedBoundary &Zone,
   }
 }
 
+/// Pick the best candidate node from either the top or bottom queue.
+SUnit *PostGenericScheduler::pickNodeBidirectional(bool &IsTopNode) {
+  // FIXME: This is similiar to GenericScheduler::pickNodeBidirectional. Factor
+  // out common parts.
+
+  // Schedule as far as possible in the direction of no choice. This is most
+  // efficient, but also provides the best heuristics for CriticalPSets.
+  if (SUnit *SU = Bot.pickOnlyChoice()) {
+    IsTopNode = false;
+    tracePick(Only1, false);
+    return SU;
+  }
+  if (SUnit *SU = Top.pickOnlyChoice()) {
+    IsTopNode = true;
+    tracePick(Only1, true);
+    return SU;
+  }
+  // Set the bottom-up policy based on the state of the current bottom zone and
+  // the instructions outside the zone, including the top zone.
+  CandPolicy BotPolicy;
+  setPolicy(BotPolicy, /*IsPostRA=*/true, Bot, &Top);
+  // Set the top-down policy based on the state of the current top zone and
+  // the instructions outside the zone, including the bottom zone.
+  CandPolicy TopPolicy;
+  setPolicy(TopPolicy, /*IsPostRA=*/true, Top, &Bot);
+
+  // See if BotCand is still valid (because we previously scheduled from Top).
+  LLVM_DEBUG(dbgs() << "Picking from Bot:\n");
+  if (!BotCand.isValid() || BotCand.SU->isScheduled ||
+      BotCand.Policy != BotPolicy) {
+    BotCand.reset(CandPolicy());
+    pickNodeFromQueue(Bot, BotCand);
+    assert(BotCand.Reason != NoCand && "failed to find the first candidate");
+  } else {
+    LLVM_DEBUG(traceCandidate(BotCand));
+#ifndef NDEBUG
+    if (VerifyScheduling) {
+      SchedCandidate TCand;
+      TCand.reset(CandPolicy());
+      pickNodeFromQueue(Bot, BotCand);
+      assert(TCand.SU == BotCand.SU &&
+             "Last pick result should correspond to re-picking right now");
+    }
+#endif
+  }
+
+  // Check if the top Q has a better candidate.
+  LLVM_DEBUG(dbgs() << "Picking from Top:\n");
+  if (!TopCand.isValid() || TopCand.SU->isScheduled ||
+      TopCand.Policy != TopPolicy) {
+    TopCand.reset(CandPolicy());
+    pickNodeFromQueue(Top, TopCand);
+    assert(TopCand.Reason != NoCand && "failed to find the first candidate");
+  } else {
+    LLVM_DEBUG(traceCandidate(TopCand));
+#ifndef NDEBUG
+    if (VerifyScheduling) {
+      SchedCandidate TCand;
+      TCand.reset(CandPolicy());
+      pickNodeFromQueue(Top, TopCand);
+      assert(TCand.SU == TopCand.SU &&
+             "Last pick result should correspond to re-picking right now");
+    }
+#endif
+  }
+
+  // Pick best from BotCand and TopCand.
+  assert(BotCand.isValid());
+  assert(TopCand.isValid());
+  SchedCandidate Cand = BotCand;
+  TopCand.Reason = NoCand;
+  if (tryCandidate(Cand, TopCand)) {
+    Cand.setBest(TopCand);
+    LLVM_DEBUG(traceCandidate(Cand));
+  }
+
+  IsTopNode = Cand.AtTop;
+  tracePick(Cand);
+  return Cand.SU;
+}
+
 /// Pick the next node to schedule.
 SUnit *PostGenericScheduler::pickNode(bool &IsTopNode) {
   if (DAG->top() == DAG->bottom()) {
@@ -3980,13 +4070,12 @@ SUnit *PostGenericScheduler::pickNode(bool &IsTopNode) {
   SUnit *SU;
   do {
     if (RegionPolicy.OnlyBottomUp) {
-      assert(!RegionPolicy.OnlyTopDown);
       SU = Bot.pickOnlyChoice();
       if (SU) {
         tracePick(Only1, true);
       } else {
         CandPolicy NoPolicy;
-        SchedCandidate BotCand(NoPolicy);
+        BotCand.reset(NoPolicy);
         // Set the bottom-up policy based on the state of the current bottom
         // zone and the instructions outside the zone, including the top zone.
         setPolicy(BotCand.Policy, /*IsPostRA=*/true, Bot, nullptr);
@@ -3996,15 +4085,13 @@ SUnit *PostGenericScheduler::pickNode(bool &IsTopNode) {
         SU = BotCand.SU;
       }
       IsTopNode = false;
-    } else {
-
-      assert(RegionPolicy.OnlyTopDown);
+    } else if (RegionPolicy.OnlyTopDown) {
       SU = Top.pickOnlyChoice();
       if (SU) {
         tracePick(Only1, true);
       } else {
         CandPolicy NoPolicy;
-        SchedCandidate TopCand(NoPolicy);
+        TopCand.reset(NoPolicy);
         // Set the top-down policy based on the state of the current top zone
         // and the instructions outside the zone, including the bottom zone.
         setPolicy(TopCand.Policy, /*IsPostRA=*/true, Top, nullptr);
@@ -4014,6 +4101,8 @@ SUnit *PostGenericScheduler::pickNode(bool &IsTopNode) {
         SU = TopCand.SU;
       }
       IsTopNode = true;
+    } else {
+      SU = pickNodeBidirectional(IsTopNode);
     }
   } while (SU->isScheduled);
 
diff --git a/llvm/test/CodeGen/RISCV/misched-postra-direction.mir b/llvm/test/CodeGen/RISCV/misched-postra-direction.mir
index 841d0e6d65da..2cca042bebee 100644
--- a/llvm/test/CodeGen/RISCV/misched-postra-direction.mir
+++ b/llvm/test/CodeGen/RISCV/misched-postra-direction.mir
@@ -1,5 +1,15 @@
-# RUN: llc -mtriple=riscv64 -mcpu=sifive-x280 -run-pass=postmisched -enable-post-misched -debug-only=machine-scheduler -misched-dump-schedule-trace -misched-postra-direction=topdown -o - %s 2>&1 | FileCheck --check-prefix=TOPDOWN %s
-# RUN: llc -mtriple=riscv64 -mcpu=sifive-x280 -run-pass=postmisched -enable-post-misched -debug-only=machine-scheduler -misched-dump-schedule-trace -misched-postra-direction=bottomup -o - %s 2>&1 | FileCheck --check-prefix=BOTTOMUP %s
+# RUN: llc -mtriple=riscv64 -mcpu=sifive-x280 -run-pass=postmisched \
+# RUN:   -enable-post-misched -debug-only=machine-scheduler \
+# RUN:  -misched-dump-schedule-trace -misched-postra-direction=topdown \
+# RUN:  -o - %s 2>&1 | FileCheck --check-prefix=TOPDOWN %s
+# RUN: llc -mtriple=riscv64 -mcpu=sifive-x280 -run-pass=postmisched \
+# RUN:   -enable-post-misched -debug-only=machine-scheduler \
+# RUN:   -misched-dump-schedule-trace -misched-postra-direction=bottomup \
+# RUN:   -o - %s 2>&1 | FileCheck --check-prefix=BOTTOMUP %s
+# RUN: llc -mtriple=riscv64 -mcpu=sifive-x280 -run-pass=postmisched \
+# RUN:   -enable-post-misched -debug-only=machine-scheduler \
+# RUN:   -misched-dump-schedule-trace -misched-postra-direction=bidirectional \
+# RUN:   -o - %s 2>&1 | FileCheck --check-prefix=BIDIRECTIONAL %s
 
 # REQUIRES: asserts
 
@@ -51,3 +61,9 @@ body:             |
 # BOTTOMUP-NEXT: SU(1):   renamable $x13 = ADD renamable $x11, renamable $x10
 # BOTTOMUP-NEXT: SU(0):   renamable $x12 = MUL renamable $x11, renamable $x10
 # BOTTOMUP-NEXT: SU(2):   renamable $x14 = DIVW renamable $x12, renamable $x13
+
+# BIDIRECTIONAL: *** Final schedule for %bb.0 ***
+# BIDIRECTIONAL-NEXT: * Schedule table (Bidirectional): not implemented
+# BIDIRECTIONAL-NEXT: SU(1):   renamable $x13 = ADD renamable $x11, renamable $x10
+# BIDIRECTIONAL-NEXT: SU(0):   renamable $x12 = MUL renamable $x11, renamable $x10
+# BIDIRECTIONAL-NEXT: SU(2):   renamable $x14 = DIVW renamable $x12, renamable $x13
-- 
GitLab


From ded6252dfd96ee8904b6d2250bc9e4841ca5c802 Mon Sep 17 00:00:00 2001
From: Chris B <chris.bieneman@me.com>
Date: Mon, 25 Mar 2024 09:16:18 -0500
Subject: [PATCH 125/404] [CMake][HLSL] Add SPIRV to target list for build
 (#86323)

This change just enables the SPIR-V target by default in the HLSL build
configurations. Since SPIR-V support is something we expect from the
full HLSL compiler releases for pairity with DXC we should enable it in
the default developer workflow.
---
 clang/cmake/caches/HLSL.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/cmake/caches/HLSL.cmake b/clang/cmake/caches/HLSL.cmake
index 71f81e53f6bd..84850c86f12c 100644
--- a/clang/cmake/caches/HLSL.cmake
+++ b/clang/cmake/caches/HLSL.cmake
@@ -4,7 +4,7 @@ set(LLVM_TARGETS_TO_BUILD Native CACHE STRING "")
 
 # Include the DirectX target for DXIL code generation, eventually we'll include
 # SPIR-V here too.
-set(LLVM_EXPERIMENTAL_TARGETS_TO_BUILD DirectX CACHE STRING "")
+set(LLVM_EXPERIMENTAL_TARGETS_TO_BUILD "DirectX;SPIRV" CACHE STRING "")
 
 # HLSL support is currently limted to clang, eventually it will expand to
 # clang-tools-extra too.
-- 
GitLab


From 32b828306e346bc8e86c0b763f3514329de60ed1 Mon Sep 17 00:00:00 2001
From: Balazs Benics <benicsbalazs@gmail.com>
Date: Mon, 25 Mar 2024 15:24:03 +0100
Subject: [PATCH 126/404] [analyzer] Set and display CSA analysis entry points
 as notes on debugging (#84823)

When debugging CSA issues, sometimes it would be useful to have a
dedicated note for the analysis entry point, aka. the function name you
would need to pass as "-analyze-function=XYZ" to reproduce a specific
issue.
One way we use (or will use) this downstream is to provide tooling on
top of creduce to enhance to supercharge productivity by automatically
reduce cases on crashes for example.

This will be added only if the "-analyzer-note-analysis-entry-points" is
set or the "analyzer-display-progress" is on.

This additional entry point marker will be the first "note" if enabled,
with the following message: "[debug] analyzing from XYZ". They are
prefixed by "[debug]" to remind the CSA developer that this is only
 meant to be visible for them, for debugging purposes.

CPP-5012
---
 clang/include/clang/Analysis/PathDiagnostic.h |  8 +-
 clang/include/clang/Driver/Options.td         |  3 +
 .../StaticAnalyzer/Core/AnalyzerOptions.h     |  9 ++-
 .../Core/BugReporter/BugReporter.h            | 12 +++
 .../Core/PathSensitive/ExprEngine.h           |  2 +
 clang/lib/Analysis/PathDiagnostic.cpp         |  7 +-
 clang/lib/StaticAnalyzer/Core/BugReporter.cpp | 36 ++++++---
 .../Frontend/AnalysisConsumer.cpp             |  4 +-
 .../Analysis/analyzer-display-progress.cpp    | 42 ++++++++---
 .../test/Analysis/analyzer-display-progress.m | 31 +++++---
 .../analyzer-note-analysis-entry-points.cpp   | 75 +++++++++++++++++++
 11 files changed, 193 insertions(+), 36 deletions(-)
 create mode 100644 clang/test/Analysis/analyzer-note-analysis-entry-points.cpp

diff --git a/clang/include/clang/Analysis/PathDiagnostic.h b/clang/include/clang/Analysis/PathDiagnostic.h
index 90559e7efb06..5907df022e44 100644
--- a/clang/include/clang/Analysis/PathDiagnostic.h
+++ b/clang/include/clang/Analysis/PathDiagnostic.h
@@ -780,6 +780,9 @@ class PathDiagnostic : public llvm::FoldingSetNode {
   PathDiagnosticLocation UniqueingLoc;
   const Decl *UniqueingDecl;
 
+  /// The top-level entry point from which this issue was discovered.
+  const Decl *AnalysisEntryPoint = nullptr;
+
   /// Lines executed in the path.
   std::unique_ptr<FilesToLineNumsMap> ExecutedLines;
 
@@ -788,7 +791,7 @@ public:
   PathDiagnostic(StringRef CheckerName, const Decl *DeclWithIssue,
                  StringRef bugtype, StringRef verboseDesc, StringRef shortDesc,
                  StringRef category, PathDiagnosticLocation LocationToUnique,
-                 const Decl *DeclToUnique,
+                 const Decl *DeclToUnique, const Decl *AnalysisEntryPoint,
                  std::unique_ptr<FilesToLineNumsMap> ExecutedLines);
   ~PathDiagnostic();
 
@@ -852,6 +855,9 @@ public:
     return *ExecutedLines;
   }
 
+  /// Get the top-level entry point from which this issue was discovered.
+  const Decl *getAnalysisEntryPoint() const { return AnalysisEntryPoint; }
+
   /// Return the semantic context where an issue occurred.  If the
   /// issue occurs along a path, this represents the "central" area
   /// where the bug manifests.
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 4a954258ce40..b0d90c776b58 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -6688,6 +6688,9 @@ def analyzer_opt_analyze_headers : Flag<["-"], "analyzer-opt-analyze-headers">,
 def analyzer_display_progress : Flag<["-"], "analyzer-display-progress">,
   HelpText<"Emit verbose output about the analyzer's progress">,
   MarshallingInfoFlag<AnalyzerOpts<"AnalyzerDisplayProgress">>;
+def analyzer_note_analysis_entry_points : Flag<["-"], "analyzer-note-analysis-entry-points">,
+  HelpText<"Add a note for each bug report to denote their analysis entry points">,
+  MarshallingInfoFlag<AnalyzerOpts<"AnalyzerNoteAnalysisEntryPoints">>;
 def analyze_function : Separate<["-"], "analyze-function">,
   HelpText<"Run analysis on specific function (for C++ include parameters in name)">,
   MarshallingInfoString<AnalyzerOpts<"AnalyzeSpecificFunction">>;
diff --git a/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.h b/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.h
index 276d11e80a5b..3a3c1a13d67d 100644
--- a/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.h
+++ b/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.h
@@ -227,6 +227,7 @@ public:
   unsigned ShouldEmitErrorsOnInvalidConfigValue : 1;
   unsigned AnalyzeAll : 1;
   unsigned AnalyzerDisplayProgress : 1;
+  unsigned AnalyzerNoteAnalysisEntryPoints : 1;
 
   unsigned eagerlyAssumeBinOpBifurcation : 1;
 
@@ -291,10 +292,10 @@ public:
         ShowCheckerOptionDeveloperList(false), ShowEnabledCheckerList(false),
         ShowConfigOptionsList(false),
         ShouldEmitErrorsOnInvalidConfigValue(false), AnalyzeAll(false),
-        AnalyzerDisplayProgress(false), eagerlyAssumeBinOpBifurcation(false),
-        TrimGraph(false), visualizeExplodedGraphWithGraphViz(false),
-        UnoptimizedCFG(false), PrintStats(false), NoRetryExhausted(false),
-        AnalyzerWerror(false) {}
+        AnalyzerDisplayProgress(false), AnalyzerNoteAnalysisEntryPoints(false),
+        eagerlyAssumeBinOpBifurcation(false), TrimGraph(false),
+        visualizeExplodedGraphWithGraphViz(false), UnoptimizedCFG(false),
+        PrintStats(false), NoRetryExhausted(false), AnalyzerWerror(false) {}
 
   /// Interprets an option's string value as a boolean. The "true" string is
   /// interpreted as true and the "false" string is interpreted as false.
diff --git a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
index e762f7548e0b..ead96ce6891c 100644
--- a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
+++ b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
@@ -586,6 +586,9 @@ class BugReporter {
 private:
   BugReporterData& D;
 
+  /// The top-level entry point for the issue to be reported.
+  const Decl *AnalysisEntryPoint = nullptr;
+
   /// Generate and flush the diagnostics for the given bug report.
   void FlushReport(BugReportEquivClass& EQ);
 
@@ -623,6 +626,14 @@ public:
 
   Preprocessor &getPreprocessor() { return D.getPreprocessor(); }
 
+  /// Get the top-level entry point for the issue to be reported.
+  const Decl *getAnalysisEntryPoint() const { return AnalysisEntryPoint; }
+
+  void setAnalysisEntryPoint(const Decl *EntryPoint) {
+    assert(EntryPoint);
+    AnalysisEntryPoint = EntryPoint;
+  }
+
   /// Add the given report to the set of reports tracked by BugReporter.
   ///
   /// The reports are usually generated by the checkers. Further, they are
@@ -713,6 +724,7 @@ public:
   virtual ~BugReporterContext() = default;
 
   PathSensitiveBugReporter& getBugReporter() { return BR; }
+  const PathSensitiveBugReporter &getBugReporter() const { return BR; }
 
   ProgramStateManager& getStateManager() const {
     return BR.getStateManager();
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
index 859c1497d7e6..e38a3bb56ece 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
@@ -187,6 +187,8 @@ public:
 
   /// Returns true if there is still simulation state on the worklist.
   bool ExecuteWorkList(const LocationContext *L, unsigned Steps = 150000) {
+    assert(L->inTopFrame());
+    BR.setAnalysisEntryPoint(L->getDecl());
     return Engine.ExecuteWorkList(L, Steps, nullptr);
   }
 
diff --git a/clang/lib/Analysis/PathDiagnostic.cpp b/clang/lib/Analysis/PathDiagnostic.cpp
index 79f337a91ec8..35472e705cfd 100644
--- a/clang/lib/Analysis/PathDiagnostic.cpp
+++ b/clang/lib/Analysis/PathDiagnostic.cpp
@@ -115,14 +115,17 @@ PathDiagnostic::PathDiagnostic(
     StringRef CheckerName, const Decl *declWithIssue, StringRef bugtype,
     StringRef verboseDesc, StringRef shortDesc, StringRef category,
     PathDiagnosticLocation LocationToUnique, const Decl *DeclToUnique,
+    const Decl *AnalysisEntryPoint,
     std::unique_ptr<FilesToLineNumsMap> ExecutedLines)
     : CheckerName(CheckerName), DeclWithIssue(declWithIssue),
       BugType(StripTrailingDots(bugtype)),
       VerboseDesc(StripTrailingDots(verboseDesc)),
       ShortDesc(StripTrailingDots(shortDesc)),
       Category(StripTrailingDots(category)), UniqueingLoc(LocationToUnique),
-      UniqueingDecl(DeclToUnique), ExecutedLines(std::move(ExecutedLines)),
-      path(pathImpl) {}
+      UniqueingDecl(DeclToUnique), AnalysisEntryPoint(AnalysisEntryPoint),
+      ExecutedLines(std::move(ExecutedLines)), path(pathImpl) {
+  assert(AnalysisEntryPoint);
+}
 
 void PathDiagnosticConsumer::anchor() {}
 
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
index 3617fdd778e3..14ca507a16d5 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
@@ -138,7 +138,8 @@ public:
 public:
   PathDiagnosticConstruct(const PathDiagnosticConsumer *PDC,
                           const ExplodedNode *ErrorNode,
-                          const PathSensitiveBugReport *R);
+                          const PathSensitiveBugReport *R,
+                          const Decl *AnalysisEntryPoint);
 
   /// \returns the location context associated with the current position in the
   /// bug path.
@@ -1323,24 +1324,26 @@ void PathDiagnosticBuilder::generatePathDiagnosticsForNode(
 }
 
 static std::unique_ptr<PathDiagnostic>
-generateDiagnosticForBasicReport(const BasicBugReport *R) {
+generateDiagnosticForBasicReport(const BasicBugReport *R,
+                                 const Decl *AnalysisEntryPoint) {
   const BugType &BT = R->getBugType();
   return std::make_unique<PathDiagnostic>(
       BT.getCheckerName(), R->getDeclWithIssue(), BT.getDescription(),
       R->getDescription(), R->getShortDescription(/*UseFallback=*/false),
       BT.getCategory(), R->getUniqueingLocation(), R->getUniqueingDecl(),
-      std::make_unique<FilesToLineNumsMap>());
+      AnalysisEntryPoint, std::make_unique<FilesToLineNumsMap>());
 }
 
 static std::unique_ptr<PathDiagnostic>
 generateEmptyDiagnosticForReport(const PathSensitiveBugReport *R,
-                                 const SourceManager &SM) {
+                                 const SourceManager &SM,
+                                 const Decl *AnalysisEntryPoint) {
   const BugType &BT = R->getBugType();
   return std::make_unique<PathDiagnostic>(
       BT.getCheckerName(), R->getDeclWithIssue(), BT.getDescription(),
       R->getDescription(), R->getShortDescription(/*UseFallback=*/false),
       BT.getCategory(), R->getUniqueingLocation(), R->getUniqueingDecl(),
-      findExecutedLines(SM, R->getErrorNode()));
+      AnalysisEntryPoint, findExecutedLines(SM, R->getErrorNode()));
 }
 
 static const Stmt *getStmtParent(const Stmt *S, const ParentMap &PM) {
@@ -1976,10 +1979,11 @@ static void updateExecutedLinesWithDiagnosticPieces(PathDiagnostic &PD) {
 
 PathDiagnosticConstruct::PathDiagnosticConstruct(
     const PathDiagnosticConsumer *PDC, const ExplodedNode *ErrorNode,
-    const PathSensitiveBugReport *R)
+    const PathSensitiveBugReport *R, const Decl *AnalysisEntryPoint)
     : Consumer(PDC), CurrentNode(ErrorNode),
       SM(CurrentNode->getCodeDecl().getASTContext().getSourceManager()),
-      PD(generateEmptyDiagnosticForReport(R, getSourceManager())) {
+      PD(generateEmptyDiagnosticForReport(R, getSourceManager(),
+                                          AnalysisEntryPoint)) {
   LCM[&PD->getActivePath()] = ErrorNode->getLocationContext();
 }
 
@@ -1993,13 +1997,14 @@ PathDiagnosticBuilder::PathDiagnosticBuilder(
 
 std::unique_ptr<PathDiagnostic>
 PathDiagnosticBuilder::generate(const PathDiagnosticConsumer *PDC) const {
-  PathDiagnosticConstruct Construct(PDC, ErrorNode, R);
+  const Decl *EntryPoint = getBugReporter().getAnalysisEntryPoint();
+  PathDiagnosticConstruct Construct(PDC, ErrorNode, R, EntryPoint);
 
   const SourceManager &SM = getSourceManager();
   const AnalyzerOptions &Opts = getAnalyzerOptions();
 
   if (!PDC->shouldGenerateDiagnostics())
-    return generateEmptyDiagnosticForReport(R, getSourceManager());
+    return generateEmptyDiagnosticForReport(R, getSourceManager(), EntryPoint);
 
   // Construct the final (warning) event for the bug report.
   auto EndNotes = VisitorsDiagnostics->find(ErrorNode);
@@ -3123,6 +3128,16 @@ void BugReporter::FlushReport(BugReportEquivClass& EQ) {
       Pieces.back()->addFixit(I);
 
     updateExecutedLinesWithDiagnosticPieces(*PD);
+
+    // If we are debugging, let's have the entry point as the first note.
+    if (getAnalyzerOptions().AnalyzerDisplayProgress ||
+        getAnalyzerOptions().AnalyzerNoteAnalysisEntryPoints) {
+      const Decl *EntryPoint = getAnalysisEntryPoint();
+      Pieces.push_front(std::make_shared<PathDiagnosticEventPiece>(
+          PathDiagnosticLocation{EntryPoint->getLocation(), getSourceManager()},
+          "[debug] analyzing from " +
+              AnalysisDeclContext::getFunctionName(EntryPoint)));
+    }
     Consumer->HandlePathDiagnostic(std::move(PD));
   }
 }
@@ -3211,7 +3226,8 @@ BugReporter::generateDiagnosticForConsumerMap(
   auto *basicReport = cast<BasicBugReport>(exampleReport);
   auto Out = std::make_unique<DiagnosticForConsumerMapTy>();
   for (auto *Consumer : consumers)
-    (*Out)[Consumer] = generateDiagnosticForBasicReport(basicReport);
+    (*Out)[Consumer] =
+        generateDiagnosticForBasicReport(basicReport, AnalysisEntryPoint);
   return Out;
 }
 
diff --git a/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp b/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
index b6ef40595e3c..03bc40804d73 100644
--- a/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
+++ b/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
@@ -527,7 +527,8 @@ static void reportAnalyzerFunctionMisuse(const AnalyzerOptions &Opts,
 
 void AnalysisConsumer::runAnalysisOnTranslationUnit(ASTContext &C) {
   BugReporter BR(*Mgr);
-  TranslationUnitDecl *TU = C.getTranslationUnitDecl();
+  const TranslationUnitDecl *TU = C.getTranslationUnitDecl();
+  BR.setAnalysisEntryPoint(TU);
   if (SyntaxCheckTimer)
     SyntaxCheckTimer->startTimer();
   checkerMgr->runCheckersOnASTDecl(TU, *Mgr, BR);
@@ -675,6 +676,7 @@ void AnalysisConsumer::HandleCode(Decl *D, AnalysisMode Mode,
 
   DisplayFunction(D, Mode, IMode);
   BugReporter BR(*Mgr);
+  BR.setAnalysisEntryPoint(D);
 
   if (Mode & AM_Syntax) {
     llvm::TimeRecord CheckerStartTime;
diff --git a/clang/test/Analysis/analyzer-display-progress.cpp b/clang/test/Analysis/analyzer-display-progress.cpp
index dc8e27a8c3b4..fa1860004d03 100644
--- a/clang/test/Analysis/analyzer-display-progress.cpp
+++ b/clang/test/Analysis/analyzer-display-progress.cpp
@@ -1,22 +1,46 @@
-// RUN: %clang_analyze_cc1 -analyzer-display-progress %s 2>&1 | FileCheck %s
+// RUN: %clang_analyze_cc1 -verify %s 2>&1 \
+// RUN:   -analyzer-display-progress \
+// RUN:   -analyzer-checker=debug.ExprInspection \
+// RUN:   -analyzer-output=text \
+// RUN: | FileCheck %s
 
-void f() {};
-void g() {};
-void h() {}
+void clang_analyzer_warnIfReached();
+
+// expected-note@+2 {{[debug] analyzing from f()}}
+// expected-warning@+1 {{REACHABLE}} expected-note@+1 {{REACHABLE}}
+void f() { clang_analyzer_warnIfReached(); }
+
+// expected-note@+2 {{[debug] analyzing from g()}}
+// expected-warning@+1 {{REACHABLE}} expected-note@+1 {{REACHABLE}}
+void g() { clang_analyzer_warnIfReached(); }
+
+// expected-note@+2 {{[debug] analyzing from h()}}
+// expected-warning@+1 {{REACHABLE}} expected-note@+1 {{REACHABLE}}
+void h() { clang_analyzer_warnIfReached(); }
 
 struct SomeStruct {
-  void f() {}
+  // expected-note@+2 {{[debug] analyzing from SomeStruct::f()}}
+  // expected-warning@+1 {{REACHABLE}} expected-note@+1 {{REACHABLE}}
+  void f() { clang_analyzer_warnIfReached(); }
 };
 
 struct SomeOtherStruct {
-  void f() {}
+  // expected-note@+2 {{[debug] analyzing from SomeOtherStruct::f()}}
+  // expected-warning@+1 {{REACHABLE}} expected-note@+1 {{REACHABLE}}
+  void f() { clang_analyzer_warnIfReached(); }
 };
 
 namespace ns {
   struct SomeStruct {
-    void f(int) {}
-    void f(float, ::SomeStruct) {}
-    void f(float, SomeStruct) {}
+    // expected-note@+2 {{[debug] analyzing from ns::SomeStruct::f(int)}}
+    // expected-warning@+1 {{REACHABLE}} expected-note@+1 {{REACHABLE}}
+    void f(int) { clang_analyzer_warnIfReached(); }
+    // expected-note@+2 {{[debug] analyzing from ns::SomeStruct::f(float, ::SomeStruct)}}
+    // expected-warning@+1 {{REACHABLE}} expected-note@+1 {{REACHABLE}}
+    void f(float, ::SomeStruct) { clang_analyzer_warnIfReached(); }
+    // expected-note@+2 {{[debug] analyzing from ns::SomeStruct::f(float, SomeStruct)}}
+    // expected-warning@+1 {{REACHABLE}} expected-note@+1 {{REACHABLE}}
+    void f(float, SomeStruct) { clang_analyzer_warnIfReached(); }
   };
 }
 
diff --git a/clang/test/Analysis/analyzer-display-progress.m b/clang/test/Analysis/analyzer-display-progress.m
index 24414f659c39..90f223b34861 100644
--- a/clang/test/Analysis/analyzer-display-progress.m
+++ b/clang/test/Analysis/analyzer-display-progress.m
@@ -1,8 +1,16 @@
-// RUN: %clang_analyze_cc1 -fblocks -analyzer-display-progress %s 2>&1 | FileCheck %s
+// RUN: %clang_analyze_cc1 -fblocks -verify %s 2>&1 \
+// RUN:   -analyzer-display-progress \
+// RUN:   -analyzer-checker=debug.ExprInspection \
+// RUN:   -analyzer-output=text \
+// RUN: | FileCheck %s
 
 #include "Inputs/system-header-simulator-objc.h"
 
-static void f(void) {}
+void clang_analyzer_warnIfReached();
+
+// expected-note@+2 {{[debug] analyzing from f}}
+// expected-warning@+1 {{REACHABLE}} expected-note@+1 {{REACHABLE}}
+static void f(void) { clang_analyzer_warnIfReached(); }
 
 @interface I: NSObject
 -(void)instanceMethod:(int)arg1 with:(int)arg2;
@@ -10,21 +18,26 @@ static void f(void) {}
 @end
 
 @implementation I
--(void)instanceMethod:(int)arg1 with:(int)arg2 {}
-+(void)classMethod {}
+// expected-warning@+1 {{REACHABLE}} expected-note@+1 {{REACHABLE}}
+-(void)instanceMethod:(int)arg1 with:(int)arg2 { clang_analyzer_warnIfReached(); }
+
+// expected-warning@+1 {{REACHABLE}} expected-note@+1 {{REACHABLE}}
++(void)classMethod { clang_analyzer_warnIfReached(); }
 @end
 
+// expected-note@+1 3 {{[debug] analyzing from g}}
 void g(I *i, int x, int y) {
-  [I classMethod];
-  [i instanceMethod: x with: y];
+  [I classMethod]; // expected-note {{Calling 'classMethod'}}
+  [i instanceMethod: x with: y]; // expected-note {{Calling 'instanceMethod:with:'}}
 
   void (^block)(void);
-  block = ^{};
-  block();
+  // expected-warning@+1 {{REACHABLE}} expected-note@+1 {{REACHABLE}}
+  block = ^{ clang_analyzer_warnIfReached(); };
+  block(); // expected-note {{Calling anonymous block}}
 }
 
 // CHECK: analyzer-display-progress.m f
 // CHECK: analyzer-display-progress.m -[I instanceMethod:with:]
 // CHECK: analyzer-display-progress.m +[I classMethod]
 // CHECK: analyzer-display-progress.m g
-// CHECK: analyzer-display-progress.m block (line: 22, col: 11)
+// CHECK: analyzer-display-progress.m block (line: 35, col: 11)
diff --git a/clang/test/Analysis/analyzer-note-analysis-entry-points.cpp b/clang/test/Analysis/analyzer-note-analysis-entry-points.cpp
new file mode 100644
index 000000000000..7d321bfae61c
--- /dev/null
+++ b/clang/test/Analysis/analyzer-note-analysis-entry-points.cpp
@@ -0,0 +1,75 @@
+// RUN: %clang_analyze_cc1 -verify=common %s \
+// RUN:   -analyzer-checker=deadcode.DeadStores,debug.ExprInspection \
+// RUN:   -analyzer-note-analysis-entry-points
+
+// RUN: %clang_analyze_cc1 -verify=common,textout %s \
+// RUN:   -analyzer-checker=deadcode.DeadStores,debug.ExprInspection \
+// RUN:   -analyzer-note-analysis-entry-points \
+// RUN:   -analyzer-output=text
+
+// Test the actual source locations/ranges of entry point notes.
+// RUN: %clang_analyze_cc1 %s \
+// RUN:   -analyzer-checker=deadcode.DeadStores,debug.ExprInspection \
+// RUN:   -analyzer-note-analysis-entry-points \
+// RUN:   -analyzer-output=text 2>&1 \
+// RUN: | FileCheck --strict-whitespace %s
+
+
+void clang_analyzer_warnIfReached();
+
+void other() {
+  // common-warning@+1 {{REACHABLE}} textout-note@+1 {{REACHABLE}}
+  clang_analyzer_warnIfReached();
+}
+
+struct SomeOtherStruct {
+  // CHECK: note: [debug] analyzing from SomeOtherStruct::f()
+  // CHECK-NEXT: |   void f() {
+  // CHECK-NEXT: |        ^
+  // textout-note@+1 {{[debug] analyzing from SomeOtherStruct::f()}}
+  void f() {
+    other(); // textout-note {{Calling 'other'}}
+  }
+};
+
+// CHECK: note: [debug] analyzing from operator""_w(const char *)
+// CHECK-NEXT: | unsigned operator ""_w(const char*) {
+// CHECK-NEXT: |          ^
+// textout-note@+1 {{[debug] analyzing from operator""_w(const char *)}}
+unsigned operator ""_w(const char*) {
+  // common-warning@+1 {{REACHABLE}} textout-note@+1 {{REACHABLE}}
+  clang_analyzer_warnIfReached();
+  return 404;
+}
+
+// textout-note@+1 {{[debug] analyzing from checkASTCodeBodyHasAnalysisEntryPoints()}}
+void checkASTCodeBodyHasAnalysisEntryPoints() {
+  int z = 1;
+  z = 2;
+  // common-warning@-1 {{Value stored to 'z' is never read}}
+  // textout-note@-2    {{Value stored to 'z' is never read}}
+}
+
+void notInvokedLambdaScope() {
+  // CHECK: note: [debug] analyzing from notInvokedLambdaScope()::(anonymous class)::operator()()
+  // CHECK-NEXT: |   auto notInvokedLambda = []() {
+  // CHECK-NEXT: |                           ^
+  // textout-note@+1 {{[debug] analyzing from notInvokedLambdaScope()::(anonymous class)::operator()()}}
+  auto notInvokedLambda = []() {
+    // common-warning@+1 {{REACHABLE}} textout-note@+1 {{REACHABLE}}
+    clang_analyzer_warnIfReached();
+  };
+  (void)notInvokedLambda; // Not invoking the lambda.
+}
+
+// CHECK: note: [debug] analyzing from invokedLambdaScope()
+// CHECK-NEXT: | void invokedLambdaScope() {
+// CHECK-NEXT: |      ^
+// textout-note@+1 {{[debug] analyzing from invokedLambdaScope()}}
+void invokedLambdaScope() {
+  auto invokedLambda = []() {
+    // common-warning@+1 {{REACHABLE}} textout-note@+1 {{REACHABLE}}
+    clang_analyzer_warnIfReached();
+  };
+  invokedLambda(); // textout-note {{Calling 'operator()'}}
+}
\ No newline at end of file
-- 
GitLab


From b282259711af7b295cb80a12880bcd8e0d1fcefa Mon Sep 17 00:00:00 2001
From: OverMighty <its.overmighty@gmail.com>
Date: Mon, 25 Mar 2024 14:26:22 +0000
Subject: [PATCH 127/404] [libc][math][c23] Add {,u}fromfp{,x}{,f,l,f128}
 functions (#86003)

Fixes #85279.

cc @lntue
---
 libc/config/linux/x86_64/entrypoints.txt      |  16 +
 libc/docs/dev/undefined_behavior.rst          |   8 +
 libc/docs/math/index.rst                      |  32 +
 libc/include/llvm-libc-macros/math-macros.h   |   6 +
 libc/spec/stdc.td                             |  26 +
 .../FPUtil/NearestIntegerOperations.h         |  94 ++-
 libc/src/math/CMakeLists.txt                  |  20 +
 libc/src/math/fromfp.h                        |  18 +
 libc/src/math/fromfpf.h                       |  18 +
 libc/src/math/fromfpf128.h                    |  20 +
 libc/src/math/fromfpl.h                       |  18 +
 libc/src/math/fromfpx.h                       |  18 +
 libc/src/math/fromfpxf.h                      |  18 +
 libc/src/math/fromfpxf128.h                   |  20 +
 libc/src/math/fromfpxl.h                      |  18 +
 libc/src/math/generic/CMakeLists.txt          | 196 ++++++
 libc/src/math/generic/fromfp.cpp              |  19 +
 libc/src/math/generic/fromfpf.cpp             |  19 +
 libc/src/math/generic/fromfpf128.cpp          |  20 +
 libc/src/math/generic/fromfpl.cpp             |  20 +
 libc/src/math/generic/fromfpx.cpp             |  19 +
 libc/src/math/generic/fromfpxf.cpp            |  19 +
 libc/src/math/generic/fromfpxf128.cpp         |  20 +
 libc/src/math/generic/fromfpxl.cpp            |  20 +
 libc/src/math/generic/ufromfp.cpp             |  19 +
 libc/src/math/generic/ufromfpf.cpp            |  19 +
 libc/src/math/generic/ufromfpf128.cpp         |  20 +
 libc/src/math/generic/ufromfpl.cpp            |  20 +
 libc/src/math/generic/ufromfpx.cpp            |  19 +
 libc/src/math/generic/ufromfpxf.cpp           |  19 +
 libc/src/math/generic/ufromfpxf128.cpp        |  20 +
 libc/src/math/generic/ufromfpxl.cpp           |  20 +
 libc/src/math/ufromfp.h                       |  18 +
 libc/src/math/ufromfpf.h                      |  18 +
 libc/src/math/ufromfpf128.h                   |  20 +
 libc/src/math/ufromfpl.h                      |  18 +
 libc/src/math/ufromfpx.h                      |  18 +
 libc/src/math/ufromfpxf.h                     |  18 +
 libc/src/math/ufromfpxf128.h                  |  20 +
 libc/src/math/ufromfpxl.h                     |  18 +
 libc/test/UnitTest/CMakeLists.txt             |   1 +
 libc/test/UnitTest/FPMatcher.h                |  10 +-
 libc/test/src/math/smoke/CMakeLists.txt       | 192 ++++++
 libc/test/src/math/smoke/FromfpTest.h         | 416 +++++++++++++
 libc/test/src/math/smoke/FromfpxTest.h        | 578 ++++++++++++++++++
 libc/test/src/math/smoke/UfromfpTest.h        | 348 +++++++++++
 libc/test/src/math/smoke/UfromfpxTest.h       | 437 +++++++++++++
 libc/test/src/math/smoke/fromfp_test.cpp      |  13 +
 libc/test/src/math/smoke/fromfpf128_test.cpp  |  13 +
 libc/test/src/math/smoke/fromfpf_test.cpp     |  13 +
 libc/test/src/math/smoke/fromfpl_test.cpp     |  13 +
 libc/test/src/math/smoke/fromfpx_test.cpp     |  13 +
 libc/test/src/math/smoke/fromfpxf128_test.cpp |  13 +
 libc/test/src/math/smoke/fromfpxf_test.cpp    |  13 +
 libc/test/src/math/smoke/fromfpxl_test.cpp    |  13 +
 libc/test/src/math/smoke/ufromfp_test.cpp     |  13 +
 libc/test/src/math/smoke/ufromfpf128_test.cpp |  13 +
 libc/test/src/math/smoke/ufromfpf_test.cpp    |  13 +
 libc/test/src/math/smoke/ufromfpl_test.cpp    |  13 +
 libc/test/src/math/smoke/ufromfpx_test.cpp    |  13 +
 .../test/src/math/smoke/ufromfpxf128_test.cpp |  13 +
 libc/test/src/math/smoke/ufromfpxf_test.cpp   |  13 +
 libc/test/src/math/smoke/ufromfpxl_test.cpp   |  13 +
 .../libc/test/UnitTest/BUILD.bazel            |   1 +
 64 files changed, 3181 insertions(+), 16 deletions(-)
 create mode 100644 libc/src/math/fromfp.h
 create mode 100644 libc/src/math/fromfpf.h
 create mode 100644 libc/src/math/fromfpf128.h
 create mode 100644 libc/src/math/fromfpl.h
 create mode 100644 libc/src/math/fromfpx.h
 create mode 100644 libc/src/math/fromfpxf.h
 create mode 100644 libc/src/math/fromfpxf128.h
 create mode 100644 libc/src/math/fromfpxl.h
 create mode 100644 libc/src/math/generic/fromfp.cpp
 create mode 100644 libc/src/math/generic/fromfpf.cpp
 create mode 100644 libc/src/math/generic/fromfpf128.cpp
 create mode 100644 libc/src/math/generic/fromfpl.cpp
 create mode 100644 libc/src/math/generic/fromfpx.cpp
 create mode 100644 libc/src/math/generic/fromfpxf.cpp
 create mode 100644 libc/src/math/generic/fromfpxf128.cpp
 create mode 100644 libc/src/math/generic/fromfpxl.cpp
 create mode 100644 libc/src/math/generic/ufromfp.cpp
 create mode 100644 libc/src/math/generic/ufromfpf.cpp
 create mode 100644 libc/src/math/generic/ufromfpf128.cpp
 create mode 100644 libc/src/math/generic/ufromfpl.cpp
 create mode 100644 libc/src/math/generic/ufromfpx.cpp
 create mode 100644 libc/src/math/generic/ufromfpxf.cpp
 create mode 100644 libc/src/math/generic/ufromfpxf128.cpp
 create mode 100644 libc/src/math/generic/ufromfpxl.cpp
 create mode 100644 libc/src/math/ufromfp.h
 create mode 100644 libc/src/math/ufromfpf.h
 create mode 100644 libc/src/math/ufromfpf128.h
 create mode 100644 libc/src/math/ufromfpl.h
 create mode 100644 libc/src/math/ufromfpx.h
 create mode 100644 libc/src/math/ufromfpxf.h
 create mode 100644 libc/src/math/ufromfpxf128.h
 create mode 100644 libc/src/math/ufromfpxl.h
 create mode 100644 libc/test/src/math/smoke/FromfpTest.h
 create mode 100644 libc/test/src/math/smoke/FromfpxTest.h
 create mode 100644 libc/test/src/math/smoke/UfromfpTest.h
 create mode 100644 libc/test/src/math/smoke/UfromfpxTest.h
 create mode 100644 libc/test/src/math/smoke/fromfp_test.cpp
 create mode 100644 libc/test/src/math/smoke/fromfpf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/fromfpf_test.cpp
 create mode 100644 libc/test/src/math/smoke/fromfpl_test.cpp
 create mode 100644 libc/test/src/math/smoke/fromfpx_test.cpp
 create mode 100644 libc/test/src/math/smoke/fromfpxf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/fromfpxf_test.cpp
 create mode 100644 libc/test/src/math/smoke/fromfpxl_test.cpp
 create mode 100644 libc/test/src/math/smoke/ufromfp_test.cpp
 create mode 100644 libc/test/src/math/smoke/ufromfpf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/ufromfpf_test.cpp
 create mode 100644 libc/test/src/math/smoke/ufromfpl_test.cpp
 create mode 100644 libc/test/src/math/smoke/ufromfpx_test.cpp
 create mode 100644 libc/test/src/math/smoke/ufromfpxf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/ufromfpxf_test.cpp
 create mode 100644 libc/test/src/math/smoke/ufromfpxl_test.cpp

diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 8b1cd3fb1052..de741e218a37 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -387,6 +387,12 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.frexp
     libc.src.math.frexpf
     libc.src.math.frexpl
+    libc.src.math.fromfp
+    libc.src.math.fromfpf
+    libc.src.math.fromfpl
+    libc.src.math.fromfpx
+    libc.src.math.fromfpxf
+    libc.src.math.fromfpxl
     libc.src.math.hypot
     libc.src.math.hypotf
     libc.src.math.ilogb
@@ -471,6 +477,12 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.trunc
     libc.src.math.truncf
     libc.src.math.truncl
+    libc.src.math.ufromfp
+    libc.src.math.ufromfpf
+    libc.src.math.ufromfpl
+    libc.src.math.ufromfpx
+    libc.src.math.ufromfpxf
+    libc.src.math.ufromfpxl
 )
 
 if(LIBC_TYPES_HAS_FLOAT128)
@@ -485,6 +497,8 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.fminf128
     libc.src.math.fmodf128
     libc.src.math.frexpf128
+    libc.src.math.fromfpf128
+    libc.src.math.fromfpxf128
     libc.src.math.ilogbf128
     libc.src.math.ldexpf128
     libc.src.math.llogbf128
@@ -502,6 +516,8 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.roundf128
     libc.src.math.sqrtf128
     libc.src.math.truncf128
+    libc.src.math.ufromfpf128
+    libc.src.math.ufromfpxf128
   )
 endif()
 
diff --git a/libc/docs/dev/undefined_behavior.rst b/libc/docs/dev/undefined_behavior.rst
index 50e8bdde89dd..c97a539ca8da 100644
--- a/libc/docs/dev/undefined_behavior.rst
+++ b/libc/docs/dev/undefined_behavior.rst
@@ -81,3 +81,11 @@ The C standard does not specify behavior for ``printf("%s", NULL)``. We will
 print the string literal ``(null)`` unless using the 
 ``LIBC_COPT_PRINTF_NO_NULLPTR_CHECKS`` option described in :ref:`printf 
 behavior<printf_behavior>`.
+
+Unknown Math Rounding Direction
+-------------------------------
+The C23 standard states that if the value of the ``rnd`` argument of the
+``fromfp``, ``ufromfp``, ``fromfpx`` and ``ufromfpx`` functions is not equal to
+the value of a math rounding direction macro, the direction of rounding is
+unspecified. LLVM's libc chooses to use the ``FP_INT_TONEAREST`` rounding
+direction in this case.
diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst
index d337d060fb5d..fa604faaec2f 100644
--- a/libc/docs/math/index.rst
+++ b/libc/docs/math/index.rst
@@ -181,6 +181,22 @@ Basic Operations
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | frexpf128    | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fromfp       | |check| |         |         |         |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fromfpf      | |check| |         |         |         |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fromfpl      | |check| |         |         |         |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fromfpf128   | |check| |         |         |         |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fromfpx      | |check| |         |         |         |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fromfpxf     | |check| |         |         |         |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fromfpxl     | |check| |         |         |         |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fromfpxf128  | |check| |         |         |         |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | ilogb        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | ilogbf       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
@@ -339,6 +355,22 @@ Basic Operations
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | truncf128    | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ufromfp      | |check| |         |         |         |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ufromfpf     | |check| |         |         |         |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ufromfpl     | |check| |         |         |         |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ufromfpf128  | |check| |         |         |         |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ufromfpx     | |check| |         |         |         |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ufromfpxf    | |check| |         |         |         |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ufromfpxl    | |check| |         |         |         |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ufromfpxf128 | |check| |         |         |         |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 
 
 Higher Math Functions
diff --git a/libc/include/llvm-libc-macros/math-macros.h b/libc/include/llvm-libc-macros/math-macros.h
index 2605535b927d..03c7a823e6e9 100644
--- a/libc/include/llvm-libc-macros/math-macros.h
+++ b/libc/include/llvm-libc-macros/math-macros.h
@@ -17,6 +17,12 @@
 #define FP_SUBNORMAL 3
 #define FP_NORMAL 4
 
+#define FP_INT_UPWARD 0
+#define FP_INT_DOWNWARD 1
+#define FP_INT_TOWARDZERO 2
+#define FP_INT_TONEARESTFROMZERO 3
+#define FP_INT_TONEAREST 4
+
 #define MATH_ERRNO 1
 #define MATH_ERREXCEPT 2
 
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index 3e58e3b88645..7407ead17711 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -352,6 +352,12 @@ def StdC : StandardSpec<"stdc"> {
           Macro<"INFINITY">,
           Macro<"NAN">,
 
+          Macro<"FP_INT_UPWARD">,
+          Macro<"FP_INT_DOWNWARD">,
+          Macro<"FP_INT_TOWARDZERO">,
+          Macro<"FP_INT_TONEARESTFROMZERO">,
+          Macro<"FP_INT_TONEAREST">,
+
           Macro<"FP_ILOGB0">,
           Macro<"FP_ILOGBNAN">,
 
@@ -414,6 +420,26 @@ def StdC : StandardSpec<"stdc"> {
           FunctionSpec<"frexpl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntPtr>]>,
           GuardedFunctionSpec<"frexpf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<IntPtr>], "LIBC_TYPES_HAS_FLOAT128">,
 
+          FunctionSpec<"fromfp", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
+          FunctionSpec<"fromfpf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
+          FunctionSpec<"fromfpl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
+          GuardedFunctionSpec<"fromfpf128", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>], "LIBC_TYPES_HAS_FLOAT128">,
+
+          FunctionSpec<"fromfpx", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
+          FunctionSpec<"fromfpxf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
+          FunctionSpec<"fromfpxl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
+          GuardedFunctionSpec<"fromfpxf128", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>], "LIBC_TYPES_HAS_FLOAT128">,
+
+          FunctionSpec<"ufromfp", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
+          FunctionSpec<"ufromfpf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
+          FunctionSpec<"ufromfpl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
+          GuardedFunctionSpec<"ufromfpf128", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>], "LIBC_TYPES_HAS_FLOAT128">,
+
+          FunctionSpec<"ufromfpx", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
+          FunctionSpec<"ufromfpxf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
+          FunctionSpec<"ufromfpxl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>]>,
+          GuardedFunctionSpec<"ufromfpxf128", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntType>, ArgSpec<UnsignedIntType>], "LIBC_TYPES_HAS_FLOAT128">,
+
           FunctionSpec<"hypot", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"hypotf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
 
diff --git a/libc/src/__support/FPUtil/NearestIntegerOperations.h b/libc/src/__support/FPUtil/NearestIntegerOperations.h
index e890e38ba4ae..579584076b12 100644
--- a/libc/src/__support/FPUtil/NearestIntegerOperations.h
+++ b/libc/src/__support/FPUtil/NearestIntegerOperations.h
@@ -140,8 +140,9 @@ LIBC_INLINE T round(T x) {
   }
 }
 
-template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0>
-LIBC_INLINE T round_using_current_rounding_mode(T x) {
+template <typename T>
+LIBC_INLINE constexpr cpp::enable_if_t<cpp::is_floating_point_v<T>, T>
+round_using_specific_rounding_mode(T x, int rnd) {
   using StorageType = typename FPBits<T>::StorageType;
   FPBits<T> bits(x);
 
@@ -151,7 +152,6 @@ LIBC_INLINE T round_using_current_rounding_mode(T x) {
 
   bool is_neg = bits.is_neg();
   int exponent = bits.get_exponent();
-  int rounding_mode = quick_get_round();
 
   // If the exponent is greater than the most negative mantissa
   // exponent, then x is already an integer.
@@ -159,20 +159,23 @@ LIBC_INLINE T round_using_current_rounding_mode(T x) {
     return x;
 
   if (exponent <= -1) {
-    switch (rounding_mode) {
-    case FE_DOWNWARD:
+    switch (rnd) {
+    case FP_INT_DOWNWARD:
       return is_neg ? T(-1.0) : T(0.0);
-    case FE_UPWARD:
+    case FP_INT_UPWARD:
       return is_neg ? T(-0.0) : T(1.0);
-    case FE_TOWARDZERO:
+    case FP_INT_TOWARDZERO:
       return is_neg ? T(-0.0) : T(0.0);
-    case FE_TONEAREST:
+    case FP_INT_TONEARESTFROMZERO:
+      if (exponent < -1)
+        return is_neg ? T(-0.0) : T(0.0); // abs(x) < 0.5
+      return is_neg ? T(-1.0) : T(1.0);   // abs(x) >= 0.5
+    case FP_INT_TONEAREST:
+    default:
       if (exponent <= -2 || bits.get_mantissa() == 0)
         return is_neg ? T(-0.0) : T(0.0); // abs(x) <= 0.5
       else
         return is_neg ? T(-1.0) : T(1.0); // abs(x) > 0.5
-    default:
-      __builtin_unreachable();
     }
   }
 
@@ -194,14 +197,19 @@ LIBC_INLINE T round_using_current_rounding_mode(T x) {
   StorageType trunc_is_odd =
       new_bits.get_mantissa() & (StorageType(1) << trim_size);
 
-  switch (rounding_mode) {
-  case FE_DOWNWARD:
+  switch (rnd) {
+  case FP_INT_DOWNWARD:
     return is_neg ? trunc_value - T(1.0) : trunc_value;
-  case FE_UPWARD:
+  case FP_INT_UPWARD:
     return is_neg ? trunc_value : trunc_value + T(1.0);
-  case FE_TOWARDZERO:
+  case FP_INT_TOWARDZERO:
     return trunc_value;
-  case FE_TONEAREST:
+  case FP_INT_TONEARESTFROMZERO:
+    if (trim_value >= half_value)
+      return is_neg ? trunc_value - T(1.0) : trunc_value + T(1.0);
+    return trunc_value;
+  case FP_INT_TONEAREST:
+  default:
     if (trim_value > half_value) {
       return is_neg ? trunc_value - T(1.0) : trunc_value + T(1.0);
     } else if (trim_value == half_value) {
@@ -214,11 +222,67 @@ LIBC_INLINE T round_using_current_rounding_mode(T x) {
     } else {
       return trunc_value;
     }
+  }
+}
+
+template <typename T>
+LIBC_INLINE cpp::enable_if_t<cpp::is_floating_point_v<T>, T>
+round_using_current_rounding_mode(T x) {
+  int rounding_mode = quick_get_round();
+
+  switch (rounding_mode) {
+  case FE_DOWNWARD:
+    return round_using_specific_rounding_mode(x, FP_INT_DOWNWARD);
+  case FE_UPWARD:
+    return round_using_specific_rounding_mode(x, FP_INT_UPWARD);
+  case FE_TOWARDZERO:
+    return round_using_specific_rounding_mode(x, FP_INT_TOWARDZERO);
+  case FE_TONEAREST:
+    return round_using_specific_rounding_mode(x, FP_INT_TONEAREST);
   default:
     __builtin_unreachable();
   }
 }
 
+template <bool IsSigned, typename T>
+LIBC_INLINE constexpr cpp::enable_if_t<cpp::is_floating_point_v<T>, T>
+fromfp(T x, int rnd, unsigned int width) {
+  if (width == 0U)
+    return FPBits<T>::quiet_nan().get_val();
+
+  T rounded_value = round_using_specific_rounding_mode(x, rnd);
+
+  if constexpr (IsSigned) {
+    // T can't hold a finite number >= 2.0 * 2^EXP_BIAS.
+    if (width - 1 > FPBits<T>::EXP_BIAS)
+      return rounded_value;
+    if (rounded_value < -T(1U << (width - 1U)))
+      return FPBits<T>::quiet_nan().get_val();
+    if (rounded_value > T((1U << (width - 1U)) - 1U))
+      return FPBits<T>::quiet_nan().get_val();
+    return rounded_value;
+  }
+
+  if (rounded_value < T(0.0))
+    return FPBits<T>::quiet_nan().get_val();
+  // T can't hold a finite number >= 2.0 * 2^EXP_BIAS.
+  if (width <= FPBits<T>::EXP_BIAS && rounded_value > T(1U << width) - 1U)
+    return FPBits<T>::quiet_nan().get_val();
+  return rounded_value;
+}
+
+template <bool IsSigned, typename T>
+LIBC_INLINE constexpr cpp::enable_if_t<cpp::is_floating_point_v<T>, T>
+fromfpx(T x, int rnd, unsigned int width) {
+  T rounded_value = fromfp<IsSigned>(x, rnd, width);
+  FPBits<T> bits(rounded_value);
+
+  if (!bits.is_nan() && rounded_value != x)
+    raise_except_if_required(FE_INEXACT);
+
+  return rounded_value;
+}
+
 namespace internal {
 
 template <typename F, typename I,
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index 5e2e6e699d0e..2f1d65780013 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -127,6 +127,16 @@ add_math_entrypoint_object(frexpf)
 add_math_entrypoint_object(frexpl)
 add_math_entrypoint_object(frexpf128)
 
+add_math_entrypoint_object(fromfp)
+add_math_entrypoint_object(fromfpf)
+add_math_entrypoint_object(fromfpl)
+add_math_entrypoint_object(fromfpf128)
+
+add_math_entrypoint_object(fromfpx)
+add_math_entrypoint_object(fromfpxf)
+add_math_entrypoint_object(fromfpxl)
+add_math_entrypoint_object(fromfpxf128)
+
 add_math_entrypoint_object(hypot)
 add_math_entrypoint_object(hypotf)
 
@@ -267,3 +277,13 @@ add_math_entrypoint_object(trunc)
 add_math_entrypoint_object(truncf)
 add_math_entrypoint_object(truncl)
 add_math_entrypoint_object(truncf128)
+
+add_math_entrypoint_object(ufromfp)
+add_math_entrypoint_object(ufromfpf)
+add_math_entrypoint_object(ufromfpl)
+add_math_entrypoint_object(ufromfpf128)
+
+add_math_entrypoint_object(ufromfpx)
+add_math_entrypoint_object(ufromfpxf)
+add_math_entrypoint_object(ufromfpxl)
+add_math_entrypoint_object(ufromfpxf128)
diff --git a/libc/src/math/fromfp.h b/libc/src/math/fromfp.h
new file mode 100644
index 000000000000..d3de2dd34608
--- /dev/null
+++ b/libc/src/math/fromfp.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for fromfp ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FROMFP_H
+#define LLVM_LIBC_SRC_MATH_FROMFP_H
+
+namespace LIBC_NAMESPACE {
+
+double fromfp(double x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FROMFP_H
diff --git a/libc/src/math/fromfpf.h b/libc/src/math/fromfpf.h
new file mode 100644
index 000000000000..11d432148d01
--- /dev/null
+++ b/libc/src/math/fromfpf.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for fromfpf -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FROMFPF_H
+#define LLVM_LIBC_SRC_MATH_FROMFPF_H
+
+namespace LIBC_NAMESPACE {
+
+float fromfpf(float x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FROMFPF_H
diff --git a/libc/src/math/fromfpf128.h b/libc/src/math/fromfpf128.h
new file mode 100644
index 000000000000..5f85fde570a0
--- /dev/null
+++ b/libc/src/math/fromfpf128.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for fromfpf128 --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FROMFPF128_H
+#define LLVM_LIBC_SRC_MATH_FROMFPF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float128 fromfpf128(float128 x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FROMFPF128_H
diff --git a/libc/src/math/fromfpl.h b/libc/src/math/fromfpl.h
new file mode 100644
index 000000000000..dd8e1eebdea9
--- /dev/null
+++ b/libc/src/math/fromfpl.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for fromfpl -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FROMFPL_H
+#define LLVM_LIBC_SRC_MATH_FROMFPL_H
+
+namespace LIBC_NAMESPACE {
+
+long double fromfpl(long double x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FROMFPL_H
diff --git a/libc/src/math/fromfpx.h b/libc/src/math/fromfpx.h
new file mode 100644
index 000000000000..3fc96e1e648b
--- /dev/null
+++ b/libc/src/math/fromfpx.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for fromfpx -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FROMFPX_H
+#define LLVM_LIBC_SRC_MATH_FROMFPX_H
+
+namespace LIBC_NAMESPACE {
+
+double fromfpx(double x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FROMFPX_H
diff --git a/libc/src/math/fromfpxf.h b/libc/src/math/fromfpxf.h
new file mode 100644
index 000000000000..b55bc4c75139
--- /dev/null
+++ b/libc/src/math/fromfpxf.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for fromfpxf ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FROMFPXF_H
+#define LLVM_LIBC_SRC_MATH_FROMFPXF_H
+
+namespace LIBC_NAMESPACE {
+
+float fromfpxf(float x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FROMFPXF_H
diff --git a/libc/src/math/fromfpxf128.h b/libc/src/math/fromfpxf128.h
new file mode 100644
index 000000000000..88932646cdcc
--- /dev/null
+++ b/libc/src/math/fromfpxf128.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for fromfpxf128 -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FROMFPXF128_H
+#define LLVM_LIBC_SRC_MATH_FROMFPXF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float128 fromfpxf128(float128 x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FROMFPXF128_H
diff --git a/libc/src/math/fromfpxl.h b/libc/src/math/fromfpxl.h
new file mode 100644
index 000000000000..545288834b8f
--- /dev/null
+++ b/libc/src/math/fromfpxl.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for fromfpxl ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FROMFPXL_H
+#define LLVM_LIBC_SRC_MATH_FROMFPXL_H
+
+namespace LIBC_NAMESPACE {
+
+long double fromfpxl(long double x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FROMFPXL_H
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index 176c32e47768..5198e9c58a98 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -2025,6 +2025,202 @@ add_entrypoint_object(
     -O3
 )
 
+add_entrypoint_object(
+  fromfp
+  SRCS
+    fromfp.cpp
+  HDRS
+    ../fromfp.h
+  DEPENDS
+    libc.src.__support.FPUtil.nearest_integer
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  fromfpf
+  SRCS
+    fromfpf.cpp
+  HDRS
+    ../fromfpf.h
+  DEPENDS
+    libc.src.__support.FPUtil.nearest_integer
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  fromfpl
+  SRCS
+    fromfpl.cpp
+  HDRS
+    ../fromfpl.h
+  DEPENDS
+    libc.src.__support.FPUtil.nearest_integer
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  fromfpf128
+  SRCS
+    fromfpf128.cpp
+  HDRS
+    ../fromfpf128.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.nearest_integer
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  fromfpx
+  SRCS
+    fromfpx.cpp
+  HDRS
+    ../fromfpx.h
+  DEPENDS
+    libc.src.__support.FPUtil.nearest_integer
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  fromfpxf
+  SRCS
+    fromfpxf.cpp
+  HDRS
+    ../fromfpxf.h
+  DEPENDS
+    libc.src.__support.FPUtil.nearest_integer
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  fromfpxl
+  SRCS
+    fromfpxl.cpp
+  HDRS
+    ../fromfpxl.h
+  DEPENDS
+    libc.src.__support.FPUtil.nearest_integer
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  fromfpxf128
+  SRCS
+    fromfpxf128.cpp
+  HDRS
+    ../fromfpxf128.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.nearest_integer
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  ufromfp
+  SRCS
+    ufromfp.cpp
+  HDRS
+    ../ufromfp.h
+  DEPENDS
+    libc.src.__support.FPUtil.nearest_integer
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  ufromfpf
+  SRCS
+    ufromfpf.cpp
+  HDRS
+    ../ufromfpf.h
+  DEPENDS
+    libc.src.__support.FPUtil.nearest_integer
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  ufromfpl
+  SRCS
+    ufromfpl.cpp
+  HDRS
+    ../ufromfpl.h
+  DEPENDS
+    libc.src.__support.FPUtil.nearest_integer
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  ufromfpf128
+  SRCS
+    ufromfpf128.cpp
+  HDRS
+    ../ufromfpf128.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.nearest_integer
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  ufromfpx
+  SRCS
+    ufromfpx.cpp
+  HDRS
+    ../ufromfpx.h
+  DEPENDS
+    libc.src.__support.FPUtil.nearest_integer
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  ufromfpxf
+  SRCS
+    ufromfpxf.cpp
+  HDRS
+    ../ufromfpxf.h
+  DEPENDS
+    libc.src.__support.FPUtil.nearest_integer
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  ufromfpxl
+  SRCS
+    ufromfpxl.cpp
+  HDRS
+    ../ufromfpxl.h
+  DEPENDS
+    libc.src.__support.FPUtil.nearest_integer
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  ufromfpxf128
+  SRCS
+    ufromfpxf128.cpp
+  HDRS
+    ../ufromfpxf128.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.nearest_integer
+  COMPILE_OPTIONS
+    -O3
+)
+
 #TODO: Add errno include to the hyperbolic functions.
 add_object_library(
   explogxf
diff --git a/libc/src/math/generic/fromfp.cpp b/libc/src/math/generic/fromfp.cpp
new file mode 100644
index 000000000000..ba3f0a133cbc
--- /dev/null
+++ b/libc/src/math/generic/fromfp.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fromfp function ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fromfp.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, fromfp, (double x, int rnd, unsigned int width)) {
+  return fputil::fromfp</*IsSigned=*/true>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fromfpf.cpp b/libc/src/math/generic/fromfpf.cpp
new file mode 100644
index 000000000000..fd058a13201c
--- /dev/null
+++ b/libc/src/math/generic/fromfpf.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fromfpf function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fromfpf.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, fromfpf, (float x, int rnd, unsigned int width)) {
+  return fputil::fromfp</*IsSigned=*/true>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fromfpf128.cpp b/libc/src/math/generic/fromfpf128.cpp
new file mode 100644
index 000000000000..440a5da75693
--- /dev/null
+++ b/libc/src/math/generic/fromfpf128.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of fromfpf128 function -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fromfpf128.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float128, fromfpf128,
+                   (float128 x, int rnd, unsigned int width)) {
+  return fputil::fromfp</*IsSigned=*/true>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fromfpl.cpp b/libc/src/math/generic/fromfpl.cpp
new file mode 100644
index 000000000000..ee3abeaf98f8
--- /dev/null
+++ b/libc/src/math/generic/fromfpl.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of fromfpl function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fromfpl.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long double, fromfpl,
+                   (long double x, int rnd, unsigned int width)) {
+  return fputil::fromfp</*IsSigned=*/true>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fromfpx.cpp b/libc/src/math/generic/fromfpx.cpp
new file mode 100644
index 000000000000..b9e7e4a7aa12
--- /dev/null
+++ b/libc/src/math/generic/fromfpx.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fromfpx function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fromfpx.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, fromfpx, (double x, int rnd, unsigned int width)) {
+  return fputil::fromfpx</*IsSigned=*/true>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fromfpxf.cpp b/libc/src/math/generic/fromfpxf.cpp
new file mode 100644
index 000000000000..1473499244d3
--- /dev/null
+++ b/libc/src/math/generic/fromfpxf.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fromfpxf function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fromfpxf.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, fromfpxf, (float x, int rnd, unsigned int width)) {
+  return fputil::fromfpx</*IsSigned=*/true>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fromfpxf128.cpp b/libc/src/math/generic/fromfpxf128.cpp
new file mode 100644
index 000000000000..5d930d22ae5e
--- /dev/null
+++ b/libc/src/math/generic/fromfpxf128.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of fromfpxf128 function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fromfpxf128.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float128, fromfpxf128,
+                   (float128 x, int rnd, unsigned int width)) {
+  return fputil::fromfpx</*IsSigned=*/true>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fromfpxl.cpp b/libc/src/math/generic/fromfpxl.cpp
new file mode 100644
index 000000000000..c3db055246f2
--- /dev/null
+++ b/libc/src/math/generic/fromfpxl.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of fromfpxl function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fromfpxl.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long double, fromfpxl,
+                   (long double x, int rnd, unsigned int width)) {
+  return fputil::fromfpx</*IsSigned=*/true>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/ufromfp.cpp b/libc/src/math/generic/ufromfp.cpp
new file mode 100644
index 000000000000..15800d67fd8d
--- /dev/null
+++ b/libc/src/math/generic/ufromfp.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of ufromfp function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/ufromfp.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, ufromfp, (double x, int rnd, unsigned int width)) {
+  return fputil::fromfp</*IsSigned=*/false>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/ufromfpf.cpp b/libc/src/math/generic/ufromfpf.cpp
new file mode 100644
index 000000000000..898446ec45aa
--- /dev/null
+++ b/libc/src/math/generic/ufromfpf.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of ufromfpf function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/ufromfpf.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, ufromfpf, (float x, int rnd, unsigned int width)) {
+  return fputil::fromfp</*IsSigned=*/false>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/ufromfpf128.cpp b/libc/src/math/generic/ufromfpf128.cpp
new file mode 100644
index 000000000000..cc728f35551c
--- /dev/null
+++ b/libc/src/math/generic/ufromfpf128.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of ufromfpf128 function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/ufromfpf128.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float128, ufromfpf128,
+                   (float128 x, int rnd, unsigned int width)) {
+  return fputil::fromfp</*IsSigned=*/false>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/ufromfpl.cpp b/libc/src/math/generic/ufromfpl.cpp
new file mode 100644
index 000000000000..bd353e9ebbb5
--- /dev/null
+++ b/libc/src/math/generic/ufromfpl.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of ufromfpl function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/ufromfpl.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long double, ufromfpl,
+                   (long double x, int rnd, unsigned int width)) {
+  return fputil::fromfp</*IsSigned=*/false>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/ufromfpx.cpp b/libc/src/math/generic/ufromfpx.cpp
new file mode 100644
index 000000000000..5ad95ff3061e
--- /dev/null
+++ b/libc/src/math/generic/ufromfpx.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of ufromfpx function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/ufromfpx.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, ufromfpx, (double x, int rnd, unsigned int width)) {
+  return fputil::fromfpx</*IsSigned=*/false>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/ufromfpxf.cpp b/libc/src/math/generic/ufromfpxf.cpp
new file mode 100644
index 000000000000..7c878489e8d2
--- /dev/null
+++ b/libc/src/math/generic/ufromfpxf.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of ufromfpxf function ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/ufromfpxf.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, ufromfpxf, (float x, int rnd, unsigned int width)) {
+  return fputil::fromfpx</*IsSigned=*/false>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/ufromfpxf128.cpp b/libc/src/math/generic/ufromfpxf128.cpp
new file mode 100644
index 000000000000..57c290365e69
--- /dev/null
+++ b/libc/src/math/generic/ufromfpxf128.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of ufromfpxf128 function ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/ufromfpxf128.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float128, ufromfpxf128,
+                   (float128 x, int rnd, unsigned int width)) {
+  return fputil::fromfpx</*IsSigned=*/false>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/ufromfpxl.cpp b/libc/src/math/generic/ufromfpxl.cpp
new file mode 100644
index 000000000000..9a8ba7aa5b91
--- /dev/null
+++ b/libc/src/math/generic/ufromfpxl.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of ufromfpxl function ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/ufromfpxl.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long double, ufromfpxl,
+                   (long double x, int rnd, unsigned int width)) {
+  return fputil::fromfpx</*IsSigned=*/false>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/ufromfp.h b/libc/src/math/ufromfp.h
new file mode 100644
index 000000000000..f4667486440c
--- /dev/null
+++ b/libc/src/math/ufromfp.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for ufromfp -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_UFROMFP_H
+#define LLVM_LIBC_SRC_MATH_UFROMFP_H
+
+namespace LIBC_NAMESPACE {
+
+double ufromfp(double x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_UFROMFP_H
diff --git a/libc/src/math/ufromfpf.h b/libc/src/math/ufromfpf.h
new file mode 100644
index 000000000000..40c6773d143e
--- /dev/null
+++ b/libc/src/math/ufromfpf.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for ufromfpf ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_UFROMFPF_H
+#define LLVM_LIBC_SRC_MATH_UFROMFPF_H
+
+namespace LIBC_NAMESPACE {
+
+float ufromfpf(float x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_UFROMFPF_H
diff --git a/libc/src/math/ufromfpf128.h b/libc/src/math/ufromfpf128.h
new file mode 100644
index 000000000000..785fa82becbc
--- /dev/null
+++ b/libc/src/math/ufromfpf128.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for ufromfpf128 -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_UFROMFPF128_H
+#define LLVM_LIBC_SRC_MATH_UFROMFPF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float128 ufromfpf128(float128 x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_UFROMFPF128_H
diff --git a/libc/src/math/ufromfpl.h b/libc/src/math/ufromfpl.h
new file mode 100644
index 000000000000..f05a77dc2f10
--- /dev/null
+++ b/libc/src/math/ufromfpl.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for ufromfpl ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_UFROMFPL_H
+#define LLVM_LIBC_SRC_MATH_UFROMFPL_H
+
+namespace LIBC_NAMESPACE {
+
+long double ufromfpl(long double x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_UFROMFPL_H
diff --git a/libc/src/math/ufromfpx.h b/libc/src/math/ufromfpx.h
new file mode 100644
index 000000000000..79c413af968a
--- /dev/null
+++ b/libc/src/math/ufromfpx.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for ufromfpx ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_UFROMFPX_H
+#define LLVM_LIBC_SRC_MATH_UFROMFPX_H
+
+namespace LIBC_NAMESPACE {
+
+double ufromfpx(double x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_UFROMFPX_H
diff --git a/libc/src/math/ufromfpxf.h b/libc/src/math/ufromfpxf.h
new file mode 100644
index 000000000000..f6bd8f7d5995
--- /dev/null
+++ b/libc/src/math/ufromfpxf.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for ufromfpxf ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_UFROMFPXF_H
+#define LLVM_LIBC_SRC_MATH_UFROMFPXF_H
+
+namespace LIBC_NAMESPACE {
+
+float ufromfpxf(float x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_UFROMFPXF_H
diff --git a/libc/src/math/ufromfpxf128.h b/libc/src/math/ufromfpxf128.h
new file mode 100644
index 000000000000..f3b43ff54f37
--- /dev/null
+++ b/libc/src/math/ufromfpxf128.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for ufromfpxf128 ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_UFROMFPXF128_H
+#define LLVM_LIBC_SRC_MATH_UFROMFPXF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float128 ufromfpxf128(float128 x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_UFROMFPXF128_H
diff --git a/libc/src/math/ufromfpxl.h b/libc/src/math/ufromfpxl.h
new file mode 100644
index 000000000000..180b8f93d218
--- /dev/null
+++ b/libc/src/math/ufromfpxl.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for ufromfpxl ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_UFROMFPXL_H
+#define LLVM_LIBC_SRC_MATH_UFROMFPXL_H
+
+namespace LIBC_NAMESPACE {
+
+long double ufromfpxl(long double x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_UFROMFPXL_H
diff --git a/libc/test/UnitTest/CMakeLists.txt b/libc/test/UnitTest/CMakeLists.txt
index f7a6f4a91fab..d830d22bb540 100644
--- a/libc/test/UnitTest/CMakeLists.txt
+++ b/libc/test/UnitTest/CMakeLists.txt
@@ -118,6 +118,7 @@ add_unittest_framework_library(
   DEPENDS
     LibcTest
     libc.test.UnitTest.string_utils
+    libc.src.__support.CPP.array
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.fpbits_str
     libc.src.__support.FPUtil.fenv_impl
diff --git a/libc/test/UnitTest/FPMatcher.h b/libc/test/UnitTest/FPMatcher.h
index ee618a623efe..f4553eac5c8a 100644
--- a/libc/test/UnitTest/FPMatcher.h
+++ b/libc/test/UnitTest/FPMatcher.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_LIBC_TEST_UNITTEST_FPMATCHER_H
 #define LLVM_LIBC_TEST_UNITTEST_FPMATCHER_H
 
+#include "src/__support/CPP/array.h"
 #include "src/__support/CPP/type_traits.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
@@ -105,7 +106,14 @@ template <typename T> struct FPTest : public Test {
   const T neg_max_normal = FPBits::max_normal(Sign::NEG).get_val();            \
   const T min_denormal = FPBits::min_subnormal(Sign::POS).get_val();           \
   const T neg_min_denormal = FPBits::min_subnormal(Sign::NEG).get_val();       \
-  const T max_denormal = FPBits::max_subnormal().get_val();
+  const T max_denormal = FPBits::max_subnormal().get_val();                    \
+  static constexpr int UNKNOWN_MATH_ROUNDING_DIRECTION = 99;                   \
+  static constexpr LIBC_NAMESPACE::cpp::array<int, 6>                          \
+      MATH_ROUNDING_DIRECTIONS_INCLUDING_UNKNOWN = {                           \
+          FP_INT_UPWARD,     FP_INT_DOWNWARD,                                  \
+          FP_INT_TOWARDZERO, FP_INT_TONEARESTFROMZERO,                         \
+          FP_INT_TONEAREST,  UNKNOWN_MATH_ROUNDING_DIRECTION,                  \
+  };
 
 #define EXPECT_FP_EQ(expected, actual)                                         \
   EXPECT_THAT(actual, LIBC_NAMESPACE::testing::getMatcher<                     \
diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index 85dacce3b21d..56d864e78083 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -839,6 +839,198 @@ add_fp_unittest(
     libc.src.math.frexpf128
 )
 
+add_fp_unittest(
+  fromfp_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fromfp_test.cpp
+  HDRS
+    FromfpTest.h
+  DEPENDS
+    libc.src.math.fromfp
+)
+
+add_fp_unittest(
+  fromfpf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fromfpf_test.cpp
+  HDRS
+    FromfpTest.h
+  DEPENDS
+    libc.src.math.fromfpf
+)
+
+add_fp_unittest(
+  fromfpl_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fromfpl_test.cpp
+  HDRS
+    FromfpTest.h
+  DEPENDS
+    libc.src.math.fromfpl
+)
+
+add_fp_unittest(
+  fromfpf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fromfpf128_test.cpp
+  HDRS
+    FromfpTest.h
+  DEPENDS
+    libc.src.math.fromfpf128
+)
+
+add_fp_unittest(
+  fromfpx_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fromfpx_test.cpp
+  HDRS
+    FromfpxTest.h
+  DEPENDS
+    libc.src.math.fromfpx
+)
+
+add_fp_unittest(
+  fromfpxf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fromfpxf_test.cpp
+  HDRS
+    FromfpxTest.h
+  DEPENDS
+    libc.src.math.fromfpxf
+)
+
+add_fp_unittest(
+  fromfpxl_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fromfpxl_test.cpp
+  HDRS
+    FromfpxTest.h
+  DEPENDS
+    libc.src.math.fromfpxl
+)
+
+add_fp_unittest(
+  fromfpxf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fromfpxf128_test.cpp
+  HDRS
+    FromfpxTest.h
+  DEPENDS
+    libc.src.math.fromfpxf128
+)
+
+add_fp_unittest(
+  ufromfp_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    ufromfp_test.cpp
+  HDRS
+    UfromfpTest.h
+  DEPENDS
+    libc.src.math.ufromfp
+)
+
+add_fp_unittest(
+  ufromfpf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    ufromfpf_test.cpp
+  HDRS
+    UfromfpTest.h
+  DEPENDS
+    libc.src.math.ufromfpf
+)
+
+add_fp_unittest(
+  ufromfpl_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    ufromfpl_test.cpp
+  HDRS
+    UfromfpTest.h
+  DEPENDS
+    libc.src.math.ufromfpl
+)
+
+add_fp_unittest(
+  ufromfpf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    ufromfpf128_test.cpp
+  HDRS
+    UfromfpTest.h
+  DEPENDS
+    libc.src.math.ufromfpf128
+)
+
+add_fp_unittest(
+  ufromfpx_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    ufromfpx_test.cpp
+  HDRS
+    UfromfpxTest.h
+  DEPENDS
+    libc.src.math.ufromfpx
+)
+
+add_fp_unittest(
+  ufromfpxf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    ufromfpxf_test.cpp
+  HDRS
+    UfromfpxTest.h
+  DEPENDS
+    libc.src.math.ufromfpxf
+)
+
+add_fp_unittest(
+  ufromfpxl_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    ufromfpxl_test.cpp
+  HDRS
+    UfromfpxTest.h
+  DEPENDS
+    libc.src.math.ufromfpxl
+)
+
+add_fp_unittest(
+  ufromfpxf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    ufromfpxf128_test.cpp
+  HDRS
+    UfromfpxTest.h
+  DEPENDS
+    libc.src.math.ufromfpxf128
+)
+
 add_fp_unittest(
   ilogb_test
   SUITE
diff --git a/libc/test/src/math/smoke/FromfpTest.h b/libc/test/src/math/smoke/FromfpTest.h
new file mode 100644
index 000000000000..2d4c4c5314c9
--- /dev/null
+++ b/libc/test/src/math/smoke/FromfpTest.h
@@ -0,0 +1,416 @@
+//===-- Utility class to test different flavors of fromfp -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LIBC_TEST_SRC_MATH_SMOKE_FROMFPTEST_H
+#define LIBC_TEST_SRC_MATH_SMOKE_FROMFPTEST_H
+
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+template <typename T>
+class FromfpTestTemplate : public LIBC_NAMESPACE::testing::Test {
+
+  DECLARE_SPECIAL_CONSTANTS(T)
+
+public:
+  typedef T (*FromfpFunc)(T, int, unsigned int);
+
+  void testSpecialNumbersNonzeroWidth(FromfpFunc func) {
+    for (int rnd : MATH_ROUNDING_DIRECTIONS_INCLUDING_UNKNOWN) {
+      EXPECT_FP_EQ(zero, func(zero, rnd, 32U));
+      EXPECT_FP_EQ(neg_zero, func(neg_zero, rnd, 32U));
+
+      EXPECT_FP_EQ(aNaN, func(inf, rnd, 32U));
+      EXPECT_FP_EQ(aNaN, func(neg_inf, rnd, 32U));
+
+      EXPECT_FP_EQ(aNaN, func(aNaN, rnd, 32U));
+    }
+  }
+
+  void testSpecialNumbersZeroWidth(FromfpFunc func) {
+    for (int rnd : MATH_ROUNDING_DIRECTIONS_INCLUDING_UNKNOWN) {
+      EXPECT_FP_EQ(aNaN, func(zero, rnd, 0U));
+      EXPECT_FP_EQ(aNaN, func(neg_zero, rnd, 0U));
+
+      EXPECT_FP_EQ(aNaN, func(inf, rnd, 0U));
+      EXPECT_FP_EQ(aNaN, func(neg_inf, rnd, 0U));
+
+      EXPECT_FP_EQ(aNaN, func(aNaN, rnd, 0U));
+    }
+  }
+
+  void testRoundedNumbersWithinRange(FromfpFunc func) {
+    for (int rnd : MATH_ROUNDING_DIRECTIONS_INCLUDING_UNKNOWN) {
+      EXPECT_FP_EQ(T(1.0), func(T(1.0), rnd, 2U));
+      EXPECT_FP_EQ(T(-1.0), func(T(-1.0), rnd, 1U));
+      EXPECT_FP_EQ(T(10.0), func(T(10.0), rnd, 5U));
+      EXPECT_FP_EQ(T(-10.0), func(T(-10.0), rnd, 5U));
+      EXPECT_FP_EQ(T(1234.0), func(T(1234.0), rnd, 12U));
+      EXPECT_FP_EQ(T(-1234.0), func(T(-1234.0), rnd, 12U));
+    }
+  }
+
+  void testRoundedNumbersOutsideRange(FromfpFunc func) {
+    for (int rnd : MATH_ROUNDING_DIRECTIONS_INCLUDING_UNKNOWN) {
+      EXPECT_FP_EQ(aNaN, func(T(1.0), rnd, 1U));
+      EXPECT_FP_EQ(aNaN, func(T(10.0), rnd, 4U));
+      EXPECT_FP_EQ(aNaN, func(T(-10.0), rnd, 4U));
+      EXPECT_FP_EQ(aNaN, func(T(1234.0), rnd, 11U));
+      EXPECT_FP_EQ(aNaN, func(T(-1234.0), rnd, 11U));
+    }
+  }
+
+  void testFractionsUpwardWithinRange(FromfpFunc func) {
+    EXPECT_FP_EQ(T(1.0), func(T(0.5), FP_INT_UPWARD, 2U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.5), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(0.115), FP_INT_UPWARD, 2U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.115), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(0.715), FP_INT_UPWARD, 2U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.715), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(T(2.0), func(T(1.3), FP_INT_UPWARD, 3U));
+    EXPECT_FP_EQ(T(-1.0), func(T(-1.3), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(T(2.0), func(T(1.5), FP_INT_UPWARD, 3U));
+    EXPECT_FP_EQ(T(-1.0), func(T(-1.5), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(T(2.0), func(T(1.75), FP_INT_UPWARD, 3U));
+    EXPECT_FP_EQ(T(-1.0), func(T(-1.75), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(T(11.0), func(T(10.32), FP_INT_UPWARD, 5U));
+    EXPECT_FP_EQ(T(-10.0), func(T(-10.32), FP_INT_UPWARD, 5U));
+    EXPECT_FP_EQ(T(11.0), func(T(10.65), FP_INT_UPWARD, 5U));
+    EXPECT_FP_EQ(T(-10.0), func(T(-10.65), FP_INT_UPWARD, 5U));
+    EXPECT_FP_EQ(T(1235.0), func(T(1234.38), FP_INT_UPWARD, 12U));
+    EXPECT_FP_EQ(T(-1234.0), func(T(-1234.38), FP_INT_UPWARD, 12U));
+    EXPECT_FP_EQ(T(1235.0), func(T(1234.96), FP_INT_UPWARD, 12U));
+    EXPECT_FP_EQ(T(-1234.0), func(T(-1234.96), FP_INT_UPWARD, 12U));
+  }
+
+  void testFractionsUpwardOutsideRange(FromfpFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(0.5), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(0.115), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(0.715), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.3), FP_INT_UPWARD, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), FP_INT_UPWARD, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), FP_INT_UPWARD, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_UPWARD, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_UPWARD, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_UPWARD, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_UPWARD, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_UPWARD, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_UPWARD, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_UPWARD, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_UPWARD, 11U));
+  }
+
+  void testFractionsDownwardWithinRange(FromfpFunc func) {
+    EXPECT_FP_EQ(T(0.0), func(T(0.5), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(T(-1.0), func(T(-0.5), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(T(0.0), func(T(0.115), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(T(-1.0), func(T(-0.115), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(T(0.0), func(T(0.715), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(T(-1.0), func(T(-0.715), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(1.3), FP_INT_DOWNWARD, 2U));
+    EXPECT_FP_EQ(T(-2.0), func(T(-1.3), FP_INT_DOWNWARD, 2U));
+    EXPECT_FP_EQ(T(1.0), func(T(1.5), FP_INT_DOWNWARD, 2U));
+    EXPECT_FP_EQ(T(-2.0), func(T(-1.5), FP_INT_DOWNWARD, 2U));
+    EXPECT_FP_EQ(T(1.0), func(T(1.75), FP_INT_DOWNWARD, 2U));
+    EXPECT_FP_EQ(T(-2.0), func(T(-1.75), FP_INT_DOWNWARD, 2U));
+    EXPECT_FP_EQ(T(10.0), func(T(10.32), FP_INT_DOWNWARD, 5U));
+    EXPECT_FP_EQ(T(-11.0), func(T(-10.32), FP_INT_DOWNWARD, 5U));
+    EXPECT_FP_EQ(T(10.0), func(T(10.65), FP_INT_DOWNWARD, 5U));
+    EXPECT_FP_EQ(T(-11.0), func(T(-10.65), FP_INT_DOWNWARD, 5U));
+    EXPECT_FP_EQ(T(1234.0), func(T(1234.38), FP_INT_DOWNWARD, 12U));
+    EXPECT_FP_EQ(T(-1235.0), func(T(-1234.38), FP_INT_DOWNWARD, 12U));
+    EXPECT_FP_EQ(T(1234.0), func(T(1234.96), FP_INT_DOWNWARD, 12U));
+    EXPECT_FP_EQ(T(-1235.0), func(T(-1234.96), FP_INT_DOWNWARD, 12U));
+  }
+
+  void testFractionsDownwardOutsideRange(FromfpFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(1.3), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.3), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_DOWNWARD, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_DOWNWARD, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_DOWNWARD, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_DOWNWARD, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_DOWNWARD, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_DOWNWARD, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_DOWNWARD, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_DOWNWARD, 11U));
+  }
+
+  void testFractionsTowardZeroWithinRange(FromfpFunc func) {
+    EXPECT_FP_EQ(T(0.0), func(T(0.5), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.5), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(T(0.0), func(T(0.115), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.115), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(T(0.0), func(T(0.715), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.715), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(1.3), FP_INT_TOWARDZERO, 2U));
+    EXPECT_FP_EQ(T(-1.0), func(T(-1.3), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(1.5), FP_INT_TOWARDZERO, 2U));
+    EXPECT_FP_EQ(T(-1.0), func(T(-1.5), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(1.75), FP_INT_TOWARDZERO, 2U));
+    EXPECT_FP_EQ(T(-1.0), func(T(-1.75), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(T(10.0), func(T(10.32), FP_INT_TOWARDZERO, 5U));
+    EXPECT_FP_EQ(T(-10.0), func(T(-10.32), FP_INT_TOWARDZERO, 5U));
+    EXPECT_FP_EQ(T(10.0), func(T(10.65), FP_INT_TOWARDZERO, 5U));
+    EXPECT_FP_EQ(T(-10.0), func(T(-10.65), FP_INT_TOWARDZERO, 5U));
+    EXPECT_FP_EQ(T(1234.0), func(T(1234.38), FP_INT_TOWARDZERO, 12U));
+    EXPECT_FP_EQ(T(-1234.0), func(T(-1234.38), FP_INT_TOWARDZERO, 12U));
+    EXPECT_FP_EQ(T(1234.0), func(T(1234.96), FP_INT_TOWARDZERO, 12U));
+    EXPECT_FP_EQ(T(-1234.0), func(T(-1234.96), FP_INT_TOWARDZERO, 12U));
+  }
+
+  void testFractionsTowardZeroOutsideRange(FromfpFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(1.3), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_TOWARDZERO, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_TOWARDZERO, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_TOWARDZERO, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_TOWARDZERO, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_TOWARDZERO, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_TOWARDZERO, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_TOWARDZERO, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_TOWARDZERO, 11U));
+  }
+
+  void testFractionsToNearestFromZeroWithinRange(FromfpFunc func) {
+    EXPECT_FP_EQ(T(1.0), func(T(0.5), FP_INT_TONEARESTFROMZERO, 2U));
+    EXPECT_FP_EQ(T(-1.0), func(T(-0.5), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(T(0.0), func(T(0.115), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.115), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(0.715), FP_INT_TONEARESTFROMZERO, 2U));
+    EXPECT_FP_EQ(T(-1.0), func(T(-0.715), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(1.3), FP_INT_TONEARESTFROMZERO, 2U));
+    EXPECT_FP_EQ(T(-1.0), func(T(-1.3), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(T(2.0), func(T(1.5), FP_INT_TONEARESTFROMZERO, 3U));
+    EXPECT_FP_EQ(T(-2.0), func(T(-1.5), FP_INT_TONEARESTFROMZERO, 2U));
+    EXPECT_FP_EQ(T(2.0), func(T(1.75), FP_INT_TONEARESTFROMZERO, 3U));
+    EXPECT_FP_EQ(T(-2.0), func(T(-1.75), FP_INT_TONEARESTFROMZERO, 2U));
+    EXPECT_FP_EQ(T(10.0), func(T(10.32), FP_INT_TONEARESTFROMZERO, 5U));
+    EXPECT_FP_EQ(T(-10.0), func(T(-10.32), FP_INT_TONEARESTFROMZERO, 5U));
+    EXPECT_FP_EQ(T(11.0), func(T(10.65), FP_INT_TONEARESTFROMZERO, 5U));
+    EXPECT_FP_EQ(T(-11.0), func(T(-10.65), FP_INT_TONEARESTFROMZERO, 5U));
+    EXPECT_FP_EQ(T(1234.0), func(T(1234.38), FP_INT_TONEARESTFROMZERO, 12U));
+    EXPECT_FP_EQ(T(-1234.0), func(T(-1234.38), FP_INT_TONEARESTFROMZERO, 12U));
+    EXPECT_FP_EQ(T(1235.0), func(T(1234.96), FP_INT_TONEARESTFROMZERO, 12U));
+    EXPECT_FP_EQ(T(-1235.0), func(T(-1234.96), FP_INT_TONEARESTFROMZERO, 12U));
+  }
+
+  void testFractionsToNearestFromZeroOutsideRange(FromfpFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(0.5), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(0.715), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.3), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), FP_INT_TONEARESTFROMZERO, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), FP_INT_TONEARESTFROMZERO, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_TONEARESTFROMZERO, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_TONEARESTFROMZERO, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_TONEARESTFROMZERO, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_TONEARESTFROMZERO, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_TONEARESTFROMZERO, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_TONEARESTFROMZERO, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_TONEARESTFROMZERO, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_TONEARESTFROMZERO, 11U));
+  }
+
+  void testFractionsToNearestWithinRange(FromfpFunc func) {
+    EXPECT_FP_EQ(T(0.0), func(T(0.5), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.5), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(T(0.0), func(T(0.115), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.115), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(0.715), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(T(-1.0), func(T(-0.715), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(1.3), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(T(-1.0), func(T(-1.3), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(T(2.0), func(T(1.5), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(T(-2.0), func(T(-1.5), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(T(2.0), func(T(1.75), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(T(-2.0), func(T(-1.75), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(T(10.0), func(T(10.32), FP_INT_TONEAREST, 5U));
+    EXPECT_FP_EQ(T(-10.0), func(T(-10.32), FP_INT_TONEAREST, 5U));
+    EXPECT_FP_EQ(T(11.0), func(T(10.65), FP_INT_TONEAREST, 5U));
+    EXPECT_FP_EQ(T(-11.0), func(T(-10.65), FP_INT_TONEAREST, 5U));
+    EXPECT_FP_EQ(T(1234.0), func(T(1234.38), FP_INT_TONEAREST, 12U));
+    EXPECT_FP_EQ(T(-1234.0), func(T(-1234.38), FP_INT_TONEAREST, 12U));
+    EXPECT_FP_EQ(T(1235.0), func(T(1234.96), FP_INT_TONEAREST, 12U));
+    EXPECT_FP_EQ(T(-1235.0), func(T(-1234.96), FP_INT_TONEAREST, 12U));
+
+    EXPECT_FP_EQ(T(2.0), func(T(2.3), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(T(-2.0), func(T(-2.3), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(T(2.0), func(T(2.5), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(T(-2.0), func(T(-2.5), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(T(3.0), func(T(2.75), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(T(-3.0), func(T(-2.75), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(T(5.0), func(T(5.3), FP_INT_TONEAREST, 4U));
+    EXPECT_FP_EQ(T(-5.0), func(T(-5.3), FP_INT_TONEAREST, 4U));
+    EXPECT_FP_EQ(T(6.0), func(T(5.5), FP_INT_TONEAREST, 4U));
+    EXPECT_FP_EQ(T(-6.0), func(T(-5.5), FP_INT_TONEAREST, 4U));
+    EXPECT_FP_EQ(T(6.0), func(T(5.75), FP_INT_TONEAREST, 4U));
+    EXPECT_FP_EQ(T(-6.0), func(T(-5.75), FP_INT_TONEAREST, 4U));
+  }
+
+  void testFractionsToNearestOutsideRange(FromfpFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(0.715), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.3), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_TONEAREST, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_TONEAREST, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_TONEAREST, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_TONEAREST, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_TONEAREST, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_TONEAREST, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_TONEAREST, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_TONEAREST, 11U));
+
+    EXPECT_FP_EQ(aNaN, func(T(2.3), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.3), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(2.5), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.5), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(2.75), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.75), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(5.3), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.3), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(5.5), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.5), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(5.75), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.75), FP_INT_TONEAREST, 3U));
+  }
+
+  void testFractionsToNearestFallbackWithinRange(FromfpFunc func) {
+    EXPECT_FP_EQ(T(0.0), func(T(0.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(T(0.0), func(T(0.115), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.115), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(0.715), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(T(-1.0), func(T(-0.715), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(1.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(T(-1.0), func(T(-1.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(T(2.0), func(T(1.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(T(-2.0), func(T(-1.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(T(2.0), func(T(1.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(T(-2.0), func(T(-1.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(T(10.0), func(T(10.32), UNKNOWN_MATH_ROUNDING_DIRECTION, 5U));
+    EXPECT_FP_EQ(T(-10.0),
+                 func(T(-10.32), UNKNOWN_MATH_ROUNDING_DIRECTION, 5U));
+    EXPECT_FP_EQ(T(11.0), func(T(10.65), UNKNOWN_MATH_ROUNDING_DIRECTION, 5U));
+    EXPECT_FP_EQ(T(-11.0),
+                 func(T(-10.65), UNKNOWN_MATH_ROUNDING_DIRECTION, 5U));
+    EXPECT_FP_EQ(T(1234.0),
+                 func(T(1234.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 12U));
+    EXPECT_FP_EQ(T(-1234.0),
+                 func(T(-1234.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 12U));
+    EXPECT_FP_EQ(T(1235.0),
+                 func(T(1234.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 12U));
+    EXPECT_FP_EQ(T(-1235.0),
+                 func(T(-1234.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 12U));
+
+    EXPECT_FP_EQ(T(2.0), func(T(2.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(T(-2.0), func(T(-2.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(T(2.0), func(T(2.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(T(-2.0), func(T(-2.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(T(3.0), func(T(2.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(T(-3.0), func(T(-2.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(T(5.0), func(T(5.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U));
+    EXPECT_FP_EQ(T(-5.0), func(T(-5.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U));
+    EXPECT_FP_EQ(T(6.0), func(T(5.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U));
+    EXPECT_FP_EQ(T(-6.0), func(T(-5.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U));
+    EXPECT_FP_EQ(T(6.0), func(T(5.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U));
+    EXPECT_FP_EQ(T(-6.0), func(T(-5.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U));
+  }
+
+  void testFractionsToNearestFallbackOutsideRange(FromfpFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(0.715), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 11U));
+
+    EXPECT_FP_EQ(aNaN, func(T(2.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(2.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(2.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(5.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(5.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(5.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+  }
+};
+
+#define LIST_FROMFP_TESTS(T, func)                                             \
+  using LlvmLibcFromfpTest = FromfpTestTemplate<T>;                            \
+  TEST_F(LlvmLibcFromfpTest, SpecialNumbersNonzeroWidth) {                     \
+    testSpecialNumbersNonzeroWidth(&func);                                     \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpTest, SpecialNumbersZeroWidth) {                        \
+    testSpecialNumbersZeroWidth(&func);                                        \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpTest, RoundedNumbersWithinRange) {                      \
+    testRoundedNumbersWithinRange(&func);                                      \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpTest, RoundedNumbersOutsideRange) {                     \
+    testRoundedNumbersOutsideRange(&func);                                     \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpTest, FractionsUpwardWithinRange) {                     \
+    testFractionsUpwardWithinRange(&func);                                     \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpTest, FractionsUpwardOutsideRange) {                    \
+    testFractionsUpwardOutsideRange(&func);                                    \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpTest, FractionsDownwardWithinRange) {                   \
+    testFractionsDownwardWithinRange(&func);                                   \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpTest, FractionsDownwardOutsideRange) {                  \
+    testFractionsDownwardOutsideRange(&func);                                  \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpTest, FractionsTowardZeroWithinRange) {                 \
+    testFractionsTowardZeroWithinRange(&func);                                 \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpTest, FractionsTowardZeroOutsideRange) {                \
+    testFractionsTowardZeroOutsideRange(&func);                                \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpTest, FractionsToNearestFromZeroWithinRange) {          \
+    testFractionsToNearestFromZeroWithinRange(&func);                          \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpTest, FractionsToNearestFromZeroOutsideRange) {         \
+    testFractionsToNearestFromZeroOutsideRange(&func);                         \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpTest, FractionsToNearestWithinRange) {                  \
+    testFractionsToNearestWithinRange(&func);                                  \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpTest, FractionsToNearestOutsideRange) {                 \
+    testFractionsToNearestOutsideRange(&func);                                 \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpTest, FractionsToNearestFallbackWithinRange) {          \
+    testFractionsToNearestFallbackWithinRange(&func);                          \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpTest, FractionsToNearestFallbackOutsideRange) {         \
+    testFractionsToNearestFallbackOutsideRange(&func);                         \
+  }
+
+#endif // LIBC_TEST_SRC_MATH_SMOKE_FROMFPTEST_H
diff --git a/libc/test/src/math/smoke/FromfpxTest.h b/libc/test/src/math/smoke/FromfpxTest.h
new file mode 100644
index 000000000000..b3c14c7c09ba
--- /dev/null
+++ b/libc/test/src/math/smoke/FromfpxTest.h
@@ -0,0 +1,578 @@
+//===-- Utility class to test different flavors of fromfpx ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LIBC_TEST_SRC_MATH_SMOKE_FROMFPXTEST_H
+#define LIBC_TEST_SRC_MATH_SMOKE_FROMFPXTEST_H
+
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+template <typename T>
+class FromfpxTestTemplate : public LIBC_NAMESPACE::testing::Test {
+
+  DECLARE_SPECIAL_CONSTANTS(T)
+
+public:
+  typedef T (*FromfpxFunc)(T, int, unsigned int);
+
+  void testSpecialNumbersNonzeroWidth(FromfpxFunc func) {
+    for (int rnd : MATH_ROUNDING_DIRECTIONS_INCLUDING_UNKNOWN) {
+      EXPECT_FP_EQ(zero, func(zero, rnd, 32U));
+      EXPECT_FP_EQ(neg_zero, func(neg_zero, rnd, 32U));
+
+      EXPECT_FP_EQ(aNaN, func(inf, rnd, 32U));
+      EXPECT_FP_EQ(aNaN, func(neg_inf, rnd, 32U));
+
+      EXPECT_FP_EQ(aNaN, func(aNaN, rnd, 32U));
+    }
+  }
+
+  void testSpecialNumbersZeroWidth(FromfpxFunc func) {
+    for (int rnd : MATH_ROUNDING_DIRECTIONS_INCLUDING_UNKNOWN) {
+      EXPECT_FP_EQ(aNaN, func(zero, rnd, 0U));
+      EXPECT_FP_EQ(aNaN, func(neg_zero, rnd, 0U));
+
+      EXPECT_FP_EQ(aNaN, func(inf, rnd, 0U));
+      EXPECT_FP_EQ(aNaN, func(neg_inf, rnd, 0U));
+
+      EXPECT_FP_EQ(aNaN, func(aNaN, rnd, 0U));
+    }
+  }
+
+  void testRoundedNumbersWithinRange(FromfpxFunc func) {
+    for (int rnd : MATH_ROUNDING_DIRECTIONS_INCLUDING_UNKNOWN) {
+      EXPECT_FP_EQ(T(1.0), func(T(1.0), rnd, 2U));
+      EXPECT_FP_EQ(T(-1.0), func(T(-1.0), rnd, 1U));
+      EXPECT_FP_EQ(T(10.0), func(T(10.0), rnd, 5U));
+      EXPECT_FP_EQ(T(-10.0), func(T(-10.0), rnd, 5U));
+      EXPECT_FP_EQ(T(1234.0), func(T(1234.0), rnd, 12U));
+      EXPECT_FP_EQ(T(-1234.0), func(T(-1234.0), rnd, 12U));
+    }
+  }
+
+  void testRoundedNumbersOutsideRange(FromfpxFunc func) {
+    for (int rnd : MATH_ROUNDING_DIRECTIONS_INCLUDING_UNKNOWN) {
+      EXPECT_FP_EQ(aNaN, func(T(1.0), rnd, 1U));
+      EXPECT_FP_EQ(aNaN, func(T(10.0), rnd, 4U));
+      EXPECT_FP_EQ(aNaN, func(T(-10.0), rnd, 4U));
+      EXPECT_FP_EQ(aNaN, func(T(1234.0), rnd, 11U));
+      EXPECT_FP_EQ(aNaN, func(T(-1234.0), rnd, 11U));
+    }
+  }
+
+  void testFractionsUpwardWithinRange(FromfpxFunc func) {
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(0.5), FP_INT_UPWARD, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-0.0), func(T(-0.5), FP_INT_UPWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(0.115), FP_INT_UPWARD, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-0.0), func(T(-0.115), FP_INT_UPWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(0.715), FP_INT_UPWARD, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-0.0), func(T(-0.715), FP_INT_UPWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(2.0), func(T(1.3), FP_INT_UPWARD, 3U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-1.0), func(T(-1.3), FP_INT_UPWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(2.0), func(T(1.5), FP_INT_UPWARD, 3U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-1.0), func(T(-1.5), FP_INT_UPWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(2.0), func(T(1.75), FP_INT_UPWARD, 3U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-1.0), func(T(-1.75), FP_INT_UPWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(11.0), func(T(10.32), FP_INT_UPWARD, 5U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-10.0), func(T(-10.32), FP_INT_UPWARD, 5U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(11.0), func(T(10.65), FP_INT_UPWARD, 5U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-10.0), func(T(-10.65), FP_INT_UPWARD, 5U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1235.0), func(T(1234.38), FP_INT_UPWARD, 12U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-1234.0), func(T(-1234.38), FP_INT_UPWARD, 12U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1235.0), func(T(1234.96), FP_INT_UPWARD, 12U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-1234.0), func(T(-1234.96), FP_INT_UPWARD, 12U), FE_INEXACT);
+  }
+
+  void testFractionsUpwardOutsideRange(FromfpxFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(0.5), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(0.115), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(0.715), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.3), FP_INT_UPWARD, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), FP_INT_UPWARD, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), FP_INT_UPWARD, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_UPWARD, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_UPWARD, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_UPWARD, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_UPWARD, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_UPWARD, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_UPWARD, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_UPWARD, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_UPWARD, 11U));
+  }
+
+  void testFractionsDownwardWithinRange(FromfpxFunc func) {
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(0.0), func(T(0.5), FP_INT_DOWNWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-1.0), func(T(-0.5), FP_INT_DOWNWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(0.0), func(T(0.115), FP_INT_DOWNWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-1.0), func(T(-0.115), FP_INT_DOWNWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(0.0), func(T(0.715), FP_INT_DOWNWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-1.0), func(T(-0.715), FP_INT_DOWNWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(1.3), FP_INT_DOWNWARD, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-2.0), func(T(-1.3), FP_INT_DOWNWARD, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(1.5), FP_INT_DOWNWARD, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-2.0), func(T(-1.5), FP_INT_DOWNWARD, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(1.75), FP_INT_DOWNWARD, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-2.0), func(T(-1.75), FP_INT_DOWNWARD, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(10.0), func(T(10.32), FP_INT_DOWNWARD, 5U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-11.0), func(T(-10.32), FP_INT_DOWNWARD, 5U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(10.0), func(T(10.65), FP_INT_DOWNWARD, 5U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-11.0), func(T(-10.65), FP_INT_DOWNWARD, 5U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1234.0), func(T(1234.38), FP_INT_DOWNWARD, 12U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-1235.0), func(T(-1234.38), FP_INT_DOWNWARD, 12U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1234.0), func(T(1234.96), FP_INT_DOWNWARD, 12U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-1235.0), func(T(-1234.96), FP_INT_DOWNWARD, 12U), FE_INEXACT);
+  }
+
+  void testFractionsDownwardOutsideRange(FromfpxFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(1.3), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.3), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_DOWNWARD, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_DOWNWARD, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_DOWNWARD, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_DOWNWARD, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_DOWNWARD, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_DOWNWARD, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_DOWNWARD, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_DOWNWARD, 11U));
+  }
+
+  void testFractionsTowardZeroWithinRange(FromfpxFunc func) {
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(0.0), func(T(0.5), FP_INT_TOWARDZERO, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-0.0), func(T(-0.5), FP_INT_TOWARDZERO, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(0.0), func(T(0.115), FP_INT_TOWARDZERO, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-0.0), func(T(-0.115), FP_INT_TOWARDZERO, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(0.0), func(T(0.715), FP_INT_TOWARDZERO, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-0.0), func(T(-0.715), FP_INT_TOWARDZERO, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(1.3), FP_INT_TOWARDZERO, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-1.0), func(T(-1.3), FP_INT_TOWARDZERO, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(1.5), FP_INT_TOWARDZERO, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-1.0), func(T(-1.5), FP_INT_TOWARDZERO, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(1.75), FP_INT_TOWARDZERO, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-1.0), func(T(-1.75), FP_INT_TOWARDZERO, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(10.0), func(T(10.32), FP_INT_TOWARDZERO, 5U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-10.0), func(T(-10.32), FP_INT_TOWARDZERO, 5U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(10.0), func(T(10.65), FP_INT_TOWARDZERO, 5U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-10.0), func(T(-10.65), FP_INT_TOWARDZERO, 5U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1234.0), func(T(1234.38), FP_INT_TOWARDZERO, 12U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-1234.0), func(T(-1234.38), FP_INT_TOWARDZERO, 12U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1234.0), func(T(1234.96), FP_INT_TOWARDZERO, 12U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-1234.0), func(T(-1234.96), FP_INT_TOWARDZERO, 12U), FE_INEXACT);
+  }
+
+  void testFractionsTowardZeroOutsideRange(FromfpxFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(1.3), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_TOWARDZERO, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_TOWARDZERO, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_TOWARDZERO, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_TOWARDZERO, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_TOWARDZERO, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_TOWARDZERO, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_TOWARDZERO, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_TOWARDZERO, 11U));
+  }
+
+  void testFractionsToNearestFromZeroWithinRange(FromfpxFunc func) {
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1.0), func(T(0.5), FP_INT_TONEARESTFROMZERO, 2U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-1.0), func(T(-0.5), FP_INT_TONEARESTFROMZERO, 1U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(0.0), func(T(0.115), FP_INT_TONEARESTFROMZERO, 1U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-0.0), func(T(-0.115), FP_INT_TONEARESTFROMZERO, 1U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1.0), func(T(0.715), FP_INT_TONEARESTFROMZERO, 2U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-1.0), func(T(-0.715), FP_INT_TONEARESTFROMZERO, 1U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1.0), func(T(1.3), FP_INT_TONEARESTFROMZERO, 2U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-1.0), func(T(-1.3), FP_INT_TONEARESTFROMZERO, 1U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(2.0), func(T(1.5), FP_INT_TONEARESTFROMZERO, 3U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-2.0), func(T(-1.5), FP_INT_TONEARESTFROMZERO, 2U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(2.0), func(T(1.75), FP_INT_TONEARESTFROMZERO, 3U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-2.0), func(T(-1.75), FP_INT_TONEARESTFROMZERO, 2U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(10.0), func(T(10.32), FP_INT_TONEARESTFROMZERO, 5U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-10.0), func(T(-10.32), FP_INT_TONEARESTFROMZERO, 5U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(11.0), func(T(10.65), FP_INT_TONEARESTFROMZERO, 5U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-11.0), func(T(-10.65), FP_INT_TONEARESTFROMZERO, 5U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1234.0), func(T(1234.38), FP_INT_TONEARESTFROMZERO, 12U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-1234.0), func(T(-1234.38), FP_INT_TONEARESTFROMZERO, 12U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1235.0), func(T(1234.96), FP_INT_TONEARESTFROMZERO, 12U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-1235.0), func(T(-1234.96), FP_INT_TONEARESTFROMZERO, 12U),
+        FE_INEXACT);
+  }
+
+  void testFractionsToNearestFromZeroOutsideRange(FromfpxFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(0.5), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(0.715), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.3), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), FP_INT_TONEARESTFROMZERO, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), FP_INT_TONEARESTFROMZERO, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_TONEARESTFROMZERO, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_TONEARESTFROMZERO, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_TONEARESTFROMZERO, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_TONEARESTFROMZERO, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_TONEARESTFROMZERO, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_TONEARESTFROMZERO, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_TONEARESTFROMZERO, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_TONEARESTFROMZERO, 11U));
+  }
+
+  void testFractionsToNearestWithinRange(FromfpxFunc func) {
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(0.0), func(T(0.5), FP_INT_TONEAREST, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-0.0), func(T(-0.5), FP_INT_TONEAREST, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(0.0), func(T(0.115), FP_INT_TONEAREST, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-0.0), func(T(-0.115), FP_INT_TONEAREST, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(0.715), FP_INT_TONEAREST, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-1.0), func(T(-0.715), FP_INT_TONEAREST, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(1.3), FP_INT_TONEAREST, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-1.0), func(T(-1.3), FP_INT_TONEAREST, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(2.0), func(T(1.5), FP_INT_TONEAREST, 3U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-2.0), func(T(-1.5), FP_INT_TONEAREST, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(2.0), func(T(1.75), FP_INT_TONEAREST, 3U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-2.0), func(T(-1.75), FP_INT_TONEAREST, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(10.0), func(T(10.32), FP_INT_TONEAREST, 5U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-10.0), func(T(-10.32), FP_INT_TONEAREST, 5U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(11.0), func(T(10.65), FP_INT_TONEAREST, 5U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-11.0), func(T(-10.65), FP_INT_TONEAREST, 5U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1234.0), func(T(1234.38), FP_INT_TONEAREST, 12U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-1234.0), func(T(-1234.38), FP_INT_TONEAREST, 12U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1235.0), func(T(1234.96), FP_INT_TONEAREST, 12U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-1235.0), func(T(-1234.96), FP_INT_TONEAREST, 12U), FE_INEXACT);
+
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(2.0), func(T(2.3), FP_INT_TONEAREST, 3U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-2.0), func(T(-2.3), FP_INT_TONEAREST, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(2.0), func(T(2.5), FP_INT_TONEAREST, 3U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-2.0), func(T(-2.5), FP_INT_TONEAREST, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(3.0), func(T(2.75), FP_INT_TONEAREST, 3U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-3.0), func(T(-2.75), FP_INT_TONEAREST, 3U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(5.0), func(T(5.3), FP_INT_TONEAREST, 4U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-5.0), func(T(-5.3), FP_INT_TONEAREST, 4U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(6.0), func(T(5.5), FP_INT_TONEAREST, 4U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-6.0), func(T(-5.5), FP_INT_TONEAREST, 4U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(6.0), func(T(5.75), FP_INT_TONEAREST, 4U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-6.0), func(T(-5.75), FP_INT_TONEAREST, 4U),
+                                FE_INEXACT);
+  }
+
+  void testFractionsToNearestOutsideRange(FromfpxFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(0.715), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.3), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_TONEAREST, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_TONEAREST, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_TONEAREST, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_TONEAREST, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_TONEAREST, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_TONEAREST, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_TONEAREST, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_TONEAREST, 11U));
+
+    EXPECT_FP_EQ(aNaN, func(T(2.3), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.3), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(2.5), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.5), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(2.75), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.75), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(5.3), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.3), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(5.5), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.5), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(5.75), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.75), FP_INT_TONEAREST, 3U));
+  }
+
+  void testFractionsToNearestFallbackWithinRange(FromfpxFunc func) {
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(0.0), func(T(0.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-0.0), func(T(-0.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(0.0), func(T(0.115), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-0.0), func(T(-0.115), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1.0), func(T(0.715), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-1.0), func(T(-0.715), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1.0), func(T(1.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-1.0), func(T(-1.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(2.0), func(T(1.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-2.0), func(T(-1.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(2.0), func(T(1.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-2.0), func(T(-1.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(10.0), func(T(10.32), UNKNOWN_MATH_ROUNDING_DIRECTION, 5U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-10.0), func(T(-10.32), UNKNOWN_MATH_ROUNDING_DIRECTION, 5U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(11.0), func(T(10.65), UNKNOWN_MATH_ROUNDING_DIRECTION, 5U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-11.0), func(T(-10.65), UNKNOWN_MATH_ROUNDING_DIRECTION, 5U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1234.0), func(T(1234.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 12U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-1234.0), func(T(-1234.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 12U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1235.0), func(T(1234.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 12U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-1235.0), func(T(-1234.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 12U),
+        FE_INEXACT);
+
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(2.0), func(T(2.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-2.0), func(T(-2.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(2.0), func(T(2.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-2.0), func(T(-2.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(3.0), func(T(2.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-3.0), func(T(-2.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(5.0), func(T(5.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-5.0), func(T(-5.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(6.0), func(T(5.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-6.0), func(T(-5.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(6.0), func(T(5.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-6.0), func(T(-5.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U),
+        FE_INEXACT);
+  }
+
+  void testFractionsToNearestFallbackOutsideRange(FromfpxFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(0.715), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 11U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 11U));
+
+    EXPECT_FP_EQ(aNaN, func(T(2.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(2.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(2.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(5.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(5.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(5.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+  }
+};
+
+#define LIST_FROMFPX_TESTS(T, func)                                            \
+  using LlvmLibcFromfpxTest = FromfpxTestTemplate<T>;                          \
+  TEST_F(LlvmLibcFromfpxTest, SpecialNumbersNonzeroWidth) {                    \
+    testSpecialNumbersNonzeroWidth(&func);                                     \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpxTest, SpecialNumbersZeroWidth) {                       \
+    testSpecialNumbersZeroWidth(&func);                                        \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpxTest, RoundedNumbersWithinRange) {                     \
+    testRoundedNumbersWithinRange(&func);                                      \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpxTest, RoundedNumbersOutsideRange) {                    \
+    testRoundedNumbersOutsideRange(&func);                                     \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpxTest, FractionsUpwardWithinRange) {                    \
+    testFractionsUpwardWithinRange(&func);                                     \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpxTest, FractionsUpwardOutsideRange) {                   \
+    testFractionsUpwardOutsideRange(&func);                                    \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpxTest, FractionsDownwardWithinRange) {                  \
+    testFractionsDownwardWithinRange(&func);                                   \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpxTest, FractionsDownwardOutsideRange) {                 \
+    testFractionsDownwardOutsideRange(&func);                                  \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpxTest, FractionsTowardZeroWithinRange) {                \
+    testFractionsTowardZeroWithinRange(&func);                                 \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpxTest, FractionsTowardZeroOutsideRange) {               \
+    testFractionsTowardZeroOutsideRange(&func);                                \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpxTest, FractionsToNearestFromZeroWithinRange) {         \
+    testFractionsToNearestFromZeroWithinRange(&func);                          \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpxTest, FractionsToNearestFromZeroOutsideRange) {        \
+    testFractionsToNearestFromZeroOutsideRange(&func);                         \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpxTest, FractionsToNearestWithinRange) {                 \
+    testFractionsToNearestWithinRange(&func);                                  \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpxTest, FractionsToNearestOutsideRange) {                \
+    testFractionsToNearestOutsideRange(&func);                                 \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpxTest, FractionsToNearestFallbackWithinRange) {         \
+    testFractionsToNearestFallbackWithinRange(&func);                          \
+  }                                                                            \
+  TEST_F(LlvmLibcFromfpxTest, FractionsToNearestFallbackOutsideRange) {        \
+    testFractionsToNearestFallbackOutsideRange(&func);                         \
+  }
+
+#endif // LIBC_TEST_SRC_MATH_SMOKE_FROMFPXTEST_H
diff --git a/libc/test/src/math/smoke/UfromfpTest.h b/libc/test/src/math/smoke/UfromfpTest.h
new file mode 100644
index 000000000000..e753fc489cde
--- /dev/null
+++ b/libc/test/src/math/smoke/UfromfpTest.h
@@ -0,0 +1,348 @@
+//===-- Utility class to test different flavors of ufromfp ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LIBC_TEST_SRC_MATH_SMOKE_UFROMFPTEST_H
+#define LIBC_TEST_SRC_MATH_SMOKE_UFROMFPTEST_H
+
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+template <typename T>
+class UfromfpTestTemplate : public LIBC_NAMESPACE::testing::Test {
+
+  DECLARE_SPECIAL_CONSTANTS(T)
+
+public:
+  typedef T (*UfromfpFunc)(T, int, unsigned int);
+
+  void testSpecialNumbersNonzeroWidth(UfromfpFunc func) {
+    for (int rnd : MATH_ROUNDING_DIRECTIONS_INCLUDING_UNKNOWN) {
+      EXPECT_FP_EQ(zero, func(zero, rnd, 32U));
+      EXPECT_FP_EQ(neg_zero, func(neg_zero, rnd, 32U));
+
+      EXPECT_FP_EQ(aNaN, func(inf, rnd, 32U));
+      EXPECT_FP_EQ(aNaN, func(neg_inf, rnd, 32U));
+
+      EXPECT_FP_EQ(aNaN, func(aNaN, rnd, 32U));
+    }
+  }
+
+  void testSpecialNumbersZeroWidth(UfromfpFunc func) {
+    for (int rnd : MATH_ROUNDING_DIRECTIONS_INCLUDING_UNKNOWN) {
+      EXPECT_FP_EQ(aNaN, func(zero, rnd, 0U));
+      EXPECT_FP_EQ(aNaN, func(neg_zero, rnd, 0U));
+
+      EXPECT_FP_EQ(aNaN, func(inf, rnd, 0U));
+      EXPECT_FP_EQ(aNaN, func(neg_inf, rnd, 0U));
+
+      EXPECT_FP_EQ(aNaN, func(aNaN, rnd, 0U));
+    }
+  }
+
+  void testRoundedNumbersWithinRange(UfromfpFunc func) {
+    for (int rnd : MATH_ROUNDING_DIRECTIONS_INCLUDING_UNKNOWN) {
+      EXPECT_FP_EQ(T(1.0), func(T(1.0), rnd, 1U));
+      EXPECT_FP_EQ(T(10.0), func(T(10.0), rnd, 4U));
+      EXPECT_FP_EQ(T(1234.0), func(T(1234.0), rnd, 11U));
+    }
+  }
+
+  void testRoundedNumbersOutsideRange(UfromfpFunc func) {
+    for (int rnd : MATH_ROUNDING_DIRECTIONS_INCLUDING_UNKNOWN) {
+      EXPECT_FP_EQ(aNaN, func(T(-1.0), rnd, 32U));
+      EXPECT_FP_EQ(aNaN, func(T(10.0), rnd, 3U));
+      EXPECT_FP_EQ(aNaN, func(T(-10.0), rnd, 32U));
+      EXPECT_FP_EQ(aNaN, func(T(1234.0), rnd, 10U));
+      EXPECT_FP_EQ(aNaN, func(T(-1234.0), rnd, 32U));
+    }
+  }
+
+  void testFractionsUpwardWithinRange(UfromfpFunc func) {
+    EXPECT_FP_EQ(T(1.0), func(T(0.5), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.5), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(0.115), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.115), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(0.715), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.715), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(T(2.0), func(T(1.3), FP_INT_UPWARD, 2U));
+    EXPECT_FP_EQ(T(2.0), func(T(1.5), FP_INT_UPWARD, 2U));
+    EXPECT_FP_EQ(T(2.0), func(T(1.75), FP_INT_UPWARD, 2U));
+    EXPECT_FP_EQ(T(11.0), func(T(10.32), FP_INT_UPWARD, 4U));
+    EXPECT_FP_EQ(T(11.0), func(T(10.65), FP_INT_UPWARD, 4U));
+    EXPECT_FP_EQ(T(1235.0), func(T(1234.38), FP_INT_UPWARD, 11U));
+    EXPECT_FP_EQ(T(1235.0), func(T(1234.96), FP_INT_UPWARD, 11U));
+  }
+
+  void testFractionsUpwardOutsideRange(UfromfpFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(1.3), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.3), FP_INT_UPWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), FP_INT_UPWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), FP_INT_UPWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_UPWARD, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_UPWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_UPWARD, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_UPWARD, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_UPWARD, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_UPWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_UPWARD, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_UPWARD, 32U));
+  }
+
+  void testFractionsDownwardWithinRange(UfromfpFunc func) {
+    EXPECT_FP_EQ(T(0.0), func(T(0.5), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(T(0.0), func(T(0.115), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(T(0.0), func(T(0.715), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(1.3), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(1.5), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(1.75), FP_INT_DOWNWARD, 1U));
+    EXPECT_FP_EQ(T(10.0), func(T(10.32), FP_INT_DOWNWARD, 4U));
+    EXPECT_FP_EQ(T(10.0), func(T(10.65), FP_INT_DOWNWARD, 4U));
+    EXPECT_FP_EQ(T(1234.0), func(T(1234.38), FP_INT_DOWNWARD, 11U));
+    EXPECT_FP_EQ(T(1234.0), func(T(1234.96), FP_INT_DOWNWARD, 11U));
+  }
+
+  void testFractionsDownwardOutsideRange(UfromfpFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(-0.5), FP_INT_DOWNWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-0.115), FP_INT_DOWNWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-0.715), FP_INT_DOWNWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.3), FP_INT_DOWNWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), FP_INT_DOWNWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), FP_INT_DOWNWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_DOWNWARD, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_DOWNWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_DOWNWARD, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_DOWNWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_DOWNWARD, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_DOWNWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_DOWNWARD, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_DOWNWARD, 32U));
+  }
+
+  void testFractionsTowardZeroWithinRange(UfromfpFunc func) {
+    EXPECT_FP_EQ(T(0.0), func(T(0.5), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.5), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(T(0.0), func(T(0.115), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.115), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(T(0.0), func(T(0.715), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.715), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(1.3), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(1.5), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(1.75), FP_INT_TOWARDZERO, 1U));
+    EXPECT_FP_EQ(T(10.0), func(T(10.32), FP_INT_TOWARDZERO, 4U));
+    EXPECT_FP_EQ(T(10.0), func(T(10.65), FP_INT_TOWARDZERO, 4U));
+    EXPECT_FP_EQ(T(1234.0), func(T(1234.38), FP_INT_TOWARDZERO, 11U));
+    EXPECT_FP_EQ(T(1234.0), func(T(1234.96), FP_INT_TOWARDZERO, 11U));
+  }
+
+  void testFractionsTowardZeroOutsideRange(UfromfpFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(-1.3), FP_INT_TOWARDZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), FP_INT_TOWARDZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), FP_INT_TOWARDZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_TOWARDZERO, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_TOWARDZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_TOWARDZERO, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_TOWARDZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_TOWARDZERO, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_TOWARDZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_TOWARDZERO, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_TOWARDZERO, 32U));
+  }
+
+  void testFractionsToNearestFromZeroWithinRange(UfromfpFunc func) {
+    EXPECT_FP_EQ(T(1.0), func(T(0.5), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(T(0.0), func(T(0.115), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.115), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(0.715), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(1.3), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(T(2.0), func(T(1.5), FP_INT_TONEARESTFROMZERO, 2U));
+    EXPECT_FP_EQ(T(2.0), func(T(1.75), FP_INT_TONEARESTFROMZERO, 2U));
+    EXPECT_FP_EQ(T(10.0), func(T(10.32), FP_INT_TONEARESTFROMZERO, 4U));
+    EXPECT_FP_EQ(T(11.0), func(T(10.65), FP_INT_TONEARESTFROMZERO, 4U));
+    EXPECT_FP_EQ(T(1234.0), func(T(1234.38), FP_INT_TONEARESTFROMZERO, 11U));
+    EXPECT_FP_EQ(T(1235.0), func(T(1234.96), FP_INT_TONEARESTFROMZERO, 11U));
+  }
+
+  void testFractionsToNearestFromZeroOutsideRange(UfromfpFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(-0.5), FP_INT_TONEARESTFROMZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-0.715), FP_INT_TONEARESTFROMZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.3), FP_INT_TONEARESTFROMZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), FP_INT_TONEARESTFROMZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), FP_INT_TONEARESTFROMZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_TONEARESTFROMZERO, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_TONEARESTFROMZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_TONEARESTFROMZERO, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_TONEARESTFROMZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_TONEARESTFROMZERO, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_TONEARESTFROMZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_TONEARESTFROMZERO, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_TONEARESTFROMZERO, 32U));
+  }
+
+  void testFractionsToNearestWithinRange(UfromfpFunc func) {
+    EXPECT_FP_EQ(T(0.0), func(T(0.5), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.5), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(T(0.0), func(T(0.115), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.115), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(0.715), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(1.3), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(T(2.0), func(T(1.5), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(T(2.0), func(T(1.75), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(T(10.0), func(T(10.32), FP_INT_TONEAREST, 4U));
+    EXPECT_FP_EQ(T(11.0), func(T(10.65), FP_INT_TONEAREST, 4U));
+    EXPECT_FP_EQ(T(1234.0), func(T(1234.38), FP_INT_TONEAREST, 11U));
+    EXPECT_FP_EQ(T(1235.0), func(T(1234.96), FP_INT_TONEAREST, 11U));
+
+    EXPECT_FP_EQ(T(2.0), func(T(2.3), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(T(2.0), func(T(2.5), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(T(3.0), func(T(2.75), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(T(5.0), func(T(5.3), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(T(6.0), func(T(5.5), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(T(6.0), func(T(5.75), FP_INT_TONEAREST, 3U));
+  }
+
+  void testFractionsToNearestOutsideRange(UfromfpFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(-0.715), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.3), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_TONEAREST, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_TONEAREST, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_TONEAREST, 32U));
+
+    EXPECT_FP_EQ(aNaN, func(T(2.3), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.3), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(2.5), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.5), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(2.75), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.75), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(5.3), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.3), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(5.5), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.5), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(5.75), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.75), FP_INT_TONEAREST, 32U));
+  }
+
+  void testFractionsToNearestFallbackWithinRange(UfromfpFunc func) {
+    EXPECT_FP_EQ(T(0.0), func(T(0.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(T(0.0), func(T(0.115), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(T(-0.0), func(T(-0.115), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(0.715), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(T(1.0), func(T(1.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(T(2.0), func(T(1.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(T(2.0), func(T(1.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(T(10.0), func(T(10.32), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U));
+    EXPECT_FP_EQ(T(11.0), func(T(10.65), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U));
+    EXPECT_FP_EQ(T(1234.0),
+                 func(T(1234.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 11U));
+    EXPECT_FP_EQ(T(1235.0),
+                 func(T(1234.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 11U));
+
+    EXPECT_FP_EQ(T(2.0), func(T(2.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(T(2.0), func(T(2.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(T(3.0), func(T(2.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(T(5.0), func(T(5.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(T(6.0), func(T(5.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(T(6.0), func(T(5.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+  }
+
+  void testFractionsToNearestFallbackOutsideRange(UfromfpFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(-0.715), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+
+    EXPECT_FP_EQ(aNaN, func(T(2.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(2.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(2.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(5.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(5.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(5.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+  }
+};
+
+#define LIST_UFROMFP_TESTS(T, func)                                            \
+  using LlvmLibcUfromfpTest = UfromfpTestTemplate<T>;                          \
+  TEST_F(LlvmLibcUfromfpTest, SpecialNumbersNonzeroWidth) {                    \
+    testSpecialNumbersNonzeroWidth(&func);                                     \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpTest, SpecialNumbersZeroWidth) {                       \
+    testSpecialNumbersZeroWidth(&func);                                        \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpTest, RoundedNumbersWithinRange) {                     \
+    testRoundedNumbersWithinRange(&func);                                      \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpTest, RoundedNumbersOutsideRange) {                    \
+    testRoundedNumbersOutsideRange(&func);                                     \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpTest, FractionsUpwardWithinRange) {                    \
+    testFractionsUpwardWithinRange(&func);                                     \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpTest, FractionsUpwardOutsideRange) {                   \
+    testFractionsUpwardOutsideRange(&func);                                    \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpTest, FractionsDownwardWithinRange) {                  \
+    testFractionsDownwardWithinRange(&func);                                   \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpTest, FractionsDownwardOutsideRange) {                 \
+    testFractionsDownwardOutsideRange(&func);                                  \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpTest, FractionsTowardZeroWithinRange) {                \
+    testFractionsTowardZeroWithinRange(&func);                                 \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpTest, FractionsTowardZeroOutsideRange) {               \
+    testFractionsTowardZeroOutsideRange(&func);                                \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpTest, FractionsToNearestFromZeroWithinRange) {         \
+    testFractionsToNearestFromZeroWithinRange(&func);                          \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpTest, FractionsToNearestFromZeroOutsideRange) {        \
+    testFractionsToNearestFromZeroOutsideRange(&func);                         \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpTest, FractionsToNearestWithinRange) {                 \
+    testFractionsToNearestWithinRange(&func);                                  \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpTest, FractionsToNearestOutsideRange) {                \
+    testFractionsToNearestOutsideRange(&func);                                 \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpTest, FractionsToNearestFallbackWithinRange) {         \
+    testFractionsToNearestFallbackWithinRange(&func);                          \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpTest, FractionsToNearestFallbackOutsideRange) {        \
+    testFractionsToNearestFallbackOutsideRange(&func);                         \
+  }
+
+#endif // LIBC_TEST_SRC_MATH_SMOKE_UFROMFPTEST_H
diff --git a/libc/test/src/math/smoke/UfromfpxTest.h b/libc/test/src/math/smoke/UfromfpxTest.h
new file mode 100644
index 000000000000..47bf32c80f17
--- /dev/null
+++ b/libc/test/src/math/smoke/UfromfpxTest.h
@@ -0,0 +1,437 @@
+//===-- Utility class to test different flavors of ufromfpx -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LIBC_TEST_SRC_MATH_SMOKE_UFROMFPXTEST_H
+#define LIBC_TEST_SRC_MATH_SMOKE_UFROMFPXTEST_H
+
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+template <typename T>
+class UfromfpxTestTemplate : public LIBC_NAMESPACE::testing::Test {
+
+  DECLARE_SPECIAL_CONSTANTS(T)
+
+public:
+  typedef T (*UfromfpxFunc)(T, int, unsigned int);
+
+  void testSpecialNumbersNonzeroWidth(UfromfpxFunc func) {
+    for (int rnd : MATH_ROUNDING_DIRECTIONS_INCLUDING_UNKNOWN) {
+      EXPECT_FP_EQ(zero, func(zero, rnd, 32U));
+      EXPECT_FP_EQ(neg_zero, func(neg_zero, rnd, 32U));
+
+      EXPECT_FP_EQ(aNaN, func(inf, rnd, 32U));
+      EXPECT_FP_EQ(aNaN, func(neg_inf, rnd, 32U));
+
+      EXPECT_FP_EQ(aNaN, func(aNaN, rnd, 32U));
+    }
+  }
+
+  void testSpecialNumbersZeroWidth(UfromfpxFunc func) {
+    for (int rnd : MATH_ROUNDING_DIRECTIONS_INCLUDING_UNKNOWN) {
+      EXPECT_FP_EQ(aNaN, func(zero, rnd, 0U));
+      EXPECT_FP_EQ(aNaN, func(neg_zero, rnd, 0U));
+
+      EXPECT_FP_EQ(aNaN, func(inf, rnd, 0U));
+      EXPECT_FP_EQ(aNaN, func(neg_inf, rnd, 0U));
+
+      EXPECT_FP_EQ(aNaN, func(aNaN, rnd, 0U));
+    }
+  }
+
+  void testRoundedNumbersWithinRange(UfromfpxFunc func) {
+    for (int rnd : MATH_ROUNDING_DIRECTIONS_INCLUDING_UNKNOWN) {
+      EXPECT_FP_EQ(T(1.0), func(T(1.0), rnd, 1U));
+      EXPECT_FP_EQ(T(10.0), func(T(10.0), rnd, 4U));
+      EXPECT_FP_EQ(T(1234.0), func(T(1234.0), rnd, 11U));
+    }
+  }
+
+  void testRoundedNumbersOutsideRange(UfromfpxFunc func) {
+    for (int rnd : MATH_ROUNDING_DIRECTIONS_INCLUDING_UNKNOWN) {
+      EXPECT_FP_EQ(aNaN, func(T(-1.0), rnd, 32U));
+      EXPECT_FP_EQ(aNaN, func(T(10.0), rnd, 3U));
+      EXPECT_FP_EQ(aNaN, func(T(-10.0), rnd, 32U));
+      EXPECT_FP_EQ(aNaN, func(T(1234.0), rnd, 10U));
+      EXPECT_FP_EQ(aNaN, func(T(-1234.0), rnd, 32U));
+    }
+  }
+
+  void testFractionsUpwardWithinRange(UfromfpxFunc func) {
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(0.5), FP_INT_UPWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-0.0), func(T(-0.5), FP_INT_UPWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(0.115), FP_INT_UPWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-0.0), func(T(-0.115), FP_INT_UPWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(0.715), FP_INT_UPWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-0.0), func(T(-0.715), FP_INT_UPWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(2.0), func(T(1.3), FP_INT_UPWARD, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(2.0), func(T(1.5), FP_INT_UPWARD, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(2.0), func(T(1.75), FP_INT_UPWARD, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(11.0), func(T(10.32), FP_INT_UPWARD, 4U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(11.0), func(T(10.65), FP_INT_UPWARD, 4U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1235.0), func(T(1234.38), FP_INT_UPWARD, 11U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1235.0), func(T(1234.96), FP_INT_UPWARD, 11U),
+                                FE_INEXACT);
+  }
+
+  void testFractionsUpwardOutsideRange(UfromfpxFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(1.3), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.3), FP_INT_UPWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), FP_INT_UPWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), FP_INT_UPWARD, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), FP_INT_UPWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_UPWARD, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_UPWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_UPWARD, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_UPWARD, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_UPWARD, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_UPWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_UPWARD, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_UPWARD, 32U));
+  }
+
+  void testFractionsDownwardWithinRange(UfromfpxFunc func) {
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(0.0), func(T(0.5), FP_INT_DOWNWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(0.0), func(T(0.115), FP_INT_DOWNWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(0.0), func(T(0.715), FP_INT_DOWNWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(1.3), FP_INT_DOWNWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(1.5), FP_INT_DOWNWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(1.75), FP_INT_DOWNWARD, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(10.0), func(T(10.32), FP_INT_DOWNWARD, 4U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(10.0), func(T(10.65), FP_INT_DOWNWARD, 4U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1234.0), func(T(1234.38), FP_INT_DOWNWARD, 11U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1234.0), func(T(1234.96), FP_INT_DOWNWARD, 11U), FE_INEXACT);
+  }
+
+  void testFractionsDownwardOutsideRange(UfromfpxFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(-0.5), FP_INT_DOWNWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-0.115), FP_INT_DOWNWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-0.715), FP_INT_DOWNWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.3), FP_INT_DOWNWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), FP_INT_DOWNWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), FP_INT_DOWNWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_DOWNWARD, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_DOWNWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_DOWNWARD, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_DOWNWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_DOWNWARD, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_DOWNWARD, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_DOWNWARD, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_DOWNWARD, 32U));
+  }
+
+  void testFractionsTowardZeroWithinRange(UfromfpxFunc func) {
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(0.0), func(T(0.5), FP_INT_TOWARDZERO, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-0.0), func(T(-0.5), FP_INT_TOWARDZERO, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(0.0), func(T(0.115), FP_INT_TOWARDZERO, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-0.0), func(T(-0.115), FP_INT_TOWARDZERO, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(0.0), func(T(0.715), FP_INT_TOWARDZERO, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-0.0), func(T(-0.715), FP_INT_TOWARDZERO, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(1.3), FP_INT_TOWARDZERO, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(1.5), FP_INT_TOWARDZERO, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(1.75), FP_INT_TOWARDZERO, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(10.0), func(T(10.32), FP_INT_TOWARDZERO, 4U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(10.0), func(T(10.65), FP_INT_TOWARDZERO, 4U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1234.0), func(T(1234.38), FP_INT_TOWARDZERO, 11U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1234.0), func(T(1234.96), FP_INT_TOWARDZERO, 11U), FE_INEXACT);
+  }
+
+  void testFractionsTowardZeroOutsideRange(UfromfpxFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(-1.3), FP_INT_TOWARDZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), FP_INT_TOWARDZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), FP_INT_TOWARDZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_TOWARDZERO, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_TOWARDZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_TOWARDZERO, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_TOWARDZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_TOWARDZERO, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_TOWARDZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_TOWARDZERO, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_TOWARDZERO, 32U));
+  }
+
+  void testFractionsToNearestFromZeroWithinRange(UfromfpxFunc func) {
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1.0), func(T(0.5), FP_INT_TONEARESTFROMZERO, 1U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(0.0), func(T(0.115), FP_INT_TONEARESTFROMZERO, 1U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-0.0), func(T(-0.115), FP_INT_TONEARESTFROMZERO, 1U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1.0), func(T(0.715), FP_INT_TONEARESTFROMZERO, 1U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1.0), func(T(1.3), FP_INT_TONEARESTFROMZERO, 1U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(2.0), func(T(1.5), FP_INT_TONEARESTFROMZERO, 2U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(2.0), func(T(1.75), FP_INT_TONEARESTFROMZERO, 2U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(10.0), func(T(10.32), FP_INT_TONEARESTFROMZERO, 4U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(11.0), func(T(10.65), FP_INT_TONEARESTFROMZERO, 4U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1234.0), func(T(1234.38), FP_INT_TONEARESTFROMZERO, 11U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1235.0), func(T(1234.96), FP_INT_TONEARESTFROMZERO, 11U), FE_INEXACT);
+  }
+
+  void testFractionsToNearestFromZeroOutsideRange(UfromfpxFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(-0.5), FP_INT_TONEARESTFROMZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-0.715), FP_INT_TONEARESTFROMZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.3), FP_INT_TONEARESTFROMZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), FP_INT_TONEARESTFROMZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), FP_INT_TONEARESTFROMZERO, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), FP_INT_TONEARESTFROMZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_TONEARESTFROMZERO, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_TONEARESTFROMZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_TONEARESTFROMZERO, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_TONEARESTFROMZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_TONEARESTFROMZERO, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_TONEARESTFROMZERO, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_TONEARESTFROMZERO, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_TONEARESTFROMZERO, 32U));
+  }
+
+  void testFractionsToNearestWithinRange(UfromfpxFunc func) {
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(0.0), func(T(0.5), FP_INT_TONEAREST, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-0.0), func(T(-0.5), FP_INT_TONEAREST, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(0.0), func(T(0.115), FP_INT_TONEAREST, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-0.0), func(T(-0.115), FP_INT_TONEAREST, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(0.715), FP_INT_TONEAREST, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.0), func(T(1.3), FP_INT_TONEAREST, 1U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(2.0), func(T(1.5), FP_INT_TONEAREST, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(2.0), func(T(1.75), FP_INT_TONEAREST, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(10.0), func(T(10.32), FP_INT_TONEAREST, 4U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(11.0), func(T(10.65), FP_INT_TONEAREST, 4U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1234.0), func(T(1234.38), FP_INT_TONEAREST, 11U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1235.0), func(T(1234.96), FP_INT_TONEAREST, 11U), FE_INEXACT);
+
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(2.0), func(T(2.3), FP_INT_TONEAREST, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(2.0), func(T(2.5), FP_INT_TONEAREST, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(3.0), func(T(2.75), FP_INT_TONEAREST, 2U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(5.0), func(T(5.3), FP_INT_TONEAREST, 3U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(6.0), func(T(5.5), FP_INT_TONEAREST, 3U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(6.0), func(T(5.75), FP_INT_TONEAREST, 3U),
+                                FE_INEXACT);
+  }
+
+  void testFractionsToNearestOutsideRange(UfromfpxFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(-0.715), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.3), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), FP_INT_TONEAREST, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), FP_INT_TONEAREST, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), FP_INT_TONEAREST, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), FP_INT_TONEAREST, 32U));
+
+    EXPECT_FP_EQ(aNaN, func(T(2.3), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.3), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(2.5), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.5), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(2.75), FP_INT_TONEAREST, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.75), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(5.3), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.3), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(5.5), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.5), FP_INT_TONEAREST, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(5.75), FP_INT_TONEAREST, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.75), FP_INT_TONEAREST, 32U));
+  }
+
+  void testFractionsToNearestFallbackWithinRange(UfromfpxFunc func) {
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(0.0), func(T(0.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-0.0), func(T(-0.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(0.0), func(T(0.115), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(-0.0), func(T(-0.115), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1.0), func(T(0.715), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1.0), func(T(1.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(2.0), func(T(1.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(2.0), func(T(1.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(10.0), func(T(10.32), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(11.0), func(T(10.65), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1234.0), func(T(1234.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 11U),
+        FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(1235.0), func(T(1234.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 11U),
+        FE_INEXACT);
+
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(2.0), func(T(2.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(2.0), func(T(2.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(3.0), func(T(2.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(5.0), func(T(5.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(6.0), func(T(5.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(
+        T(6.0), func(T(5.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U), FE_INEXACT);
+  }
+
+  void testFractionsToNearestFallbackOutsideRange(UfromfpxFunc func) {
+    EXPECT_FP_EQ(aNaN, func(T(-0.715), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-1.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.32), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.32), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(10.65), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
+    EXPECT_FP_EQ(aNaN, func(T(-10.65), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(1234.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 10U));
+    EXPECT_FP_EQ(aNaN, func(T(-1234.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+
+    EXPECT_FP_EQ(aNaN, func(T(2.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(2.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(2.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 1U));
+    EXPECT_FP_EQ(aNaN, func(T(-2.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(5.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(5.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+    EXPECT_FP_EQ(aNaN, func(T(5.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
+    EXPECT_FP_EQ(aNaN, func(T(-5.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 32U));
+  }
+};
+
+#define LIST_UFROMFPX_TESTS(T, func)                                           \
+  using LlvmLibcUfromfpxTest = UfromfpxTestTemplate<T>;                        \
+  TEST_F(LlvmLibcUfromfpxTest, SpecialNumbersNonzeroWidth) {                   \
+    testSpecialNumbersNonzeroWidth(&func);                                     \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpxTest, SpecialNumbersZeroWidth) {                      \
+    testSpecialNumbersZeroWidth(&func);                                        \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpxTest, RoundedNumbersWithinRange) {                    \
+    testRoundedNumbersWithinRange(&func);                                      \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpxTest, RoundedNumbersOutsideRange) {                   \
+    testRoundedNumbersOutsideRange(&func);                                     \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpxTest, FractionsUpwardWithinRange) {                   \
+    testFractionsUpwardWithinRange(&func);                                     \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpxTest, FractionsUpwardOutsideRange) {                  \
+    testFractionsUpwardOutsideRange(&func);                                    \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpxTest, FractionsDownwardWithinRange) {                 \
+    testFractionsDownwardWithinRange(&func);                                   \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpxTest, FractionsDownwardOutsideRange) {                \
+    testFractionsDownwardOutsideRange(&func);                                  \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpxTest, FractionsTowardZeroWithinRange) {               \
+    testFractionsTowardZeroWithinRange(&func);                                 \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpxTest, FractionsTowardZeroOutsideRange) {              \
+    testFractionsTowardZeroOutsideRange(&func);                                \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpxTest, FractionsToNearestFromZeroWithinRange) {        \
+    testFractionsToNearestFromZeroWithinRange(&func);                          \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpxTest, FractionsToNearestFromZeroOutsideRange) {       \
+    testFractionsToNearestFromZeroOutsideRange(&func);                         \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpxTest, FractionsToNearestWithinRange) {                \
+    testFractionsToNearestWithinRange(&func);                                  \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpxTest, FractionsToNearestOutsideRange) {               \
+    testFractionsToNearestOutsideRange(&func);                                 \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpxTest, FractionsToNearestFallbackWithinRange) {        \
+    testFractionsToNearestFallbackWithinRange(&func);                          \
+  }                                                                            \
+  TEST_F(LlvmLibcUfromfpxTest, FractionsToNearestFallbackOutsideRange) {       \
+    testFractionsToNearestFallbackOutsideRange(&func);                         \
+  }
+
+#endif // LIBC_TEST_SRC_MATH_SMOKE_UFROMFPXTEST_H
diff --git a/libc/test/src/math/smoke/fromfp_test.cpp b/libc/test/src/math/smoke/fromfp_test.cpp
new file mode 100644
index 000000000000..147a9df9afcf
--- /dev/null
+++ b/libc/test/src/math/smoke/fromfp_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fromfp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FromfpTest.h"
+
+#include "src/math/fromfp.h"
+
+LIST_FROMFP_TESTS(double, LIBC_NAMESPACE::fromfp)
diff --git a/libc/test/src/math/smoke/fromfpf128_test.cpp b/libc/test/src/math/smoke/fromfpf128_test.cpp
new file mode 100644
index 000000000000..288aadb359bf
--- /dev/null
+++ b/libc/test/src/math/smoke/fromfpf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fromfpf128 ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FromfpTest.h"
+
+#include "src/math/fromfpf128.h"
+
+LIST_FROMFP_TESTS(float128, LIBC_NAMESPACE::fromfpf128)
diff --git a/libc/test/src/math/smoke/fromfpf_test.cpp b/libc/test/src/math/smoke/fromfpf_test.cpp
new file mode 100644
index 000000000000..63f3f624716e
--- /dev/null
+++ b/libc/test/src/math/smoke/fromfpf_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fromfpf ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FromfpTest.h"
+
+#include "src/math/fromfpf.h"
+
+LIST_FROMFP_TESTS(float, LIBC_NAMESPACE::fromfpf)
diff --git a/libc/test/src/math/smoke/fromfpl_test.cpp b/libc/test/src/math/smoke/fromfpl_test.cpp
new file mode 100644
index 000000000000..c0072768870b
--- /dev/null
+++ b/libc/test/src/math/smoke/fromfpl_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fromfpl ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FromfpTest.h"
+
+#include "src/math/fromfpl.h"
+
+LIST_FROMFP_TESTS(long double, LIBC_NAMESPACE::fromfpl)
diff --git a/libc/test/src/math/smoke/fromfpx_test.cpp b/libc/test/src/math/smoke/fromfpx_test.cpp
new file mode 100644
index 000000000000..10b1eee726e1
--- /dev/null
+++ b/libc/test/src/math/smoke/fromfpx_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fromfpx ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FromfpxTest.h"
+
+#include "src/math/fromfpx.h"
+
+LIST_FROMFPX_TESTS(double, LIBC_NAMESPACE::fromfpx)
diff --git a/libc/test/src/math/smoke/fromfpxf128_test.cpp b/libc/test/src/math/smoke/fromfpxf128_test.cpp
new file mode 100644
index 000000000000..2839bed30cb7
--- /dev/null
+++ b/libc/test/src/math/smoke/fromfpxf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fromfpxf128 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FromfpxTest.h"
+
+#include "src/math/fromfpxf128.h"
+
+LIST_FROMFPX_TESTS(float128, LIBC_NAMESPACE::fromfpxf128)
diff --git a/libc/test/src/math/smoke/fromfpxf_test.cpp b/libc/test/src/math/smoke/fromfpxf_test.cpp
new file mode 100644
index 000000000000..42e47aba40ae
--- /dev/null
+++ b/libc/test/src/math/smoke/fromfpxf_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fromfpxf --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FromfpxTest.h"
+
+#include "src/math/fromfpxf.h"
+
+LIST_FROMFPX_TESTS(float, LIBC_NAMESPACE::fromfpxf)
diff --git a/libc/test/src/math/smoke/fromfpxl_test.cpp b/libc/test/src/math/smoke/fromfpxl_test.cpp
new file mode 100644
index 000000000000..cbe8d750ff2a
--- /dev/null
+++ b/libc/test/src/math/smoke/fromfpxl_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fromfpxl --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FromfpxTest.h"
+
+#include "src/math/fromfpxl.h"
+
+LIST_FROMFPX_TESTS(long double, LIBC_NAMESPACE::fromfpxl)
diff --git a/libc/test/src/math/smoke/ufromfp_test.cpp b/libc/test/src/math/smoke/ufromfp_test.cpp
new file mode 100644
index 000000000000..ff4762112400
--- /dev/null
+++ b/libc/test/src/math/smoke/ufromfp_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for ufromfp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "UfromfpTest.h"
+
+#include "src/math/ufromfp.h"
+
+LIST_UFROMFP_TESTS(double, LIBC_NAMESPACE::ufromfp)
diff --git a/libc/test/src/math/smoke/ufromfpf128_test.cpp b/libc/test/src/math/smoke/ufromfpf128_test.cpp
new file mode 100644
index 000000000000..9ba3034f6e61
--- /dev/null
+++ b/libc/test/src/math/smoke/ufromfpf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for ufromfpf128 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "UfromfpTest.h"
+
+#include "src/math/ufromfpf128.h"
+
+LIST_UFROMFP_TESTS(float128, LIBC_NAMESPACE::ufromfpf128)
diff --git a/libc/test/src/math/smoke/ufromfpf_test.cpp b/libc/test/src/math/smoke/ufromfpf_test.cpp
new file mode 100644
index 000000000000..2913b31e20bd
--- /dev/null
+++ b/libc/test/src/math/smoke/ufromfpf_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for ufromfpf --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "UfromfpTest.h"
+
+#include "src/math/ufromfpf.h"
+
+LIST_UFROMFP_TESTS(float, LIBC_NAMESPACE::ufromfpf)
diff --git a/libc/test/src/math/smoke/ufromfpl_test.cpp b/libc/test/src/math/smoke/ufromfpl_test.cpp
new file mode 100644
index 000000000000..8976056756e4
--- /dev/null
+++ b/libc/test/src/math/smoke/ufromfpl_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for ufromfpl --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "UfromfpTest.h"
+
+#include "src/math/ufromfpl.h"
+
+LIST_UFROMFP_TESTS(long double, LIBC_NAMESPACE::ufromfpl)
diff --git a/libc/test/src/math/smoke/ufromfpx_test.cpp b/libc/test/src/math/smoke/ufromfpx_test.cpp
new file mode 100644
index 000000000000..3bb45e428f28
--- /dev/null
+++ b/libc/test/src/math/smoke/ufromfpx_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for ufromfpx --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "UfromfpxTest.h"
+
+#include "src/math/ufromfpx.h"
+
+LIST_UFROMFPX_TESTS(double, LIBC_NAMESPACE::ufromfpx)
diff --git a/libc/test/src/math/smoke/ufromfpxf128_test.cpp b/libc/test/src/math/smoke/ufromfpxf128_test.cpp
new file mode 100644
index 000000000000..6defaf7fee3d
--- /dev/null
+++ b/libc/test/src/math/smoke/ufromfpxf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for ufromfpxf128 ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "UfromfpxTest.h"
+
+#include "src/math/ufromfpxf128.h"
+
+LIST_UFROMFPX_TESTS(float128, LIBC_NAMESPACE::ufromfpxf128)
diff --git a/libc/test/src/math/smoke/ufromfpxf_test.cpp b/libc/test/src/math/smoke/ufromfpxf_test.cpp
new file mode 100644
index 000000000000..862fd4c2cdac
--- /dev/null
+++ b/libc/test/src/math/smoke/ufromfpxf_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for ufromfpxf -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "UfromfpxTest.h"
+
+#include "src/math/ufromfpxf.h"
+
+LIST_UFROMFPX_TESTS(float, LIBC_NAMESPACE::ufromfpxf)
diff --git a/libc/test/src/math/smoke/ufromfpxl_test.cpp b/libc/test/src/math/smoke/ufromfpxl_test.cpp
new file mode 100644
index 000000000000..b6b93b38e6be
--- /dev/null
+++ b/libc/test/src/math/smoke/ufromfpxl_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for ufromfpxl -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "UfromfpxTest.h"
+
+#include "src/math/ufromfpxl.h"
+
+LIST_UFROMFPX_TESTS(long double, LIBC_NAMESPACE::ufromfpxl)
diff --git a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
index 2a0c071f2286..d2087a3d528f 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
@@ -76,6 +76,7 @@ libc_support_library(
     deps = [
         ":LibcUnitTest",
         ":string_utils",
+        "//libc:__support_cpp_array",
         "//libc:__support_cpp_bit",
         "//libc:__support_cpp_bitset",
         "//libc:__support_cpp_span",
-- 
GitLab


From bea17ff652bc49b2de8d6be04f77d28170a78be9 Mon Sep 17 00:00:00 2001
From: Tom Honermann <tom.honermann@intel.com>
Date: Mon, 25 Mar 2024 10:39:05 -0400
Subject: [PATCH 128/404] [clang] Correct Microsoft mangling of lifetime
 extended temporary objects. (#85529)

Lifetime extended temporary objects that are bound to references with
static storage duration may have external linkage and therefore require
mangled symbol names. Clang uses an extension of the Microsoft ABI to
give these symbols an implicit name of '$RT' followed by a discriminator
and then mangles them similarly to the variable they are bound to.
Clang's mangling scheme differs from the one used by MSVC.

Previously, the `$RT<discriminator>` portion of the name was not
registered as a back reference candidate and this resulted in incorrect
back references for enclosing class and/or namespace scopes that might
be referenced in the type of the object.

This is an ABI change and has the potential to cause backward
compatibility issues with previous Clang releases. Since MSVC uses a
different mangling scheme, this change does not affect compatibility
with MSVC.

This fixes one of the name mangling concerns reported in #85423.
---
 clang/docs/ReleaseNotes.rst                         |  6 ++++++
 clang/lib/AST/MicrosoftMangle.cpp                   |  3 ++-
 clang/test/CodeGenCXX/mangle-ms-back-references.cpp | 13 +++++++++++++
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 8054d90fc70f..7fbe2fec6ca0 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -57,6 +57,12 @@ ABI Changes in This Version
   inline member function that contains a static local variable with a dynamic
   initializer is declared with ``__declspec(dllimport)``. (#GH83616).
 
+- Fixed Microsoft name mangling of lifetime extended temporary objects. This
+  change corrects missing back reference registrations that could result in
+  incorrect back reference indexes and suprising demangled name results. Since
+  MSVC uses a different mangling for these objects, compatibility is not affected.
+  (#GH85423).
+
 AST Dumping Potentially Breaking Changes
 ----------------------------------------
 
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index aa26bb7ed46f..cf9c2093a8f6 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -3911,7 +3911,8 @@ void MicrosoftMangleContextImpl::mangleReferenceTemporary(
   msvc_hashing_ostream MHO(Out);
   MicrosoftCXXNameMangler Mangler(*this, MHO);
 
-  Mangler.getStream() << "?$RT" << ManglingNumber << '@';
+  Mangler.getStream() << "?";
+  Mangler.mangleSourceName("$RT" + llvm::utostr(ManglingNumber));
   Mangler.mangle(VD, "");
 }
 
diff --git a/clang/test/CodeGenCXX/mangle-ms-back-references.cpp b/clang/test/CodeGenCXX/mangle-ms-back-references.cpp
index b27a9c5acacb..8707bff95340 100644
--- a/clang/test/CodeGenCXX/mangle-ms-back-references.cpp
+++ b/clang/test/CodeGenCXX/mangle-ms-back-references.cpp
@@ -1,5 +1,18 @@
 // RUN: %clang_cc1 -fms-extensions -fblocks -emit-llvm %s -o - -triple=i386-pc-win32 | FileCheck %s
 
+namespace NS {
+// The name "RT1" for the name of the class below has been specifically
+// chosen to ensure that back reference lookup does not match against the
+// implicitly generated "$RT1" name of the reference temporary symbol.
+struct RT1 {
+  static const RT1& singleton;
+  int i;
+};
+const RT1& RT1::singleton = RT1{1};
+}
+// CHECK: "?$RT1@singleton@RT1@NS@@2ABU23@B"
+// CHECK: "?singleton@RT1@NS@@2ABU12@B"
+
 void f1(const char* a, const char* b) {}
 // CHECK: "?f1@@YAXPBD0@Z"
 
-- 
GitLab


From 06cfbe3cfd44cd2ca9eb970b8c0e5f4911468440 Mon Sep 17 00:00:00 2001
From: David Stuttard <david.stuttard@amd.com>
Date: Mon, 25 Mar 2024 14:44:22 +0000
Subject: [PATCH 129/404] [AMDPU] Add support for idxen and bothen buffer
 load/store merging in SILoadStoreOptimizer (#86285)

Added more buffer instruction merging support
---
 .../Target/AMDGPU/SILoadStoreOptimizer.cpp    |   16 +
 .../CodeGen/AMDGPU/merge-buffer-gfx12.mir     | 1154 +++++++++++++++++
 llvm/test/CodeGen/AMDGPU/merge-buffer.mir     | 1130 ++++++++++++++++
 llvm/test/CodeGen/AMDGPU/merge-tbuffer.mir    |   28 +
 4 files changed, 2328 insertions(+)
 create mode 100644 llvm/test/CodeGen/AMDGPU/merge-buffer-gfx12.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/merge-buffer.mir

diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index 4ddee2f6d5be..e2e70ba9733b 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -399,19 +399,35 @@ static InstClassEnum getInstClass(unsigned Opc, const SIInstrInfo &TII) {
       switch (AMDGPU::getMUBUFBaseOpcode(Opc)) {
       default:
         return UNKNOWN;
+      case AMDGPU::BUFFER_LOAD_DWORD_BOTHEN:
+      case AMDGPU::BUFFER_LOAD_DWORD_BOTHEN_exact:
+      case AMDGPU::BUFFER_LOAD_DWORD_IDXEN:
+      case AMDGPU::BUFFER_LOAD_DWORD_IDXEN_exact:
       case AMDGPU::BUFFER_LOAD_DWORD_OFFEN:
       case AMDGPU::BUFFER_LOAD_DWORD_OFFEN_exact:
       case AMDGPU::BUFFER_LOAD_DWORD_OFFSET:
       case AMDGPU::BUFFER_LOAD_DWORD_OFFSET_exact:
+      case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:
+      case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_BOTHEN_exact:
+      case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_IDXEN:
+      case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_IDXEN_exact:
       case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFEN:
       case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFEN_exact:
       case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFSET:
       case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFSET_exact:
         return BUFFER_LOAD;
+      case AMDGPU::BUFFER_STORE_DWORD_BOTHEN:
+      case AMDGPU::BUFFER_STORE_DWORD_BOTHEN_exact:
+      case AMDGPU::BUFFER_STORE_DWORD_IDXEN:
+      case AMDGPU::BUFFER_STORE_DWORD_IDXEN_exact:
       case AMDGPU::BUFFER_STORE_DWORD_OFFEN:
       case AMDGPU::BUFFER_STORE_DWORD_OFFEN_exact:
       case AMDGPU::BUFFER_STORE_DWORD_OFFSET:
       case AMDGPU::BUFFER_STORE_DWORD_OFFSET_exact:
+      case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_BOTHEN:
+      case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_BOTHEN_exact:
+      case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_IDXEN:
+      case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_IDXEN_exact:
       case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_OFFEN:
       case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact:
       case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_OFFSET:
diff --git a/llvm/test/CodeGen/AMDGPU/merge-buffer-gfx12.mir b/llvm/test/CodeGen/AMDGPU/merge-buffer-gfx12.mir
new file mode 100644
index 000000000000..d7f5d1a23789
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/merge-buffer-gfx12.mir
@@ -0,0 +1,1154 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass si-load-store-opt -o - %s | FileCheck -check-prefixes=GFX12 %s
+
+---
+name: buffer_load_dword_dwordx3
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_dwordx3
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFSET]].sub0
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vreg_96 = COPY killed [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFSET]].sub1_sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_96 = BUFFER_LOAD_DWORDX3_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx3_dword
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dwordx3_dword
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vreg_96 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFSET]].sub0_sub1_sub2
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFSET]].sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vreg_96 = BUFFER_LOAD_DWORDX3_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 16, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dwordx2
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dwordx2_dwordx2
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFSET]].sub0_sub1
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFSET]].sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 12, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dwordx2
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_dwordx2
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFSET:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFSET]].sub0
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFSET]].sub1_sub2
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dword
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dwordx2_dword
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFSET:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFSET]].sub0_sub1
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFSET]].sub2
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+
+name: buffer_load_dword_dword
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_dword
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET]].sub0
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_32
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_32
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET]].sub0
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET]].sub1
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 16, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vreg_96 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFSET]].sub0_sub1_sub2
+    ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFSET]].sub3
+    ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[COPY6]].sub0_sub1
+    ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY killed [[COPY6]].sub2
+    ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY8]].sub0
+    ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY killed [[COPY8]].sub1
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFSET:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 36, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY12:%[0-9]+]]:vreg_64 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFSET]].sub0_sub1
+    ; GFX12-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFSET]].sub2
+    ; GFX12-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY12]].sub0
+    ; GFX12-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY killed [[COPY12]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %9:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 16, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %10:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 20, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %11:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 24, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %12:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 28, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %13:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 36, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %14:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 40, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %15:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 44, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+#
+# buffer_store_dword
+#
+
+name: buffer_store_dword_xyz
+body:             |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX12-LABEL: name: buffer_store_dword_xyz
+    ; GFX12: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+    ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2
+    ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[REG_SEQUENCE1]], %subreg.sub1_sub2_sub3
+    ; GFX12-NEXT: BUFFER_STORE_DWORDX4_VBUFFER_OFFSET_exact killed [[REG_SEQUENCE2]], [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 4)
+    %7:vgpr_32 = COPY $vgpr3
+    %6:vgpr_32 = COPY $vgpr2
+    %5:vgpr_32 = COPY $vgpr1
+    %4:vgpr_32 = COPY $vgpr0
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %13:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %14:vreg_96 = REG_SEQUENCE %4:vgpr_32, %subreg.sub0, %5:vgpr_32, %subreg.sub1, %6:vgpr_32, %subreg.sub2
+    BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact %7:vgpr_32, %13:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORDX3_VBUFFER_OFFSET_exact %14:vreg_96, %13:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable store (s96), align 1, addrspace 4)
+...
+---
+
+name: buffer_store_dwordx3_dword
+body:             |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX12-LABEL: name: buffer_store_dwordx3_dword
+    ; GFX12: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+    ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2
+    ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE1]], %subreg.sub0_sub1_sub2, [[COPY]], %subreg.sub3
+    ; GFX12-NEXT: BUFFER_STORE_DWORDX4_VBUFFER_OFFSET_exact killed [[REG_SEQUENCE2]], [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 4)
+    %7:vgpr_32 = COPY $vgpr3
+    %6:vgpr_32 = COPY $vgpr2
+    %5:vgpr_32 = COPY $vgpr1
+    %4:vgpr_32 = COPY $vgpr0
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %13:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %14:vreg_96 = REG_SEQUENCE %4:vgpr_32, %subreg.sub0, %5:vgpr_32, %subreg.sub1, %6:vgpr_32, %subreg.sub2
+    BUFFER_STORE_DWORDX3_VBUFFER_OFFSET_exact %14:vreg_96, %13:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable store (s96), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact %7:vgpr_32, %13:sgpr_128, $sgpr_null, 16, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_store_dwordx2_dwordx2
+body:             |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX12-LABEL: name: buffer_store_dwordx2_dwordx2
+    ; GFX12: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+    ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+    ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
+    ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE1]], %subreg.sub0_sub1, [[REG_SEQUENCE2]], %subreg.sub2_sub3
+    ; GFX12-NEXT: BUFFER_STORE_DWORDX4_VBUFFER_OFFSET_exact killed [[REG_SEQUENCE3]], [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 4)
+    %7:vgpr_32 = COPY $vgpr3
+    %6:vgpr_32 = COPY $vgpr2
+    %5:vgpr_32 = COPY $vgpr1
+    %4:vgpr_32 = COPY $vgpr0
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %13:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %14:vreg_64 = REG_SEQUENCE %4:vgpr_32, %subreg.sub0, %5:vgpr_32, %subreg.sub1
+    %15:vreg_64 = REG_SEQUENCE %6:vgpr_32, %subreg.sub0, %7:vgpr_32, %subreg.sub1
+    BUFFER_STORE_DWORDX2_VBUFFER_OFFSET_exact %14:vreg_64, %13:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 4)
+    BUFFER_STORE_DWORDX2_VBUFFER_OFFSET_exact %15:vreg_64, %13:sgpr_128, $sgpr_null, 12, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 4)
+...
+---
+
+name: buffer_store_dword_dwordx2
+body:             |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX12-LABEL: name: buffer_store_dword_dwordx2
+    ; GFX12: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+    ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+    ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY]], %subreg.sub0, %10:vreg_64, %subreg.sub1_sub2
+    ; GFX12-NEXT: BUFFER_STORE_DWORDX3_VBUFFER_OFFSET_exact killed [[REG_SEQUENCE2]], [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable store (s96), align 1, addrspace 4)
+    %7:vgpr_32 = COPY $vgpr3
+    %6:vgpr_32 = COPY $vgpr2
+    %5:vgpr_32 = COPY $vgpr1
+    %4:vgpr_32 = COPY $vgpr0
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %13:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %14:vreg_64 = REG_SEQUENCE %4:vgpr_32, %subreg.sub0, %5:vgpr_32, %subreg.sub1
+    BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact %7:vgpr_32, %13:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORDX2_VBUFFER_OFFSET_exact %15:vreg_64, %13:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 4)
+...
+---
+
+name: buffer_store_dwordx2_dword
+body:             |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX12-LABEL: name: buffer_store_dwordx2_dword
+    ; GFX12: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+    ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+    ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[REG_SEQUENCE1]], %subreg.sub0_sub1, [[COPY]], %subreg.sub2
+    ; GFX12-NEXT: BUFFER_STORE_DWORDX3_VBUFFER_OFFSET_exact killed [[REG_SEQUENCE2]], [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable store (s96), align 1, addrspace 4)
+    %7:vgpr_32 = COPY $vgpr3
+    %6:vgpr_32 = COPY $vgpr2
+    %5:vgpr_32 = COPY $vgpr1
+    %4:vgpr_32 = COPY $vgpr0
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %13:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %14:vreg_64 = REG_SEQUENCE %4:vgpr_32, %subreg.sub0, %5:vgpr_32, %subreg.sub1
+    BUFFER_STORE_DWORDX2_VBUFFER_OFFSET_exact %14:vreg_64, %13:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact %7:vgpr_32, %13:sgpr_128, $sgpr_null, 12, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_store_dword_dword
+body:             |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX12-LABEL: name: buffer_store_dword_dword
+    ; GFX12: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+    ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
+    ; GFX12-NEXT: BUFFER_STORE_DWORDX2_VBUFFER_OFFSET_exact killed [[REG_SEQUENCE1]], [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 4)
+    %7:vgpr_32 = COPY $vgpr3
+    %6:vgpr_32 = COPY $vgpr2
+    %5:vgpr_32 = COPY $vgpr1
+    %4:vgpr_32 = COPY $vgpr0
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %13:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact %6:vgpr_32, %13:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact %7:vgpr_32, %13:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_store_dword_32
+body:             |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+    ; GFX12-LABEL: name: buffer_store_dword_32
+    ; GFX12: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY12:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY12]], %subreg.sub0, [[COPY11]], %subreg.sub1, [[COPY10]], %subreg.sub2, [[COPY9]], %subreg.sub3
+    ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY7]], %subreg.sub1
+    ; GFX12-NEXT: BUFFER_STORE_DWORDX2_VBUFFER_OFFSET_exact killed [[REG_SEQUENCE1]], [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1
+    ; GFX12-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE killed [[REG_SEQUENCE2]], %subreg.sub0_sub1, [[COPY4]], %subreg.sub2
+    ; GFX12-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:vreg_128 = REG_SEQUENCE killed [[REG_SEQUENCE3]], %subreg.sub0_sub1_sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: BUFFER_STORE_DWORDX4_VBUFFER_OFFSET_exact killed [[REG_SEQUENCE4]], [[REG_SEQUENCE]], $sgpr_null, 16, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 4)
+    ; GFX12-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
+    ; GFX12-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:vreg_96 = REG_SEQUENCE killed [[REG_SEQUENCE5]], %subreg.sub0_sub1, [[COPY]], %subreg.sub2
+    ; GFX12-NEXT: BUFFER_STORE_DWORDX3_VBUFFER_OFFSET_exact killed [[REG_SEQUENCE6]], [[REG_SEQUENCE]], $sgpr_null, 36, 0, 0, implicit $exec :: (dereferenceable store (s96), align 1, addrspace 4)
+    %12:vgpr_32 = COPY $vgpr8
+    %11:vgpr_32 = COPY $vgpr7
+    %10:vgpr_32 = COPY $vgpr6
+    %9:vgpr_32 = COPY $vgpr5
+    %8:vgpr_32 = COPY $vgpr4
+    %7:vgpr_32 = COPY $vgpr3
+    %6:vgpr_32 = COPY $vgpr2
+    %5:vgpr_32 = COPY $vgpr1
+    %4:vgpr_32 = COPY $vgpr0
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %13:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact %4:vgpr_32, %13:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact %5:vgpr_32, %13:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact %6:vgpr_32, %13:sgpr_128, $sgpr_null, 16, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact %7:vgpr_32, %13:sgpr_128, $sgpr_null, 20, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact %8:vgpr_32, %13:sgpr_128, $sgpr_null, 24, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact %9:vgpr_32, %13:sgpr_128, $sgpr_null, 28, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact %10:vgpr_32, %13:sgpr_128, $sgpr_null, 36, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact %11:vgpr_32, %13:sgpr_128, $sgpr_null, 40, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact %12:vgpr_32, %13:sgpr_128, $sgpr_null, 44, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_not_merged_swizzled_0
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_not_merged_swizzled_0
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 4, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 4, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_not_merged_swizzled_1
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_not_merged_swizzled_1
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 8, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 8, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_merge_across_swizzle
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_merge_across_swizzle
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET]].sub0
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET]].sub1
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 12, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %4:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %6:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %4:sgpr_128, $sgpr_null, 12, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %4:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_not_merge_across_swizzled_store
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_not_merge_across_swizzled_store
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GFX12-NEXT: BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 6, 0, 1, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_OFFSET1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %6:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact %4:vgpr_32, %5:sgpr_128, $sgpr_null, 6, 0, 1, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_merge_across_swizzled_store
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_merge_across_swizzled_store
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET]].sub0
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFSET]].sub1
+    ; GFX12-NEXT: BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 12, 0, 1, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %6:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact %4:vgpr_32, %5:sgpr_128, $sgpr_null, 12, 0, 1, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFSET %5:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dword_idxen
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_dword_idxen
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN]].sub0
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN %4, %5:sgpr_128, $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN %4, %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dwordx2_idxen
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_dwordx2_idxen
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX3_VBUFFER_IDXEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_VBUFFER_IDXEN [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_IDXEN]].sub0
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX3_VBUFFER_IDXEN]].sub1_sub2
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN %4, %5:sgpr_128, $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN %4, %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dwordx2_idxen
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dwordx2_dwordx2_idxen
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX4_VBUFFER_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_VBUFFER_IDXEN [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_IDXEN]].sub0_sub1
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX4_VBUFFER_IDXEN]].sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN %4, %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN %4, %5:sgpr_128, $sgpr_null, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dwordx3_idxen
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_dwordx3_idxen
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX4_VBUFFER_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_VBUFFER_IDXEN [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_IDXEN]].sub0
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vreg_96 = COPY killed [[BUFFER_LOAD_DWORDX4_VBUFFER_IDXEN]].sub1_sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN %4, %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_96 = BUFFER_LOAD_DWORDX3_VBUFFER_IDXEN %4, %5:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dword_bothen
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_dword_bothen
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN]].sub0
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN %4, %5:sgpr_128, $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN %4, %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dwordx2_bothen
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_dwordx2_bothen
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN]].sub0
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN]].sub1_sub2
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN %4, %5:sgpr_128, $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN %4, %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dwordx2_bothen
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dwordx2_dwordx2_bothen
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN]].sub0_sub1
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN]].sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN %4, %5:sgpr_128, $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN %4, %5:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dwordx3_bothen
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_dwordx3_bothen
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN]].sub0
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vreg_96 = COPY killed [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN]].sub1_sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN %4, %5:sgpr_128, $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_96 = BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN %4, %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dword_idxen_exact
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_dword_idxen_exact
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact]].sub0
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN_exact %4, %5:sgpr_128, $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN_exact %4, %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dwordx2_idxen_exact
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_dwordx2_idxen_exact
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX3_VBUFFER_IDXEN_exact:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_VBUFFER_IDXEN_exact [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_IDXEN_exact]].sub0
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX3_VBUFFER_IDXEN_exact]].sub1_sub2
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN_exact %4, %5:sgpr_128, $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact %4, %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dwordx2_idxen_exact
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dwordx2_dwordx2_idxen_exact
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX4_VBUFFER_IDXEN_exact:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_VBUFFER_IDXEN_exact [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_IDXEN_exact]].sub0_sub1
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX4_VBUFFER_IDXEN_exact]].sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact %4, %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact %4, %5:sgpr_128, $sgpr_null, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dwordx3_idxen_exact
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_dwordx3_idxen_exact
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX4_VBUFFER_IDXEN_exact:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_VBUFFER_IDXEN_exact [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_IDXEN_exact]].sub0
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vreg_96 = COPY killed [[BUFFER_LOAD_DWORDX4_VBUFFER_IDXEN_exact]].sub1_sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN_exact %4, %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_96 = BUFFER_LOAD_DWORDX3_VBUFFER_IDXEN_exact %4, %5:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dword_dword_idxen_exact
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_dword_dword_idxen_exact
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX3_VBUFFER_IDXEN_exact:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_VBUFFER_IDXEN_exact [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_IDXEN_exact]].sub0_sub1
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX3_VBUFFER_IDXEN_exact]].sub2
+    ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]].sub0
+    ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY killed [[COPY5]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN_exact %4, %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN_exact %4, %5:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %9:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN_exact %4, %5:sgpr_128, $sgpr_null, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dword_dword_idxen_exact_swizzled_0
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_dword_dword_idxen_exact_swizzled_0
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_IDXEN_exact:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN_exact [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 4, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact]].sub0
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN_exact %4, %5:sgpr_128, $sgpr_null, 4, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN_exact %4, %5:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %9:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN_exact %4, %5:sgpr_128, $sgpr_null, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dword_bothen_exact
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_dword_bothen_exact
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact]].sub0
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN_exact %4, %5:sgpr_128, $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN_exact %4, %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dwordx2_bothen_exact
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_dwordx2_bothen_exact
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN_exact:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN_exact [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN_exact]].sub0
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN_exact]].sub1_sub2
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN_exact %4, %5:sgpr_128, $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact %4, %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dwordx2_bothen_exact
+body:             |
+  bb.0.entry:
+
+    ; GFX12-LABEL: name: buffer_load_dwordx2_dwordx2_bothen_exact
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN_exact:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN_exact [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN_exact]].sub0_sub1
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN_exact]].sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact %4, %5:sgpr_128, $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact %4, %5:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dwordx3_bothen_exact
+body:             |
+  bb.0.entry:
+
+    ; GFX12-LABEL: name: buffer_load_dword_dwordx3_bothen_exact
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN_exact:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN_exact [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN_exact]].sub0
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vreg_96 = COPY killed [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN_exact]].sub1_sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN_exact %4, %5:sgpr_128, $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_96 = BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN_exact %4, %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dword_dword_bothen_exact
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_dword_dword_bothen_exact
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN_exact:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN_exact [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN_exact]].sub0_sub1
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN_exact]].sub2
+    ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]].sub0
+    ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY killed [[COPY5]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN_exact %4, %5:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN_exact %4, %5:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %9:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN_exact %4, %5:sgpr_128, $sgpr_null, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dword_dword_bothen_exact_swizzled_0
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dword_dword_dword_bothen_exact_swizzled_0
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN_exact:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN_exact [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 4, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact]].sub0
+    ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN_exact %4, %5:sgpr_128, $sgpr_null, 4, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN_exact %4, %5:sgpr_128, $sgpr_null, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %9:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN_exact %4, %5:sgpr_128, $sgpr_null, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dwordx2_bothen_exact_diff_vaddr
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dwordx2_dwordx2_bothen_exact_diff_vaddr
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY $vgpr1
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact1:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact [[COPY5]], [[REG_SEQUENCE]], $sgpr_null, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:vreg_64 = COPY $vgpr1
+    %6:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact %4, %6:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %9:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact %5, %6:sgpr_128, $sgpr_null, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dwordx2_bothen_exact_diff_srsrc
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dwordx2_dwordx2_bothen_exact_diff_srsrc
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact [[COPY5]], [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact1:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact [[COPY5]], [[REG_SEQUENCE1]], $sgpr_null, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:sgpr_32 = COPY $sgpr4
+    %5:vreg_64 = COPY $vgpr0
+    %6:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:sgpr_128 = REG_SEQUENCE %1:sgpr_32, %subreg.sub0, %2:sgpr_32, %subreg.sub1, %3:sgpr_32, %subreg.sub2, %4:sgpr_32, %subreg.sub3
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact %5, %6:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %9:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN_exact %5, %7:sgpr_128, $sgpr_null, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dwordx2_idxen_exact_diff_vaddr
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dwordx2_dwordx2_idxen_exact_diff_vaddr
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact [[COPY4]], [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact1:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact [[COPY5]], [[REG_SEQUENCE]], $sgpr_null, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:vgpr_32 = COPY $vgpr1
+    %6:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact %4, %6:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %9:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact %5, %6:sgpr_128, $sgpr_null, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dwordx2_idxen_exact_diff_srsrc
+body:             |
+  bb.0.entry:
+    ; GFX12-LABEL: name: buffer_load_dwordx2_dwordx2_idxen_exact_diff_srsrc
+    ; GFX12: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+    ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact [[COPY5]], [[REG_SEQUENCE]], $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GFX12-NEXT: [[BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact1:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact [[COPY5]], [[REG_SEQUENCE1]], $sgpr_null, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:sgpr_32 = COPY $sgpr4
+    %5:vgpr_32 = COPY $vgpr0
+    %6:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:sgpr_128 = REG_SEQUENCE %1:sgpr_32, %subreg.sub0, %2:sgpr_32, %subreg.sub1, %3:sgpr_32, %subreg.sub2, %4:sgpr_32, %subreg.sub3
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact %5, %6:sgpr_128, $sgpr_null, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %9:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_IDXEN_exact %5, %7:sgpr_128, $sgpr_null, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
diff --git a/llvm/test/CodeGen/AMDGPU/merge-buffer.mir b/llvm/test/CodeGen/AMDGPU/merge-buffer.mir
new file mode 100644
index 000000000000..1c6d429d20ea
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/merge-buffer.mir
@@ -0,0 +1,1130 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+# RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass si-load-store-opt -o - %s | FileCheck -check-prefixes=GCN %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs -run-pass si-load-store-opt -o - %s | FileCheck -check-prefixes=GCN %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs -run-pass si-load-store-opt -o - %s | FileCheck -check-prefixes=GCN %s
+
+---
+name: buffer_load_dword_dwordx3
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_dwordx3
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFSET]].sub0
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vreg_96 = COPY killed [[BUFFER_LOAD_DWORDX4_OFFSET]].sub1_sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_96 = BUFFER_LOAD_DWORDX3_OFFSET %5:sgpr_128, 0, 8, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx3_dword
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dwordx3_dword
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vreg_96 = COPY [[BUFFER_LOAD_DWORDX4_OFFSET]].sub0_sub1_sub2
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX4_OFFSET]].sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vreg_96 = BUFFER_LOAD_DWORDX3_OFFSET %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 16, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dwordx2
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dwordx2_dwordx2
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY [[BUFFER_LOAD_DWORDX4_OFFSET]].sub0_sub1
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX4_OFFSET]].sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %5:sgpr_128, 0, 12, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dwordx2
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_dwordx2
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX3_OFFSET:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_OFFSET [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_OFFSET]].sub0
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX3_OFFSET]].sub1_sub2
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %5:sgpr_128, 0, 8, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dword
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dwordx2_dword
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX3_OFFSET:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_OFFSET [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY [[BUFFER_LOAD_DWORDX3_OFFSET]].sub0_sub1
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX3_OFFSET]].sub2
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+
+name: buffer_load_dword_dword
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_dword
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX2_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFSET]].sub0
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX2_OFFSET]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_32
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_32
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX2_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFSET]].sub0
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX2_OFFSET]].sub1
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE]], 0, 16, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vreg_96 = COPY [[BUFFER_LOAD_DWORDX4_OFFSET]].sub0_sub1_sub2
+    ; GCN-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX4_OFFSET]].sub3
+    ; GCN-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[COPY6]].sub0_sub1
+    ; GCN-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY killed [[COPY6]].sub2
+    ; GCN-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY8]].sub0
+    ; GCN-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY killed [[COPY8]].sub1
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX3_OFFSET:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_OFFSET [[REG_SEQUENCE]], 0, 36, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY12:%[0-9]+]]:vreg_64 = COPY [[BUFFER_LOAD_DWORDX3_OFFSET]].sub0_sub1
+    ; GCN-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX3_OFFSET]].sub2
+    ; GCN-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY12]].sub0
+    ; GCN-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY killed [[COPY12]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %9:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 16, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %10:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 20, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %11:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 24, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %12:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 28, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %13:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 36, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %14:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 40, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %15:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 44, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+#
+# buffer_store_dword
+#
+
+name: buffer_store_dword_xyz
+body:             |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: buffer_store_dword_xyz
+    ; GCN: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+    ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2
+    ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[REG_SEQUENCE1]], %subreg.sub1_sub2_sub3
+    ; GCN-NEXT: BUFFER_STORE_DWORDX4_OFFSET_exact killed [[REG_SEQUENCE2]], [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 4)
+    %7:vgpr_32 = COPY $vgpr3
+    %6:vgpr_32 = COPY $vgpr2
+    %5:vgpr_32 = COPY $vgpr1
+    %4:vgpr_32 = COPY $vgpr0
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %13:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %14:vreg_96 = REG_SEQUENCE %4:vgpr_32, %subreg.sub0, %5:vgpr_32, %subreg.sub1, %6:vgpr_32, %subreg.sub2
+    BUFFER_STORE_DWORD_OFFSET_exact %7:vgpr_32, %13:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORDX3_OFFSET_exact %14:vreg_96, %13:sgpr_128, 0, 8, 0, 0, implicit $exec :: (dereferenceable store (s96), align 1, addrspace 4)
+...
+---
+
+name: buffer_store_dwordx3_dword
+body:             |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: buffer_store_dwordx3_dword
+    ; GCN: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+    ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2
+    ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE1]], %subreg.sub0_sub1_sub2, [[COPY]], %subreg.sub3
+    ; GCN-NEXT: BUFFER_STORE_DWORDX4_OFFSET_exact killed [[REG_SEQUENCE2]], [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 4)
+    %7:vgpr_32 = COPY $vgpr3
+    %6:vgpr_32 = COPY $vgpr2
+    %5:vgpr_32 = COPY $vgpr1
+    %4:vgpr_32 = COPY $vgpr0
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %13:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %14:vreg_96 = REG_SEQUENCE %4:vgpr_32, %subreg.sub0, %5:vgpr_32, %subreg.sub1, %6:vgpr_32, %subreg.sub2
+    BUFFER_STORE_DWORDX3_OFFSET_exact %14:vreg_96, %13:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable store (s96), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_OFFSET_exact %7:vgpr_32, %13:sgpr_128, 0, 16, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_store_dwordx2_dwordx2
+body:             |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: buffer_store_dwordx2_dwordx2
+    ; GCN: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+    ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+    ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
+    ; GCN-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE1]], %subreg.sub0_sub1, [[REG_SEQUENCE2]], %subreg.sub2_sub3
+    ; GCN-NEXT: BUFFER_STORE_DWORDX4_OFFSET_exact killed [[REG_SEQUENCE3]], [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 4)
+    %7:vgpr_32 = COPY $vgpr3
+    %6:vgpr_32 = COPY $vgpr2
+    %5:vgpr_32 = COPY $vgpr1
+    %4:vgpr_32 = COPY $vgpr0
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %13:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %14:vreg_64 = REG_SEQUENCE %4:vgpr_32, %subreg.sub0, %5:vgpr_32, %subreg.sub1
+    %15:vreg_64 = REG_SEQUENCE %6:vgpr_32, %subreg.sub0, %7:vgpr_32, %subreg.sub1
+    BUFFER_STORE_DWORDX2_OFFSET_exact %14:vreg_64, %13:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 4)
+    BUFFER_STORE_DWORDX2_OFFSET_exact %15:vreg_64, %13:sgpr_128, 0, 12, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 4)
+...
+---
+
+name: buffer_store_dword_dwordx2
+body:             |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: buffer_store_dword_dwordx2
+    ; GCN: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+    ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+    ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY]], %subreg.sub0, %10:vreg_64, %subreg.sub1_sub2
+    ; GCN-NEXT: BUFFER_STORE_DWORDX3_OFFSET_exact killed [[REG_SEQUENCE2]], [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable store (s96), align 1, addrspace 4)
+    %7:vgpr_32 = COPY $vgpr3
+    %6:vgpr_32 = COPY $vgpr2
+    %5:vgpr_32 = COPY $vgpr1
+    %4:vgpr_32 = COPY $vgpr0
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %13:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %14:vreg_64 = REG_SEQUENCE %4:vgpr_32, %subreg.sub0, %5:vgpr_32, %subreg.sub1
+    BUFFER_STORE_DWORD_OFFSET_exact %7:vgpr_32, %13:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORDX2_OFFSET_exact %15:vreg_64, %13:sgpr_128, 0, 8, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 4)
+...
+---
+
+name: buffer_store_dwordx2_dword
+body:             |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: buffer_store_dwordx2_dword
+    ; GCN: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+    ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1
+    ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[REG_SEQUENCE1]], %subreg.sub0_sub1, [[COPY]], %subreg.sub2
+    ; GCN-NEXT: BUFFER_STORE_DWORDX3_OFFSET_exact killed [[REG_SEQUENCE2]], [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable store (s96), align 1, addrspace 4)
+    %7:vgpr_32 = COPY $vgpr3
+    %6:vgpr_32 = COPY $vgpr2
+    %5:vgpr_32 = COPY $vgpr1
+    %4:vgpr_32 = COPY $vgpr0
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %13:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %14:vreg_64 = REG_SEQUENCE %4:vgpr_32, %subreg.sub0, %5:vgpr_32, %subreg.sub1
+    BUFFER_STORE_DWORDX2_OFFSET_exact %14:vreg_64, %13:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_OFFSET_exact %7:vgpr_32, %13:sgpr_128, 0, 12, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_store_dword_dword
+body:             |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: buffer_store_dword_dword
+    ; GCN: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY7:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY6]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY4]], %subreg.sub3
+    ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
+    ; GCN-NEXT: BUFFER_STORE_DWORDX2_OFFSET_exact killed [[REG_SEQUENCE1]], [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 4)
+    %7:vgpr_32 = COPY $vgpr3
+    %6:vgpr_32 = COPY $vgpr2
+    %5:vgpr_32 = COPY $vgpr1
+    %4:vgpr_32 = COPY $vgpr0
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %13:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    BUFFER_STORE_DWORD_OFFSET_exact %6:vgpr_32, %13:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_OFFSET_exact %7:vgpr_32, %13:sgpr_128, 0, 8, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_store_dword_32
+body:             |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+    ; GCN-LABEL: name: buffer_store_dword_32
+    ; GCN: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GCN-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GCN-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY12:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY12]], %subreg.sub0, [[COPY11]], %subreg.sub1, [[COPY10]], %subreg.sub2, [[COPY9]], %subreg.sub3
+    ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY7]], %subreg.sub1
+    ; GCN-NEXT: BUFFER_STORE_DWORDX2_OFFSET_exact killed [[REG_SEQUENCE1]], [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable store (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1
+    ; GCN-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE killed [[REG_SEQUENCE2]], %subreg.sub0_sub1, [[COPY4]], %subreg.sub2
+    ; GCN-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:vreg_128 = REG_SEQUENCE killed [[REG_SEQUENCE3]], %subreg.sub0_sub1_sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: BUFFER_STORE_DWORDX4_OFFSET_exact killed [[REG_SEQUENCE4]], [[REG_SEQUENCE]], 0, 16, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 4)
+    ; GCN-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
+    ; GCN-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:vreg_96 = REG_SEQUENCE killed [[REG_SEQUENCE5]], %subreg.sub0_sub1, [[COPY]], %subreg.sub2
+    ; GCN-NEXT: BUFFER_STORE_DWORDX3_OFFSET_exact killed [[REG_SEQUENCE6]], [[REG_SEQUENCE]], 0, 36, 0, 0, implicit $exec :: (dereferenceable store (s96), align 1, addrspace 4)
+    %12:vgpr_32 = COPY $vgpr8
+    %11:vgpr_32 = COPY $vgpr7
+    %10:vgpr_32 = COPY $vgpr6
+    %9:vgpr_32 = COPY $vgpr5
+    %8:vgpr_32 = COPY $vgpr4
+    %7:vgpr_32 = COPY $vgpr3
+    %6:vgpr_32 = COPY $vgpr2
+    %5:vgpr_32 = COPY $vgpr1
+    %4:vgpr_32 = COPY $vgpr0
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %13:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    BUFFER_STORE_DWORD_OFFSET_exact %4:vgpr_32, %13:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_OFFSET_exact %5:vgpr_32, %13:sgpr_128, 0, 8, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_OFFSET_exact %6:vgpr_32, %13:sgpr_128, 0, 16, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_OFFSET_exact %7:vgpr_32, %13:sgpr_128, 0, 20, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_OFFSET_exact %8:vgpr_32, %13:sgpr_128, 0, 24, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_OFFSET_exact %9:vgpr_32, %13:sgpr_128, 0, 28, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_OFFSET_exact %10:vgpr_32, %13:sgpr_128, 0, 36, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_OFFSET_exact %11:vgpr_32, %13:sgpr_128, 0, 40, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_OFFSET_exact %12:vgpr_32, %13:sgpr_128, 0, 44, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_not_merged_swizzled_0
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_not_merged_swizzled_0
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[REG_SEQUENCE]], 0, 4, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORD_OFFSET1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[REG_SEQUENCE]], 0, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 4, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_not_merged_swizzled_1
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_not_merged_swizzled_1
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORD_OFFSET1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[REG_SEQUENCE]], 0, 8, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 8, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_merge_across_swizzle
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_merge_across_swizzle
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX2_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFSET]].sub0
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX2_OFFSET]].sub1
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[REG_SEQUENCE]], 0, 12, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %4:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %6:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %4:sgpr_128, 0, 12, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %4:sgpr_128, 0, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_merge_across_swizzled_store
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_merge_across_swizzled_store
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GCN-NEXT: BUFFER_STORE_DWORD_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], 0, 6, 0, 1, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORD_OFFSET1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[REG_SEQUENCE]], 0, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %6:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    BUFFER_STORE_DWORD_OFFSET_exact %4:vgpr_32, %5:sgpr_128, 0, 6, 0, 1, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 4)
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %5:sgpr_128, 0, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dword_idxen
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_dword_idxen
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX2_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_IDXEN [[COPY4]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_IDXEN]].sub0
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX2_IDXEN]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN %4, %5:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN %4, %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dwordx2_idxen
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_dwordx2_idxen
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX3_IDXEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_IDXEN [[COPY4]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_IDXEN]].sub0
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX3_IDXEN]].sub1_sub2
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN %4, %5:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_IDXEN %4, %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dwordx2_idxen
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dwordx2_dwordx2_idxen
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY4]], [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[BUFFER_LOAD_DWORDX4_IDXEN]].sub0_sub1
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX4_IDXEN]].sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vreg_64 = BUFFER_LOAD_DWORDX2_IDXEN %4, %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_IDXEN %4, %5:sgpr_128, 0, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dwordx3_idxen
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_dwordx3_idxen
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY4]], [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_IDXEN]].sub0
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vreg_96 = COPY killed [[BUFFER_LOAD_DWORDX4_IDXEN]].sub1_sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN %4, %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_96 = BUFFER_LOAD_DWORDX3_IDXEN %4, %5:sgpr_128, 0, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dword_bothen
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_dword_bothen
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX2_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN [[COPY4]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_BOTHEN]].sub0
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX2_BOTHEN]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN %4, %5:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN %4, %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dwordx2_bothen
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_dwordx2_bothen
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX3_BOTHEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_BOTHEN [[COPY4]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_BOTHEN]].sub0
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX3_BOTHEN]].sub1_sub2
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN %4, %5:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN %4, %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dwordx2_bothen
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dwordx2_dwordx2_bothen
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_BOTHEN [[COPY4]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[BUFFER_LOAD_DWORDX4_BOTHEN]].sub0_sub1
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX4_BOTHEN]].sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN %4, %5:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN %4, %5:sgpr_128, 0, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dwordx3_bothen
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_dwordx3_bothen
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_BOTHEN [[COPY4]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_BOTHEN]].sub0
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vreg_96 = COPY killed [[BUFFER_LOAD_DWORDX4_BOTHEN]].sub1_sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN %4, %5:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_96 = BUFFER_LOAD_DWORDX3_BOTHEN %4, %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dword_idxen_exact
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_dword_idxen_exact
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX2_IDXEN_exact:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_IDXEN_exact [[COPY4]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_IDXEN_exact]].sub0
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX2_IDXEN_exact]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN_exact %4, %5:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN_exact %4, %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dwordx2_idxen_exact
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_dwordx2_idxen_exact
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX3_IDXEN_exact:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_IDXEN_exact [[COPY4]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_IDXEN_exact]].sub0
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX3_IDXEN_exact]].sub1_sub2
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN_exact %4, %5:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_IDXEN_exact %4, %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dwordx2_idxen_exact
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dwordx2_dwordx2_idxen_exact
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN_exact:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN_exact [[COPY4]], [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[BUFFER_LOAD_DWORDX4_IDXEN_exact]].sub0_sub1
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX4_IDXEN_exact]].sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vreg_64 = BUFFER_LOAD_DWORDX2_IDXEN_exact %4, %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_IDXEN_exact %4, %5:sgpr_128, 0, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dwordx3_idxen_exact
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_dwordx3_idxen_exact
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN_exact:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN_exact [[COPY4]], [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_IDXEN_exact]].sub0
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vreg_96 = COPY killed [[BUFFER_LOAD_DWORDX4_IDXEN_exact]].sub1_sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN_exact %4, %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_96 = BUFFER_LOAD_DWORDX3_IDXEN_exact %4, %5:sgpr_128, 0, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dword_dword_idxen_exact
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_dword_dword_idxen_exact
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX3_IDXEN_exact:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_IDXEN_exact [[COPY4]], [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[BUFFER_LOAD_DWORDX3_IDXEN_exact]].sub0_sub1
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX3_IDXEN_exact]].sub2
+    ; GCN-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]].sub0
+    ; GCN-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY killed [[COPY5]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN_exact %4, %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN_exact %4, %5:sgpr_128, 0, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %9:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN_exact %4, %5:sgpr_128, 0, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dword_dword_idxen_exact_swizzled_0
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_dword_dword_idxen_exact_swizzled_0
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORD_IDXEN_exact:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN_exact [[COPY4]], [[REG_SEQUENCE]], 0, 4, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX2_IDXEN_exact:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_IDXEN_exact [[COPY4]], [[REG_SEQUENCE]], 0, 8, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_IDXEN_exact]].sub0
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX2_IDXEN_exact]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN_exact %4, %5:sgpr_128, 0, 4, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN_exact %4, %5:sgpr_128, 0, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %9:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN_exact %4, %5:sgpr_128, 0, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dword_bothen_exact
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_dword_bothen_exact
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX2_BOTHEN_exact:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN_exact [[COPY4]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_BOTHEN_exact]].sub0
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX2_BOTHEN_exact]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN_exact %4, %5:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN_exact %4, %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dwordx2_bothen_exact
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_dwordx2_bothen_exact
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX3_BOTHEN_exact:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_BOTHEN_exact [[COPY4]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_BOTHEN_exact]].sub0
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX3_BOTHEN_exact]].sub1_sub2
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN_exact %4, %5:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN_exact %4, %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dwordx2_bothen_exact
+body:             |
+  bb.0.entry:
+
+    ; GCN-LABEL: name: buffer_load_dwordx2_dwordx2_bothen_exact
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_BOTHEN_exact:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_BOTHEN_exact [[COPY4]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[BUFFER_LOAD_DWORDX4_BOTHEN_exact]].sub0_sub1
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY killed [[BUFFER_LOAD_DWORDX4_BOTHEN_exact]].sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN_exact %4, %5:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN_exact %4, %5:sgpr_128, 0, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dwordx3_bothen_exact
+body:             |
+  bb.0.entry:
+
+    ; GCN-LABEL: name: buffer_load_dword_dwordx3_bothen_exact
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_BOTHEN_exact:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_BOTHEN_exact [[COPY4]], [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_BOTHEN_exact]].sub0
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vreg_96 = COPY killed [[BUFFER_LOAD_DWORDX4_BOTHEN_exact]].sub1_sub2_sub3
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN_exact %4, %5:sgpr_128, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vreg_96 = BUFFER_LOAD_DWORDX3_BOTHEN_exact %4, %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dword_dword_bothen_exact
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_dword_dword_bothen_exact
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX3_BOTHEN_exact:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_BOTHEN_exact [[COPY4]], [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[BUFFER_LOAD_DWORDX3_BOTHEN_exact]].sub0_sub1
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX3_BOTHEN_exact]].sub2
+    ; GCN-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]].sub0
+    ; GCN-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY killed [[COPY5]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN_exact %4, %5:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN_exact %4, %5:sgpr_128, 0, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %9:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN_exact %4, %5:sgpr_128, 0, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dword_dword_dword_bothen_exact_swizzled_0
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dword_dword_dword_bothen_exact_swizzled_0
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORD_BOTHEN_exact:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN_exact [[COPY4]], [[REG_SEQUENCE]], 0, 4, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX2_BOTHEN_exact:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN_exact [[COPY4]], [[REG_SEQUENCE]], 0, 8, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 4)
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_BOTHEN_exact]].sub0
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY killed [[BUFFER_LOAD_DWORDX2_BOTHEN_exact]].sub1
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN_exact %4, %5:sgpr_128, 0, 4, 0, 1, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %8:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN_exact %4, %5:sgpr_128, 0, 8, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %9:vgpr_32 = BUFFER_LOAD_DWORD_BOTHEN_exact %4, %5:sgpr_128, 0, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dwordx2_bothen_exact_diff_vaddr
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dwordx2_dwordx2_bothen_exact_diff_vaddr
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY $vgpr1
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX2_BOTHEN_exact:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN_exact [[COPY4]], [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX2_BOTHEN_exact1:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN_exact [[COPY5]], [[REG_SEQUENCE]], 0, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vreg_64 = COPY $vgpr0
+    %5:vreg_64 = COPY $vgpr1
+    %6:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN_exact %4, %6:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %9:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN_exact %5, %6:sgpr_128, 0, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dwordx2_bothen_exact_diff_srsrc
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dwordx2_dwordx2_bothen_exact_diff_srsrc
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX2_BOTHEN_exact:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN_exact [[COPY5]], [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX2_BOTHEN_exact1:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN_exact [[COPY5]], [[REG_SEQUENCE1]], 0, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:sgpr_32 = COPY $sgpr4
+    %5:vreg_64 = COPY $vgpr0
+    %6:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:sgpr_128 = REG_SEQUENCE %1:sgpr_32, %subreg.sub0, %2:sgpr_32, %subreg.sub1, %3:sgpr_32, %subreg.sub2, %4:sgpr_32, %subreg.sub3
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN_exact %5, %6:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %9:vreg_64 = BUFFER_LOAD_DWORDX2_BOTHEN_exact %5, %7:sgpr_128, 0, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dwordx2_idxen_exact_diff_vaddr
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dwordx2_dwordx2_idxen_exact_diff_vaddr
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX2_IDXEN_exact:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_IDXEN_exact [[COPY4]], [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX2_IDXEN_exact1:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_IDXEN_exact [[COPY5]], [[REG_SEQUENCE]], 0, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:vgpr_32 = COPY $vgpr0
+    %5:vgpr_32 = COPY $vgpr1
+    %6:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_IDXEN_exact %4, %6:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %9:vreg_64 = BUFFER_LOAD_DWORDX2_IDXEN_exact %5, %6:sgpr_128, 0, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
+---
+
+name: buffer_load_dwordx2_dwordx2_idxen_exact_diff_srsrc
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: buffer_load_dwordx2_dwordx2_idxen_exact_diff_srsrc
+    ; GCN: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX2_IDXEN_exact:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_IDXEN_exact [[COPY5]], [[REG_SEQUENCE]], 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GCN-NEXT: [[BUFFER_LOAD_DWORDX2_IDXEN_exact1:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_IDXEN_exact [[COPY5]], [[REG_SEQUENCE1]], 0, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %0:sgpr_32 = COPY $sgpr0
+    %1:sgpr_32 = COPY $sgpr1
+    %2:sgpr_32 = COPY $sgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %4:sgpr_32 = COPY $sgpr4
+    %5:vgpr_32 = COPY $vgpr0
+    %6:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %7:sgpr_128 = REG_SEQUENCE %1:sgpr_32, %subreg.sub0, %2:sgpr_32, %subreg.sub1, %3:sgpr_32, %subreg.sub2, %4:sgpr_32, %subreg.sub3
+    %8:vreg_64 = BUFFER_LOAD_DWORDX2_IDXEN_exact %5, %6:sgpr_128, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    %9:vreg_64 = BUFFER_LOAD_DWORDX2_IDXEN_exact %5, %7:sgpr_128, 0, 12, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+...
diff --git a/llvm/test/CodeGen/AMDGPU/merge-tbuffer.mir b/llvm/test/CodeGen/AMDGPU/merge-tbuffer.mir
index c86b5adec372..9766b427b432 100644
--- a/llvm/test/CodeGen/AMDGPU/merge-tbuffer.mir
+++ b/llvm/test/CodeGen/AMDGPU/merge-tbuffer.mir
@@ -7,9 +7,37 @@
 # GFX9 tests
 #
 
+---
 name: gfx9_tbuffer_load_x_xyz
 body:             |
   bb.0.entry:
+    ; GFX9-LABEL: name: gfx9_tbuffer_load_x_xyz
+    ; GFX9: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX9-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX9-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX9-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX9-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX9-NEXT: [[TBUFFER_LOAD_FORMAT_XYZW_OFFSET:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE]], 0, 4, 126, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 4)
+    ; GFX9-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_OFFSET]].sub0
+    ; GFX9-NEXT: [[COPY5:%[0-9]+]]:vreg_96 = COPY killed [[TBUFFER_LOAD_FORMAT_XYZW_OFFSET]].sub1_sub2_sub3
+    ;
+    ; GFX10-LABEL: name: gfx9_tbuffer_load_x_xyz
+    ; GFX10: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX10-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX10-NEXT: [[TBUFFER_LOAD_FORMAT_X_OFFSET:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFSET [[REG_SEQUENCE]], 0, 4, 116, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GFX10-NEXT: [[TBUFFER_LOAD_FORMAT_XYZ_OFFSET:%[0-9]+]]:vreg_96 = TBUFFER_LOAD_FORMAT_XYZ_OFFSET [[REG_SEQUENCE]], 0, 8, 125, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 4)
+    ;
+    ; GFX11-LABEL: name: gfx9_tbuffer_load_x_xyz
+    ; GFX11: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX11-NEXT: [[TBUFFER_LOAD_FORMAT_X_OFFSET:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_OFFSET [[REG_SEQUENCE]], 0, 4, 116, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
+    ; GFX11-NEXT: [[TBUFFER_LOAD_FORMAT_XYZ_OFFSET:%[0-9]+]]:vreg_96 = TBUFFER_LOAD_FORMAT_XYZ_OFFSET [[REG_SEQUENCE]], 0, 8, 125, 0, 0, implicit $exec :: (dereferenceable load (s96), align 1, addrspace 4)
     %0:sgpr_32 = COPY $sgpr0
     %1:sgpr_32 = COPY $sgpr1
     %2:sgpr_32 = COPY $sgpr2
-- 
GitLab


From 564035edb0e67a177fd911fc779cd64300a9b5ef Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Mon, 25 Mar 2024 09:54:04 -0500
Subject: [PATCH 130/404] [flang][OpenMP] Organize `genOMP` functions in
 OpenMP.cpp, NFC (#86309)

Put all of the genOMP functions together, organize them in two groups:
for declarative constructs and for other (executable) constructs.

Replace visit functions for OpenMPDeclarativeConstruct and
OpenMPConstruct from listing individual visitors for each variant
alternative to using a single generic visitor. Essentially, going from
```
  std::visit(
    [](foo x) { genOMP(foo); }
    [](bar x) { TODO }
    [](baz x) { genOMP(baz); }
  )
```
to
```
void genOMP(bar x) {  // Separate visitor for an unhandled case
  TODO
}

[...]
  std::visit([&](auto &&s) { genOMP(s); })  // generic
```

This doesn't change any functionality, just reorganizes the functions a
bit. The intent here is to improve the readability of this file.
---
 flang/lib/Lower/OpenMP/OpenMP.cpp | 541 +++++++++++++++---------------
 1 file changed, 279 insertions(+), 262 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index d91694c4f639..02901ed070a7 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -1406,34 +1406,6 @@ genOmpFlush(Fortran::lower::AbstractConverter &converter,
       converter.getCurrentLocation(), operandRange);
 }
 
-static void
-genOMP(Fortran::lower::AbstractConverter &converter,
-       Fortran::lower::SymMap &symTable,
-       Fortran::semantics::SemanticsContext &semaCtx,
-       Fortran::lower::pft::Evaluation &eval,
-       const Fortran::parser::OpenMPStandaloneConstruct &standaloneConstruct) {
-  std::visit(
-      Fortran::common::visitors{
-          [&](const Fortran::parser::OpenMPSimpleStandaloneConstruct
-                  &simpleStandaloneConstruct) {
-            genOmpSimpleStandalone(converter, semaCtx, eval,
-                                   /*genNested=*/true,
-                                   simpleStandaloneConstruct);
-          },
-          [&](const Fortran::parser::OpenMPFlushConstruct &flushConstruct) {
-            genOmpFlush(converter, semaCtx, eval, flushConstruct);
-          },
-          [&](const Fortran::parser::OpenMPCancelConstruct &cancelConstruct) {
-            TODO(converter.getCurrentLocation(), "OpenMPCancelConstruct");
-          },
-          [&](const Fortran::parser::OpenMPCancellationPointConstruct
-                  &cancellationPointConstruct) {
-            TODO(converter.getCurrentLocation(), "OpenMPCancelConstruct");
-          },
-      },
-      standaloneConstruct.u);
-}
-
 static llvm::SmallVector<const Fortran::semantics::Symbol *>
 genLoopVars(mlir::Operation *op, Fortran::lower::AbstractConverter &converter,
             mlir::Location &loc,
@@ -1672,88 +1644,181 @@ static void createSimdWsloop(
                endClauseList, loc);
 }
 
+static void
+markDeclareTarget(mlir::Operation *op,
+                  Fortran::lower::AbstractConverter &converter,
+                  mlir::omp::DeclareTargetCaptureClause captureClause,
+                  mlir::omp::DeclareTargetDeviceType deviceType) {
+  // TODO: Add support for program local variables with declare target applied
+  auto declareTargetOp = llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(op);
+  if (!declareTargetOp)
+    fir::emitFatalError(
+        converter.getCurrentLocation(),
+        "Attempt to apply declare target on unsupported operation");
+
+  // The function or global already has a declare target applied to it, very
+  // likely through implicit capture (usage in another declare target
+  // function/subroutine). It should be marked as any if it has been assigned
+  // both host and nohost, else we skip, as there is no change
+  if (declareTargetOp.isDeclareTarget()) {
+    if (declareTargetOp.getDeclareTargetDeviceType() != deviceType)
+      declareTargetOp.setDeclareTarget(mlir::omp::DeclareTargetDeviceType::any,
+                                       captureClause);
+    return;
+  }
+
+  declareTargetOp.setDeclareTarget(deviceType, captureClause);
+}
+
+//===----------------------------------------------------------------------===//
+// OpenMPDeclarativeConstruct visitors
+//===----------------------------------------------------------------------===//
+
+static void
+genOMP(Fortran::lower::AbstractConverter &converter,
+       Fortran::lower::SymMap &symTable,
+       Fortran::semantics::SemanticsContext &semaCtx,
+       Fortran::lower::pft::Evaluation &eval,
+       const Fortran::parser::OpenMPDeclarativeAllocate &declarativeAllocate) {
+  TODO(converter.getCurrentLocation(), "OpenMPDeclarativeAllocate");
+}
+
 static void genOMP(Fortran::lower::AbstractConverter &converter,
                    Fortran::lower::SymMap &symTable,
                    Fortran::semantics::SemanticsContext &semaCtx,
                    Fortran::lower::pft::Evaluation &eval,
-                   const Fortran::parser::OpenMPLoopConstruct &loopConstruct) {
-  const auto &beginLoopDirective =
-      std::get<Fortran::parser::OmpBeginLoopDirective>(loopConstruct.t);
-  const auto &loopOpClauseList =
-      std::get<Fortran::parser::OmpClauseList>(beginLoopDirective.t);
-  mlir::Location currentLocation =
-      converter.genLocation(beginLoopDirective.source);
-  const auto ompDirective =
-      std::get<Fortran::parser::OmpLoopDirective>(beginLoopDirective.t).v;
+                   const Fortran::parser::OpenMPDeclareReductionConstruct
+                       &declareReductionConstruct) {
+  TODO(converter.getCurrentLocation(), "OpenMPDeclareReductionConstruct");
+}
 
-  const auto *endClauseList = [&]() {
-    using RetTy = const Fortran::parser::OmpClauseList *;
-    if (auto &endLoopDirective =
-            std::get<std::optional<Fortran::parser::OmpEndLoopDirective>>(
-                loopConstruct.t)) {
-      return RetTy(
-          &std::get<Fortran::parser::OmpClauseList>((*endLoopDirective).t));
-    }
-    return RetTy();
-  }();
+static void genOMP(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::lower::SymMap &symTable,
+    Fortran::semantics::SemanticsContext &semaCtx,
+    Fortran::lower::pft::Evaluation &eval,
+    const Fortran::parser::OpenMPDeclareSimdConstruct &declareSimdConstruct) {
+  TODO(converter.getCurrentLocation(), "OpenMPDeclareSimdConstruct");
+}
 
-  bool validDirective = false;
-  if (llvm::omp::topTaskloopSet.test(ompDirective)) {
-    validDirective = true;
-    TODO(currentLocation, "Taskloop construct");
-  } else {
-    // Create omp.{target, teams, distribute, parallel} nested operations
-    if ((llvm::omp::allTargetSet & llvm::omp::loopConstructSet)
-            .test(ompDirective)) {
-      validDirective = true;
-      genTargetOp(converter, semaCtx, eval, /*genNested=*/false,
-                  currentLocation, loopOpClauseList, ompDirective,
-                  /*outerCombined=*/true);
-    }
-    if ((llvm::omp::allTeamsSet & llvm::omp::loopConstructSet)
-            .test(ompDirective)) {
-      validDirective = true;
-      genTeamsOp(converter, semaCtx, eval, /*genNested=*/false, currentLocation,
-                 loopOpClauseList,
-                 /*outerCombined=*/true);
-    }
-    if (llvm::omp::allDistributeSet.test(ompDirective)) {
-      validDirective = true;
-      TODO(currentLocation, "Distribute construct");
-    }
-    if ((llvm::omp::allParallelSet & llvm::omp::loopConstructSet)
-            .test(ompDirective)) {
-      validDirective = true;
-      genParallelOp(converter, symTable, semaCtx, eval, /*genNested=*/false,
-                    currentLocation, loopOpClauseList,
-                    /*outerCombined=*/true);
-    }
-  }
-  if ((llvm::omp::allDoSet | llvm::omp::allSimdSet).test(ompDirective))
-    validDirective = true;
+static void genOMP(Fortran::lower::AbstractConverter &converter,
+                   Fortran::lower::SymMap &symTable,
+                   Fortran::semantics::SemanticsContext &semaCtx,
+                   Fortran::lower::pft::Evaluation &eval,
+                   const Fortran::parser::OpenMPDeclareTargetConstruct
+                       &declareTargetConstruct) {
+  llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
+  mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
+  mlir::omp::DeclareTargetDeviceType deviceType = getDeclareTargetInfo(
+      converter, semaCtx, eval, declareTargetConstruct, symbolAndClause);
 
-  if (!validDirective) {
-    TODO(currentLocation, "Unhandled loop directive (" +
-                              llvm::omp::getOpenMPDirectiveName(ompDirective) +
-                              ")");
-  }
+  for (const DeclareTargetCapturePair &symClause : symbolAndClause) {
+    mlir::Operation *op = mod.lookupSymbol(converter.mangleName(
+        std::get<const Fortran::semantics::Symbol &>(symClause)));
 
-  if (llvm::omp::allDoSimdSet.test(ompDirective)) {
-    // 2.9.3.2 Workshare SIMD construct
-    createSimdWsloop(converter, semaCtx, eval, ompDirective, loopOpClauseList,
-                     endClauseList, currentLocation);
+    // Some symbols are deferred until later in the module, these are handled
+    // upon finalization of the module for OpenMP inside of Bridge, so we simply
+    // skip for now.
+    if (!op)
+      continue;
 
-  } else if (llvm::omp::allSimdSet.test(ompDirective)) {
-    // 2.9.3.1 SIMD construct
-    createSimdLoop(converter, semaCtx, eval, ompDirective, loopOpClauseList,
-                   currentLocation);
-    genOpenMPReduction(converter, semaCtx, loopOpClauseList);
-  } else {
-    createWsloop(converter, semaCtx, eval, ompDirective, loopOpClauseList,
-                 endClauseList, currentLocation);
+    markDeclareTarget(
+        op, converter,
+        std::get<mlir::omp::DeclareTargetCaptureClause>(symClause), deviceType);
   }
 }
 
+static void
+genOMP(Fortran::lower::AbstractConverter &converter,
+       Fortran::lower::SymMap &symTable,
+       Fortran::semantics::SemanticsContext &semaCtx,
+       Fortran::lower::pft::Evaluation &eval,
+       const Fortran::parser::OpenMPRequiresConstruct &requiresConstruct) {
+  // Requires directives are gathered and processed in semantics and
+  // then combined in the lowering bridge before triggering codegen
+  // just once. Hence, there is no need to lower each individual
+  // occurrence here.
+}
+
+static void genOMP(Fortran::lower::AbstractConverter &converter,
+                   Fortran::lower::SymMap &symTable,
+                   Fortran::semantics::SemanticsContext &semaCtx,
+                   Fortran::lower::pft::Evaluation &eval,
+                   const Fortran::parser::OpenMPThreadprivate &threadprivate) {
+  // The directive is lowered when instantiating the variable to
+  // support the case of threadprivate variable declared in module.
+}
+
+static void
+genOMP(Fortran::lower::AbstractConverter &converter,
+       Fortran::lower::SymMap &symTable,
+       Fortran::semantics::SemanticsContext &semaCtx,
+       Fortran::lower::pft::Evaluation &eval,
+       const Fortran::parser::OpenMPDeclarativeConstruct &ompDeclConstruct) {
+  std::visit(
+      [&](auto &&s) { return genOMP(converter, symTable, semaCtx, eval, s); },
+      ompDeclConstruct.u);
+}
+
+//===----------------------------------------------------------------------===//
+// OpenMPConstruct visitors
+//===----------------------------------------------------------------------===//
+
+static void
+genOMP(Fortran::lower::AbstractConverter &converter,
+       Fortran::lower::SymMap &symTable,
+       Fortran::semantics::SemanticsContext &semaCtx,
+       Fortran::lower::pft::Evaluation &eval,
+       const Fortran::parser::OpenMPAllocatorsConstruct &allocsConstruct) {
+  TODO(converter.getCurrentLocation(), "OpenMPAllocatorsConstruct");
+}
+
+static void
+genOMP(Fortran::lower::AbstractConverter &converter,
+       Fortran::lower::SymMap &symTable,
+       Fortran::semantics::SemanticsContext &semaCtx,
+       Fortran::lower::pft::Evaluation &eval,
+       const Fortran::parser::OpenMPAtomicConstruct &atomicConstruct) {
+  std::visit(
+      Fortran::common::visitors{
+          [&](const Fortran::parser::OmpAtomicRead &atomicRead) {
+            mlir::Location loc = converter.genLocation(atomicRead.source);
+            Fortran::lower::genOmpAccAtomicRead<
+                Fortran::parser::OmpAtomicRead,
+                Fortran::parser::OmpAtomicClauseList>(converter, atomicRead,
+                                                      loc);
+          },
+          [&](const Fortran::parser::OmpAtomicWrite &atomicWrite) {
+            mlir::Location loc = converter.genLocation(atomicWrite.source);
+            Fortran::lower::genOmpAccAtomicWrite<
+                Fortran::parser::OmpAtomicWrite,
+                Fortran::parser::OmpAtomicClauseList>(converter, atomicWrite,
+                                                      loc);
+          },
+          [&](const Fortran::parser::OmpAtomic &atomicConstruct) {
+            mlir::Location loc = converter.genLocation(atomicConstruct.source);
+            Fortran::lower::genOmpAtomic<Fortran::parser::OmpAtomic,
+                                         Fortran::parser::OmpAtomicClauseList>(
+                converter, atomicConstruct, loc);
+          },
+          [&](const Fortran::parser::OmpAtomicUpdate &atomicUpdate) {
+            mlir::Location loc = converter.genLocation(atomicUpdate.source);
+            Fortran::lower::genOmpAccAtomicUpdate<
+                Fortran::parser::OmpAtomicUpdate,
+                Fortran::parser::OmpAtomicClauseList>(converter, atomicUpdate,
+                                                      loc);
+          },
+          [&](const Fortran::parser::OmpAtomicCapture &atomicCapture) {
+            mlir::Location loc = converter.genLocation(atomicCapture.source);
+            Fortran::lower::genOmpAccAtomicCapture<
+                Fortran::parser::OmpAtomicCapture,
+                Fortran::parser::OmpAtomicClauseList>(converter, atomicCapture,
+                                                      loc);
+          },
+      },
+      atomicConstruct.u);
+}
+
 static void
 genOMP(Fortran::lower::AbstractConverter &converter,
        Fortran::lower::SymMap &symTable,
@@ -1934,6 +1999,107 @@ genOMP(Fortran::lower::AbstractConverter &converter,
   createBodyOfOp<mlir::omp::CriticalOp>(criticalOp, genInfo);
 }
 
+static void
+genOMP(Fortran::lower::AbstractConverter &converter,
+       Fortran::lower::SymMap &symTable,
+       Fortran::semantics::SemanticsContext &semaCtx,
+       Fortran::lower::pft::Evaluation &eval,
+       const Fortran::parser::OpenMPExecutableAllocate &execAllocConstruct) {
+  TODO(converter.getCurrentLocation(), "OpenMPExecutableAllocate");
+}
+
+static void genOMP(Fortran::lower::AbstractConverter &converter,
+                   Fortran::lower::SymMap &symTable,
+                   Fortran::semantics::SemanticsContext &semaCtx,
+                   Fortran::lower::pft::Evaluation &eval,
+                   const Fortran::parser::OpenMPLoopConstruct &loopConstruct) {
+  const auto &beginLoopDirective =
+      std::get<Fortran::parser::OmpBeginLoopDirective>(loopConstruct.t);
+  const auto &loopOpClauseList =
+      std::get<Fortran::parser::OmpClauseList>(beginLoopDirective.t);
+  mlir::Location currentLocation =
+      converter.genLocation(beginLoopDirective.source);
+  const auto ompDirective =
+      std::get<Fortran::parser::OmpLoopDirective>(beginLoopDirective.t).v;
+
+  const auto *endClauseList = [&]() {
+    using RetTy = const Fortran::parser::OmpClauseList *;
+    if (auto &endLoopDirective =
+            std::get<std::optional<Fortran::parser::OmpEndLoopDirective>>(
+                loopConstruct.t)) {
+      return RetTy(
+          &std::get<Fortran::parser::OmpClauseList>((*endLoopDirective).t));
+    }
+    return RetTy();
+  }();
+
+  bool validDirective = false;
+  if (llvm::omp::topTaskloopSet.test(ompDirective)) {
+    validDirective = true;
+    TODO(currentLocation, "Taskloop construct");
+  } else {
+    // Create omp.{target, teams, distribute, parallel} nested operations
+    if ((llvm::omp::allTargetSet & llvm::omp::loopConstructSet)
+            .test(ompDirective)) {
+      validDirective = true;
+      genTargetOp(converter, semaCtx, eval, /*genNested=*/false,
+                  currentLocation, loopOpClauseList, ompDirective,
+                  /*outerCombined=*/true);
+    }
+    if ((llvm::omp::allTeamsSet & llvm::omp::loopConstructSet)
+            .test(ompDirective)) {
+      validDirective = true;
+      genTeamsOp(converter, semaCtx, eval, /*genNested=*/false, currentLocation,
+                 loopOpClauseList,
+                 /*outerCombined=*/true);
+    }
+    if (llvm::omp::allDistributeSet.test(ompDirective)) {
+      validDirective = true;
+      TODO(currentLocation, "Distribute construct");
+    }
+    if ((llvm::omp::allParallelSet & llvm::omp::loopConstructSet)
+            .test(ompDirective)) {
+      validDirective = true;
+      genParallelOp(converter, symTable, semaCtx, eval, /*genNested=*/false,
+                    currentLocation, loopOpClauseList,
+                    /*outerCombined=*/true);
+    }
+  }
+  if ((llvm::omp::allDoSet | llvm::omp::allSimdSet).test(ompDirective))
+    validDirective = true;
+
+  if (!validDirective) {
+    TODO(currentLocation, "Unhandled loop directive (" +
+                              llvm::omp::getOpenMPDirectiveName(ompDirective) +
+                              ")");
+  }
+
+  if (llvm::omp::allDoSimdSet.test(ompDirective)) {
+    // 2.9.3.2 Workshare SIMD construct
+    createSimdWsloop(converter, semaCtx, eval, ompDirective, loopOpClauseList,
+                     endClauseList, currentLocation);
+
+  } else if (llvm::omp::allSimdSet.test(ompDirective)) {
+    // 2.9.3.1 SIMD construct
+    createSimdLoop(converter, semaCtx, eval, ompDirective, loopOpClauseList,
+                   currentLocation);
+    genOpenMPReduction(converter, semaCtx, loopOpClauseList);
+  } else {
+    createWsloop(converter, semaCtx, eval, ompDirective, loopOpClauseList,
+                 endClauseList, currentLocation);
+  }
+}
+
+static void
+genOMP(Fortran::lower::AbstractConverter &converter,
+       Fortran::lower::SymMap &symTable,
+       Fortran::semantics::SemanticsContext &semaCtx,
+       Fortran::lower::pft::Evaluation &eval,
+       const Fortran::parser::OpenMPSectionConstruct &sectionConstruct) {
+  // SECTION constructs are handled as a part of SECTIONS.
+  llvm_unreachable("Unexpected standalone OMP SECTION");
+}
+
 static void
 genOMP(Fortran::lower::AbstractConverter &converter,
        Fortran::lower::SymMap &symTable,
@@ -1999,98 +2165,27 @@ genOMP(Fortran::lower::AbstractConverter &converter,
        Fortran::lower::SymMap &symTable,
        Fortran::semantics::SemanticsContext &semaCtx,
        Fortran::lower::pft::Evaluation &eval,
-       const Fortran::parser::OpenMPAtomicConstruct &atomicConstruct) {
+       const Fortran::parser::OpenMPStandaloneConstruct &standaloneConstruct) {
   std::visit(
       Fortran::common::visitors{
-          [&](const Fortran::parser::OmpAtomicRead &atomicRead) {
-            mlir::Location loc = converter.genLocation(atomicRead.source);
-            Fortran::lower::genOmpAccAtomicRead<
-                Fortran::parser::OmpAtomicRead,
-                Fortran::parser::OmpAtomicClauseList>(converter, atomicRead,
-                                                      loc);
-          },
-          [&](const Fortran::parser::OmpAtomicWrite &atomicWrite) {
-            mlir::Location loc = converter.genLocation(atomicWrite.source);
-            Fortran::lower::genOmpAccAtomicWrite<
-                Fortran::parser::OmpAtomicWrite,
-                Fortran::parser::OmpAtomicClauseList>(converter, atomicWrite,
-                                                      loc);
+          [&](const Fortran::parser::OpenMPSimpleStandaloneConstruct
+                  &simpleStandaloneConstruct) {
+            genOmpSimpleStandalone(converter, semaCtx, eval,
+                                   /*genNested=*/true,
+                                   simpleStandaloneConstruct);
           },
-          [&](const Fortran::parser::OmpAtomic &atomicConstruct) {
-            mlir::Location loc = converter.genLocation(atomicConstruct.source);
-            Fortran::lower::genOmpAtomic<Fortran::parser::OmpAtomic,
-                                         Fortran::parser::OmpAtomicClauseList>(
-                converter, atomicConstruct, loc);
+          [&](const Fortran::parser::OpenMPFlushConstruct &flushConstruct) {
+            genOmpFlush(converter, semaCtx, eval, flushConstruct);
           },
-          [&](const Fortran::parser::OmpAtomicUpdate &atomicUpdate) {
-            mlir::Location loc = converter.genLocation(atomicUpdate.source);
-            Fortran::lower::genOmpAccAtomicUpdate<
-                Fortran::parser::OmpAtomicUpdate,
-                Fortran::parser::OmpAtomicClauseList>(converter, atomicUpdate,
-                                                      loc);
+          [&](const Fortran::parser::OpenMPCancelConstruct &cancelConstruct) {
+            TODO(converter.getCurrentLocation(), "OpenMPCancelConstruct");
           },
-          [&](const Fortran::parser::OmpAtomicCapture &atomicCapture) {
-            mlir::Location loc = converter.genLocation(atomicCapture.source);
-            Fortran::lower::genOmpAccAtomicCapture<
-                Fortran::parser::OmpAtomicCapture,
-                Fortran::parser::OmpAtomicClauseList>(converter, atomicCapture,
-                                                      loc);
+          [&](const Fortran::parser::OpenMPCancellationPointConstruct
+                  &cancellationPointConstruct) {
+            TODO(converter.getCurrentLocation(), "OpenMPCancelConstruct");
           },
       },
-      atomicConstruct.u);
-}
-
-static void
-markDeclareTarget(mlir::Operation *op,
-                  Fortran::lower::AbstractConverter &converter,
-                  mlir::omp::DeclareTargetCaptureClause captureClause,
-                  mlir::omp::DeclareTargetDeviceType deviceType) {
-  // TODO: Add support for program local variables with declare target applied
-  auto declareTargetOp = llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(op);
-  if (!declareTargetOp)
-    fir::emitFatalError(
-        converter.getCurrentLocation(),
-        "Attempt to apply declare target on unsupported operation");
-
-  // The function or global already has a declare target applied to it, very
-  // likely through implicit capture (usage in another declare target
-  // function/subroutine). It should be marked as any if it has been assigned
-  // both host and nohost, else we skip, as there is no change
-  if (declareTargetOp.isDeclareTarget()) {
-    if (declareTargetOp.getDeclareTargetDeviceType() != deviceType)
-      declareTargetOp.setDeclareTarget(mlir::omp::DeclareTargetDeviceType::any,
-                                       captureClause);
-    return;
-  }
-
-  declareTargetOp.setDeclareTarget(deviceType, captureClause);
-}
-
-static void genOMP(Fortran::lower::AbstractConverter &converter,
-                   Fortran::lower::SymMap &symTable,
-                   Fortran::semantics::SemanticsContext &semaCtx,
-                   Fortran::lower::pft::Evaluation &eval,
-                   const Fortran::parser::OpenMPDeclareTargetConstruct
-                       &declareTargetConstruct) {
-  llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
-  mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
-  mlir::omp::DeclareTargetDeviceType deviceType = getDeclareTargetInfo(
-      converter, semaCtx, eval, declareTargetConstruct, symbolAndClause);
-
-  for (const DeclareTargetCapturePair &symClause : symbolAndClause) {
-    mlir::Operation *op = mod.lookupSymbol(converter.mangleName(
-        std::get<const Fortran::semantics::Symbol &>(symClause)));
-
-    // Some symbols are deferred until later in the module, these are handled
-    // upon finalization of the module for OpenMP inside of Bridge, so we simply
-    // skip for now.
-    if (!op)
-      continue;
-
-    markDeclareTarget(
-        op, converter,
-        std::get<mlir::omp::DeclareTargetCaptureClause>(symClause), deviceType);
-  }
+      standaloneConstruct.u);
 }
 
 static void genOMP(Fortran::lower::AbstractConverter &converter,
@@ -2099,88 +2194,10 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
                    Fortran::lower::pft::Evaluation &eval,
                    const Fortran::parser::OpenMPConstruct &ompConstruct) {
   std::visit(
-      Fortran::common::visitors{
-          [&](const Fortran::parser::OpenMPStandaloneConstruct
-                  &standaloneConstruct) {
-            genOMP(converter, symTable, semaCtx, eval, standaloneConstruct);
-          },
-          [&](const Fortran::parser::OpenMPSectionsConstruct
-                  &sectionsConstruct) {
-            genOMP(converter, symTable, semaCtx, eval, sectionsConstruct);
-          },
-          [&](const Fortran::parser::OpenMPSectionConstruct &sectionConstruct) {
-            // SECTION constructs are handled as a part of SECTIONS.
-            llvm_unreachable("Unexpected standalone OMP SECTION");
-          },
-          [&](const Fortran::parser::OpenMPLoopConstruct &loopConstruct) {
-            genOMP(converter, symTable, semaCtx, eval, loopConstruct);
-          },
-          [&](const Fortran::parser::OpenMPDeclarativeAllocate
-                  &execAllocConstruct) {
-            TODO(converter.getCurrentLocation(), "OpenMPDeclarativeAllocate");
-          },
-          [&](const Fortran::parser::OpenMPExecutableAllocate
-                  &execAllocConstruct) {
-            TODO(converter.getCurrentLocation(), "OpenMPExecutableAllocate");
-          },
-          [&](const Fortran::parser::OpenMPAllocatorsConstruct
-                  &allocsConstruct) {
-            TODO(converter.getCurrentLocation(), "OpenMPAllocatorsConstruct");
-          },
-          [&](const Fortran::parser::OpenMPBlockConstruct &blockConstruct) {
-            genOMP(converter, symTable, semaCtx, eval, blockConstruct);
-          },
-          [&](const Fortran::parser::OpenMPAtomicConstruct &atomicConstruct) {
-            genOMP(converter, symTable, semaCtx, eval, atomicConstruct);
-          },
-          [&](const Fortran::parser::OpenMPCriticalConstruct
-                  &criticalConstruct) {
-            genOMP(converter, symTable, semaCtx, eval, criticalConstruct);
-          },
-      },
+      [&](auto &&s) { return genOMP(converter, symTable, semaCtx, eval, s); },
       ompConstruct.u);
 }
 
-static void
-genOMP(Fortran::lower::AbstractConverter &converter,
-       Fortran::lower::SymMap &symTable,
-       Fortran::semantics::SemanticsContext &semaCtx,
-       Fortran::lower::pft::Evaluation &eval,
-       const Fortran::parser::OpenMPDeclarativeConstruct &ompDeclConstruct) {
-  std::visit(
-      Fortran::common::visitors{
-          [&](const Fortran::parser::OpenMPDeclarativeAllocate
-                  &declarativeAllocate) {
-            TODO(converter.getCurrentLocation(), "OpenMPDeclarativeAllocate");
-          },
-          [&](const Fortran::parser::OpenMPDeclareReductionConstruct
-                  &declareReductionConstruct) {
-            TODO(converter.getCurrentLocation(),
-                 "OpenMPDeclareReductionConstruct");
-          },
-          [&](const Fortran::parser::OpenMPDeclareSimdConstruct
-                  &declareSimdConstruct) {
-            TODO(converter.getCurrentLocation(), "OpenMPDeclareSimdConstruct");
-          },
-          [&](const Fortran::parser::OpenMPDeclareTargetConstruct
-                  &declareTargetConstruct) {
-            genOMP(converter, symTable, semaCtx, eval, declareTargetConstruct);
-          },
-          [&](const Fortran::parser::OpenMPRequiresConstruct
-                  &requiresConstruct) {
-            // Requires directives are gathered and processed in semantics and
-            // then combined in the lowering bridge before triggering codegen
-            // just once. Hence, there is no need to lower each individual
-            // occurrence here.
-          },
-          [&](const Fortran::parser::OpenMPThreadprivate &threadprivate) {
-            // The directive is lowered when instantiating the variable to
-            // support the case of threadprivate variable declared in module.
-          },
-      },
-      ompDeclConstruct.u);
-}
-
 //===----------------------------------------------------------------------===//
 // Public functions
 //===----------------------------------------------------------------------===//
-- 
GitLab


From 0bdb883e7c6ae836c68c3bd09cbe1305b3f656ec Mon Sep 17 00:00:00 2001
From: Andrzej Warzynski <andrzej.warzynski@arm.com>
Date: Thu, 14 Mar 2024 13:37:13 +0000
Subject: [PATCH 131/404] [mlir][SVE] Add e2e for 1D depthwise WC convolution

Follow-up for #81625

Relands #85225 with a minor update to the RUN line to fix buildbot
failures:
```diff
-// RUN: %{compile} | %{run} | FileCheck %s
+// RUN: rm -f %t && %{compile} && %{run} | FileCheck %s
```

Failing buildbots after landing #85225:
 * https://lab.llvm.org/buildbot/#/builders/184/builds/11363
 * https://lab.llvm.org/buildbot/#/builders/176/builds/9331
---
 .../Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir  | 60 +++++++++++++++++++
 1 file changed, 60 insertions(+)
 create mode 100644 mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir

diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir
new file mode 100644
index 000000000000..56dce1c65401
--- /dev/null
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir
@@ -0,0 +1,60 @@
+// DEFINE: %{compile} =  mlir-opt %s \
+// DEFINE:    -transform-interpreter -test-transform-dialect-erase-schedule \
+// DEFINE:    -one-shot-bufferize="bufferize-function-boundaries" -lower-vector-mask -buffer-deallocation-pipeline -cse -canonicalize -convert-vector-to-scf -arm-sve-legalize-vector-storage \
+// DEFINE:    -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm -o %t
+// DEFINE: %{entry_point} = conv
+// DEFINE: %{run} = %mcr_aarch64_cmd %t -e %{entry_point} -entry-point-result=void --march=aarch64 --mattr="+sve"\
+// DEFINE:    -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils
+
+// RUN: rm -f %t && %{compile} && %{run} | FileCheck %s
+
+func.func @conv() {
+  // Define input/output tensors
+  %input_init = tensor.empty() : tensor<1x8x6xi32>
+  %output_init = tensor.empty() : tensor<1x7x6xi32>
+
+  %five = arith.constant 5 : i32
+  %zero = arith.constant 0 : i32
+  %input = linalg.fill ins(%five : i32) outs(%input_init : tensor<1x8x6xi32>) -> tensor<1x8x6xi32>
+  %output = linalg.fill ins(%zero : i32) outs(%output_init : tensor<1x7x6xi32>) -> tensor<1x7x6xi32>
+
+  // Define the filter tensor
+  %filter = arith.constant dense<[
+    [ 1,  2, 3, 4, 5, 6],
+    [ 11, 12, 13, 14, 15, 16]
+  ]> : tensor<2x6xi32>
+
+  // static sizes -> dynamic sizes
+  %input_dyn = tensor.cast %input_init : tensor<1x8x6xi32> to tensor<1x8x?xi32>
+  %output_dyn = tensor.cast %output : tensor<1x7x6xi32> to tensor<1x7x?xi32>
+  %filter_dyn = tensor.cast %filter : tensor<2x6xi32> to tensor<2x?xi32>
+
+  // Run the convolution
+  %res = linalg.depthwise_conv_1d_nwc_wc
+    ins(%input_dyn, %filter_dyn : tensor<1x8x?xi32>, tensor<2x?xi32>)
+    outs(%output_dyn : tensor<1x7x?xi32>) -> tensor<1x7x?xi32>
+
+  // Print the results
+  // CHECK: SVE: START OF TEST OUTPUT
+  vector.print str "SVE: START OF TEST OUTPUT\n"
+
+  // CHECK-NEXT: Unranked Memref base@ = {{.*}} rank = 3 offset = 0 sizes = [1, 7, 6] strides = [42, 6, 1] data =
+  // CHECK-COUNT-7: [60, 70, 80, 90, 100, 110]
+  %xf = tensor.cast %res : tensor<1x7x?xi32> to tensor<*xi32>
+  call @printMemrefI32(%xf) : (tensor<*xi32>) -> ()
+
+  // CHECK-NEXT: SVE: END OF TEST OUTPUT
+  vector.print str "SVE: END OF TEST OUTPUT\n"
+
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.depthwise_conv_1d_nwc_wc"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+    transform.structured.vectorize %0 vector_sizes [1, 7, [8], 2] : !transform.any_op
+    transform.yield
+  }
+}
+
+func.func private @printMemrefI32(%ptr : tensor<*xi32>)
-- 
GitLab


From fb394562a3cf3522ac17a1436234ef5b926636fb Mon Sep 17 00:00:00 2001
From: Evgenii Kudriashov <evgenii.kudriashov@intel.com>
Date: Mon, 25 Mar 2024 18:46:12 +0300
Subject: [PATCH 132/404] [X86][GlobalISel] Fix referencing nonexistent operand
 in G_ICMP (#86221)

Fixes #86203
---
 .../lib/Target/X86/GISel/X86LegalizerInfo.cpp |  3 +--
 .../X86/GlobalISel/legalize-icmp-vec.mir      | 25 +++++++++++++++++++
 2 files changed, 26 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/X86/GlobalISel/legalize-icmp-vec.mir

diff --git a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
index 06389842ebb1..07041cc5b049 100644
--- a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
+++ b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
@@ -259,8 +259,7 @@ X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
   getActionDefinitionsBuilder(G_ICMP)
       .legalForCartesianProduct({s8}, Is64Bit ? IntTypes64 : IntTypes32)
       .clampScalar(0, s8, s8)
-      .clampScalar(1, s8, sMaxScalar)
-      .scalarSameSizeAs(2, 1);
+      .clampScalar(1, s8, sMaxScalar);
 
   // bswap
   getActionDefinitionsBuilder(G_BSWAP)
diff --git a/llvm/test/CodeGen/X86/GlobalISel/legalize-icmp-vec.mir b/llvm/test/CodeGen/X86/GlobalISel/legalize-icmp-vec.mir
new file mode 100644
index 000000000000..e0fb0fc9d11d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/GlobalISel/legalize-icmp-vec.mir
@@ -0,0 +1,25 @@
+# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=legalizer -global-isel-abort=2 -pass-remarks-missed='gisel*' %s -o - 2>%t | FileCheck %s
+# RUN: FileCheck -check-prefix=ILLEGAL %s < %t
+
+# ILLEGAL: remark: <unknown>:0:0: unable to legalize instruction: %2:_(<4 x s1>) = G_ICMP intpred(sle), %0:_(<4 x s64>), %1:_ (in function: test_icmp_v4i64)
+
+# PR86203
+---
+name: test_icmp_v4i64
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    ; CHECK-LABEL: name: test_icmp_v4i64
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<4 x s1>) = G_ICMP intpred(sle), [[DEF]](<4 x s64>), [[DEF1]]
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(<4 x s32>) = G_ANYEXT [[ICMP]](<4 x s1>)
+    ; CHECK-NEXT: $xmm0 = COPY [[ANYEXT]](<4 x s32>)
+    ; CHECK-NEXT: RET 0, implicit $xmm0
+    %0:_(<4 x s64>) = G_IMPLICIT_DEF
+    %1:_(<4 x s64>) = G_IMPLICIT_DEF
+    %3:_(<4 x s1>) = G_ICMP intpred(sle), %0(<4 x s64>), %1
+    %4:_(<4 x s32>) = G_ANYEXT %3(<4 x s1>)
+    $xmm0 = COPY %4(<4 x s32>)
+    RET 0, implicit $xmm0
+...
-- 
GitLab


From 2b351a3b76b8e58414fb51ff52978c79904ab331 Mon Sep 17 00:00:00 2001
From: Robin Caloudis <robin.caloudis@gmx.de>
Date: Mon, 25 Mar 2024 17:00:43 +0100
Subject: [PATCH 133/404] [HLSL] Remove double pow intrinsics (#86407)

Currently, the HLSL community is re-implementing its HLSL compiler, the
DirectX Shader Compiler (DXC), in LLVM/main piece by piece. This change
removes the pow intrinsics with parameters of type double as it is not
available in DXC.

Introduced in
https://github.com/llvm/llvm-project/commit/df5137e984a607248cd31ed67aa3822e8ac2a083.

Documentation for HLSL pow function is available here:
https://learn.microsoft.com/en-us/windows/win32/direct3dhlsl/dx-graphics-hlsl-pow#parameters

Closes https://github.com/llvm/llvm-project/issues/86181
---
 clang/lib/Headers/hlsl/hlsl_intrinsics.h |  9 ---------
 clang/test/CodeGenHLSL/builtins/pow.hlsl | 13 -------------
 2 files changed, 22 deletions(-)

diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
index 5e703772b7ee..0d6baccd7c0a 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
@@ -1174,15 +1174,6 @@ float3 pow(float3, float3);
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_pow)
 float4 pow(float4, float4);
 
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_pow)
-double pow(double, double);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_pow)
-double2 pow(double2, double2);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_pow)
-double3 pow(double3, double3);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_pow)
-double4 pow(double4, double4);
-
 //===----------------------------------------------------------------------===//
 // reversebits builtins
 //===----------------------------------------------------------------------===//
diff --git a/clang/test/CodeGenHLSL/builtins/pow.hlsl b/clang/test/CodeGenHLSL/builtins/pow.hlsl
index e996ca2f3364..057cd7215aa5 100644
--- a/clang/test/CodeGenHLSL/builtins/pow.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/pow.hlsl
@@ -39,16 +39,3 @@ float3 test_pow_float3(float3 p0, float3 p1) { return pow(p0, p1); }
 // CHECK: define noundef <4 x float> @"?test_pow_float4
 // CHECK: call <4 x float> @llvm.pow.v4f32
 float4 test_pow_float4(float4 p0, float4 p1) { return pow(p0, p1); }
-
-// CHECK: define noundef double @"?test_pow_double@@YANNN@Z"(
-// CHECK: call double @llvm.pow.f64(
-double test_pow_double(double p0, double p1) { return pow(p0, p1); }
-// CHECK: define noundef <2 x double> @"?test_pow_double2@@YAT?$__vector@N$01@__clang@@T12@0@Z"(
-// CHECK: call <2 x double> @llvm.pow.v2f64
-double2 test_pow_double2(double2 p0, double2 p1) { return pow(p0, p1); }
-// CHECK: define noundef <3 x double> @"?test_pow_double3@@YAT?$__vector@N$02@__clang@@T12@0@Z"(
-// CHECK: call <3 x double> @llvm.pow.v3f64
-double3 test_pow_double3(double3 p0, double3 p1) { return pow(p0, p1); }
-// CHECK: define noundef <4 x double> @"?test_pow_double4@@YAT?$__vector@N$03@__clang@@T12@0@Z"(
-// CHECK: call <4 x double> @llvm.pow.v4f64
-double4 test_pow_double4(double4 p0, double4 p1) { return pow(p0, p1); }
-- 
GitLab


From 6fd3677fb4f4e806f2efa43bfa814aaf063c506c Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Mon, 25 Mar 2024 12:01:23 -0400
Subject: [PATCH 134/404] Fixing a think-o with swapped information

---
 clang/www/c_status.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/www/c_status.html b/clang/www/c_status.html
index f00d5a6b7094..dd5fee981f48 100644
--- a/clang/www/c_status.html
+++ b/clang/www/c_status.html
@@ -471,7 +471,7 @@ conformance.</p>
     <tr>
       <td>Constant expressions</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n1365.htm">N1365</a></td>
-      <td class="Clang 16" align="center">Full</td>
+      <td class="full" align="center">Clang 16</td>
     </tr>
     <tr>
       <td>Contractions and expression evaluation methods</td>
-- 
GitLab


From 7ee1007eddcfc4efe582512b78704038f17fa5a3 Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Mon, 25 Mar 2024 12:19:40 -0400
Subject: [PATCH 135/404] [C99] Claim full support for WG14 N772

No tests are added because existing tests cover this. The thrust of the
paper is that an implementation needs to support including a header
file whose name is eight character or less and contains only letters
and numbers; we definitely manage this.
---
 clang/www/c_status.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/www/c_status.html b/clang/www/c_status.html
index dd5fee981f48..e97275e02033 100644
--- a/clang/www/c_status.html
+++ b/clang/www/c_status.html
@@ -369,7 +369,7 @@ conformance.</p>
     <tr>
       <td>relaxed restrictions on portable header names</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n772.htm">N772</a></td>
-      <td class="unknown" align="center">Unknown</td>
+      <td class="full" align="center">Yes</td>
     </tr>
     <tr>
       <td>return without an expression not permitted in function that returns a value</td>
-- 
GitLab


From 002f7808c93ee28db77383de7e802ef3beab47bd Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Mon, 25 Mar 2024 12:36:17 -0400
Subject: [PATCH 136/404] [gn] port fa3d789df15b (tblgen restructure)

---
 .../llvm/unittests/TableGen/BUILD.gn          |  2 +-
 .../gn/secondary/llvm/utils/TableGen/BUILD.gn | 34 ++++---------------
 .../llvm/utils/TableGen/Basic/BUILD.gn        | 10 ++++++
 .../llvm/utils/TableGen/Common/BUILD.gn       | 32 +++++++++++++++++
 .../llvm/utils/TableGen/GlobalISel/BUILD.gn   | 13 -------
 5 files changed, 50 insertions(+), 41 deletions(-)
 create mode 100644 llvm/utils/gn/secondary/llvm/utils/TableGen/Basic/BUILD.gn
 create mode 100644 llvm/utils/gn/secondary/llvm/utils/TableGen/Common/BUILD.gn
 delete mode 100644 llvm/utils/gn/secondary/llvm/utils/TableGen/GlobalISel/BUILD.gn

diff --git a/llvm/utils/gn/secondary/llvm/unittests/TableGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/TableGen/BUILD.gn
index ce04c967c248..a08001d0b319 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/TableGen/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/TableGen/BUILD.gn
@@ -19,7 +19,7 @@ unittest("TableGenTests") {
     ":AutomataTables",
     "//llvm/lib/Support",
     "//llvm/lib/TableGen",
-    "//llvm/utils/TableGen/GlobalISel",
+    "//llvm/utils/TableGen/Common",
   ]
   include_dirs = [ "//llvm/utils/TableGen" ]
   sources = [
diff --git a/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn
index 58335000950d..1c91a775e014 100644
--- a/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn
@@ -1,51 +1,38 @@
-source_set("llvm-min-tblgen-sources") {
+executable("llvm-min-tblgen") {
   sources = [
     "Attributes.cpp",
-    "CodeGenIntrinsics.cpp",
     "DirectiveEmitter.cpp",
     "IntrinsicEmitter.cpp",
     "RISCVTargetDefEmitter.cpp",
-    "SDNodeProperties.cpp",
     "TableGen.cpp",
     "VTEmitter.cpp",
   ]
-  deps = [ "//llvm/lib/Support" ]
-}
-
-executable("llvm-min-tblgen") {
   deps = [
-    ":llvm-min-tblgen-sources",
+    "Basic",
+    "//llvm/lib/Support",
     "//llvm/lib/TableGen",
   ]
 }
 
 executable("llvm-tblgen") {
   deps = [
-    ":llvm-min-tblgen-sources",
+    "Basic",
+    "Common",
     "//llvm/include/llvm/Config:llvm-config",
     "//llvm/lib/CodeGenTypes",
     "//llvm/lib/Support",
     "//llvm/lib/TableGen",
-    "//llvm/utils/TableGen/GlobalISel",
   ]
+  include_dirs = [ "." ]
   sources = [
     "AsmMatcherEmitter.cpp",
     "AsmWriterEmitter.cpp",
-    "AsmWriterInst.cpp",
     "CTagsEmitter.cpp",
     "CallingConvEmitter.cpp",
     "CodeEmitterGen.cpp",
-    "CodeGenDAGPatterns.cpp",
-    "CodeGenHwModes.cpp",
-    "CodeGenInstAlias.cpp",
-    "CodeGenInstruction.cpp",
     "CodeGenMapTable.cpp",
-    "CodeGenRegisters.cpp",
-    "CodeGenSchedule.cpp",
-    "CodeGenTarget.cpp",
     "CompressInstEmitter.cpp",
     "DAGISelEmitter.cpp",
-    "DAGISelMatcher.cpp",
     "DAGISelMatcherEmitter.cpp",
     "DAGISelMatcherGen.cpp",
     "DAGISelMatcherOpt.cpp",
@@ -58,24 +45,17 @@ executable("llvm-tblgen") {
     "FastISelEmitter.cpp",
     "GlobalISelCombinerEmitter.cpp",
     "GlobalISelEmitter.cpp",
-    "GlobalISelMatchTable.cpp",
-    "GlobalISelMatchTableExecutorEmitter.cpp",
-    "InfoByHwMode.cpp",
     "InstrDocsEmitter.cpp",
     "InstrInfoEmitter.cpp",
     "MacroFusionPredicatorEmitter.cpp",
-    "OptEmitter.cpp",
     "OptParserEmitter.cpp",
     "OptRSTEmitter.cpp",
-    "PredicateExpander.cpp",
     "PseudoLoweringEmitter.cpp",
     "RegisterBankEmitter.cpp",
     "RegisterInfoEmitter.cpp",
     "SearchableTableEmitter.cpp",
+    "TableGen.cpp",
     "SubtargetEmitter.cpp",
-    "SubtargetFeatureInfo.cpp",
-    "Types.cpp",
-    "VarLenCodeEmitterGen.cpp",
     "WebAssemblyDisassemblerEmitter.cpp",
     "X86CompressEVEXTablesEmitter.cpp",
     "X86DisassemblerTables.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/utils/TableGen/Basic/BUILD.gn b/llvm/utils/gn/secondary/llvm/utils/TableGen/Basic/BUILD.gn
new file mode 100644
index 000000000000..2ebe393fa0fd
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/utils/TableGen/Basic/BUILD.gn
@@ -0,0 +1,10 @@
+static_library("Basic") {
+  deps = [
+    "//llvm/lib/Support",
+    "//llvm/lib/TableGen",
+  ]
+  sources = [
+    "CodeGenIntrinsics.cpp",
+    "SDNodeProperties.cpp",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/llvm/utils/TableGen/Common/BUILD.gn b/llvm/utils/gn/secondary/llvm/utils/TableGen/Common/BUILD.gn
new file mode 100644
index 000000000000..c0ea62716fd2
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/utils/TableGen/Common/BUILD.gn
@@ -0,0 +1,32 @@
+static_library("Common") {
+  deps = [
+    "//llvm/include/llvm/CodeGen:GenVT",
+    "//llvm/lib/CodeGenTypes",
+    "//llvm/lib/Support",
+    "//llvm/lib/TableGen",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "AsmWriterInst.cpp",
+    "CodeGenDAGPatterns.cpp",
+    "CodeGenHwModes.cpp",
+    "CodeGenInstAlias.cpp",
+    "CodeGenInstruction.cpp",
+    "CodeGenRegisters.cpp",
+    "CodeGenSchedule.cpp",
+    "CodeGenTarget.cpp",
+    "DAGISelMatcher.cpp",
+    "GlobalISel/CXXPredicates.cpp",
+    "GlobalISel/CodeExpander.cpp",
+    "GlobalISel/GlobalISelMatchTable.cpp",
+    "GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp",
+    "GlobalISel/MatchDataInfo.cpp",
+    "GlobalISel/Patterns.cpp",
+    "InfoByHwMode.cpp",
+    "OptEmitter.cpp",
+    "PredicateExpander.cpp",
+    "SubtargetFeatureInfo.cpp",
+    "Types.cpp",
+    "VarLenCodeEmitterGen.cpp",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/llvm/utils/TableGen/GlobalISel/BUILD.gn b/llvm/utils/gn/secondary/llvm/utils/TableGen/GlobalISel/BUILD.gn
deleted file mode 100644
index 8b0ca8561517..000000000000
--- a/llvm/utils/gn/secondary/llvm/utils/TableGen/GlobalISel/BUILD.gn
+++ /dev/null
@@ -1,13 +0,0 @@
-static_library("GlobalISel") {
-  deps = [
-    "//llvm/include/llvm/CodeGen:GenVT",
-    "//llvm/lib/CodeGenTypes",
-    "//llvm/lib/Support",
-  ]
-  sources = [
-    "CXXPredicates.cpp",
-    "CodeExpander.cpp",
-    "MatchDataInfo.cpp",
-    "Patterns.cpp",
-  ]
-}
-- 
GitLab


From 8ef451072a532a9cbc1c128468fba70bbf9a38b9 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Sun, 24 Mar 2024 16:39:02 -0500
Subject: [PATCH 137/404] [JITLink][ELF][AArch64] Make include order consistent
 with other JITLink files.

Coding my way home: 8.98112N, 79.52094W
---
 llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
index 4f64d83162b2..6b03bb3c90b2 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
@@ -11,9 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ExecutionEngine/JITLink/ELF_aarch64.h"
-#include "EHFrameSupportImpl.h"
-#include "ELFLinkGraphBuilder.h"
-#include "JITLinkGeneric.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h"
 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
@@ -21,6 +18,9 @@
 #include "llvm/Support/Endian.h"
 
 #include "DefineExternalSectionStartAndEndSymbols.h"
+#include "EHFrameSupportImpl.h"
+#include "ELFLinkGraphBuilder.h"
+#include "JITLinkGeneric.h"
 
 #define DEBUG_TYPE "jitlink"
 
-- 
GitLab


From c15218bccb570316e4971c4b70e879a4bd6a8de9 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Sun, 24 Mar 2024 21:58:41 -0500
Subject: [PATCH 138/404] [JITLink] Dump weak-ref status for external symbols
 in LinkGraphs.

Coding my way home: 8.98112N, 79.52094W
---
 llvm/lib/ExecutionEngine/JITLink/JITLink.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp
index 7f743dba60a9..b103a9ca98e1 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp
@@ -338,7 +338,8 @@ void LinkGraph::dump(raw_ostream &OS) {
   OS << "\nExternal symbols:\n";
   if (!external_symbols().empty()) {
     for (auto *Sym : external_symbols())
-      OS << "  " << Sym->getAddress() << ": " << *Sym << "\n";
+      OS << "  " << Sym->getAddress() << ": " << *Sym
+         << (Sym->isWeaklyReferenced() ? " (weakly referenced)" : "") << "\n";
   } else
     OS << "  none\n";
 }
-- 
GitLab


From 94bbc18bb6469b29dc471c4af879c2ee64d62180 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Mon, 25 Mar 2024 10:24:32 -0500
Subject: [PATCH 139/404] [ORC] Add some debug logging to
 EPCDynamicLibrarySearchGenerator.

Coding my way home: 8.98112N, 79.52094W
---
 .../Orc/EPCDynamicLibrarySearchGenerator.cpp  | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp b/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp
index 88cc3b04fb64..ec2187bad0f2 100644
--- a/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp
@@ -7,8 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
+#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
 #include "llvm/Support/Error.h"
 
+#define DEBUG_TYPE "orc"
+
 namespace llvm {
 namespace orc {
 
@@ -31,6 +34,11 @@ Error EPCDynamicLibrarySearchGenerator::tryToGenerate(
   if (Symbols.empty())
     return Error::success();
 
+  LLVM_DEBUG({
+      dbgs() << "EPCDynamicLibrarySearchGenerator trying to generate "
+             << Symbols << "\n";
+    });
+
   SymbolLookupSet LookupSymbols;
 
   for (auto &KV : Symbols) {
@@ -44,8 +52,12 @@ Error EPCDynamicLibrarySearchGenerator::tryToGenerate(
   // Copy-capture LookupSymbols, since LookupRequest keeps a reference.
   EPC.lookupSymbolsAsync(Request, [this, &JD, LS = std::move(LS),
                                    LookupSymbols](auto Result) mutable {
-    if (!Result)
+    if (!Result) {
+      LLVM_DEBUG({
+        dbgs() << "EPCDynamicLibrarySearchGenerator lookup failed due to error";
+      });
       return LS.continueLookup(Result.takeError());
+    }
 
     assert(Result->size() == 1 && "Results for more than one library returned");
     assert(Result->front().size() == LookupSymbols.size() &&
@@ -59,6 +71,11 @@ Error EPCDynamicLibrarySearchGenerator::tryToGenerate(
       ++ResultI;
     }
 
+    LLVM_DEBUG({
+      dbgs() << "EPCDynamicLibrarySearchGenerator lookup returned "
+             << NewSymbols << "\n";
+    });
+
     // If there were no resolved symbols bail out.
     if (NewSymbols.empty())
       return LS.continueLookup(Error::success());
-- 
GitLab


From c4d58815506ece1a5e5e001fadbaf34b0266fd37 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Mon, 25 Mar 2024 10:41:24 -0500
Subject: [PATCH 140/404] [JITLink][MachO] Handle MachO section start/end
 symbols.

Transform section$start$<section-name> and section$end$<section-name> external
symbols into defined symbols when a section named <section-name> is present.

rdar://125357048

Coding my way home: 8.98112N, 79.52094W
---
 .../DefineExternalSectionStartAndEndSymbols.h | 23 ++++++++++++++
 .../ExecutionEngine/JITLink/MachO_arm64.cpp   |  6 ++++
 .../ExecutionEngine/JITLink/MachO_x86_64.cpp  |  6 ++++
 .../MachO_section_start_and_stop_symbols.s    | 30 +++++++++++++++++++
 4 files changed, 65 insertions(+)
 create mode 100644 llvm/test/ExecutionEngine/JITLink/AArch64/MachO_section_start_and_stop_symbols.s

diff --git a/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h
index aef441d093a2..4a492ee2f7d0 100644
--- a/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h
+++ b/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h
@@ -127,6 +127,29 @@ identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
   return {};
 }
 
+/// MachO section start/end symbol detection.
+inline SectionRangeSymbolDesc
+identifyMachOSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
+  constexpr StringRef StartSymbolPrefix = "section$start$";
+  constexpr StringRef EndSymbolPrefix = "section$end$";
+
+  auto SymName = Sym.getName();
+  if (SymName.starts_with(StartSymbolPrefix)) {
+    auto [SegName, SecName] =
+      SymName.drop_front(StartSymbolPrefix.size()).split('$');
+    std::string SectionName = (SegName + "," + SecName).str();
+    if (auto *Sec = G.findSectionByName(SectionName))
+      return {*Sec, true};
+  } else if (SymName.starts_with(EndSymbolPrefix)) {
+    auto [SegName, SecName] =
+      SymName.drop_front(EndSymbolPrefix.size()).split('$');
+    std::string SectionName = (SegName + "," + SecName).str();
+    if (auto *Sec = G.findSectionByName(SectionName))
+      return {*Sec, false};
+  }
+  return {};
+}
+
 } // end namespace jitlink
 } // end namespace llvm
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
index 556031baaaed..8733306bab6b 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
@@ -14,6 +14,7 @@
 #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h"
 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
 
+#include "DefineExternalSectionStartAndEndSymbols.h"
 #include "MachOLinkGraphBuilder.h"
 
 #define DEBUG_TYPE "jitlink"
@@ -593,6 +594,11 @@ void link_MachO_arm64(std::unique_ptr<LinkGraph> G,
     Config.PrePrunePasses.push_back(createEHFrameSplitterPass_MachO_arm64());
     Config.PrePrunePasses.push_back(createEHFrameEdgeFixerPass_MachO_arm64());
 
+    // Resolve any external section start / end symbols.
+    Config.PostAllocationPasses.push_back(
+        createDefineExternalSectionStartAndEndSymbolsPass(
+            identifyMachOSectionStartAndEndSymbols));
+
     // Add an in-place GOT/Stubs pass.
     Config.PostPrunePasses.push_back(buildTables_MachO_arm64);
   }
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
index eeca27771ad6..2c69d61316a8 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
@@ -14,6 +14,7 @@
 #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h"
 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
 
+#include "DefineExternalSectionStartAndEndSymbols.h"
 #include "MachOLinkGraphBuilder.h"
 
 #define DEBUG_TYPE "jitlink"
@@ -516,6 +517,11 @@ void link_MachO_x86_64(std::unique_ptr<LinkGraph> G,
     else
       Config.PrePrunePasses.push_back(markAllSymbolsLive);
 
+    // Resolve any external section start / end symbols.
+    Config.PostAllocationPasses.push_back(
+        createDefineExternalSectionStartAndEndSymbolsPass(
+            identifyMachOSectionStartAndEndSymbols));
+
     // Add an in-place GOT/Stubs pass.
     Config.PostPrunePasses.push_back(buildGOTAndStubs_MachO_x86_64);
 
diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_section_start_and_stop_symbols.s b/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_section_start_and_stop_symbols.s
new file mode 100644
index 000000000000..8862dd8a28fa
--- /dev/null
+++ b/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_section_start_and_stop_symbols.s
@@ -0,0 +1,30 @@
+# RUN: llvm-mc -triple=arm64-apple-darwin24 -filetype=obj -o %t.o %s
+# RUN: llvm-jitlink -noexec -check %s %t.o
+
+# jitlink-check: *{8}_z = (*{8}_y) + 4
+
+	.section	__TEXT,__text,regular,pure_instructions
+	.globl	_main
+	.p2align	2
+_main:
+        mov     w0, #0
+        ret
+
+        .section        __DATA,__custom_section
+	.globl	_x
+	.p2align	2, 0x0
+_x:
+	.long	42
+
+	.section	__DATA,__data
+	.globl	_y
+	.p2align	3, 0x0
+_y:
+	.quad	section$start$__DATA$__custom_section
+
+	.globl	_z
+	.p2align	3, 0x0
+_z:
+	.quad	section$end$__DATA$__custom_section
+
+.subsections_via_symbols
-- 
GitLab


From 20392687d38169c7f7de484050e734cb72630b2f Mon Sep 17 00:00:00 2001
From: Andrii Levitskiy <107996072+aabysswalker@users.noreply.github.com>
Date: Mon, 25 Mar 2024 18:42:51 +0200
Subject: [PATCH 141/404] [HLSL] remove double impelementation of log, sin,
 trunc intrinsics (#86440)

This change removes the log, log2, log10, sin, trunc intrinsics with
parameters of type double as it is not available in the DXC compiler.


https://learn.microsoft.com/en-us/windows/win32/direct3dhlsl/dx-graphics-hlsl-log

https://learn.microsoft.com/en-us/windows/win32/direct3dhlsl/dx-graphics-hlsl-log2

https://learn.microsoft.com/en-us/windows/win32/direct3dhlsl/dx-graphics-hlsl-log10

https://learn.microsoft.com/en-us/windows/win32/direct3dhlsl/dx-graphics-hlsl-sin

https://learn.microsoft.com/en-us/windows/win32/direct3dhlsl/dx-graphics-hlsl-trunc

Closes #86189
---
 clang/lib/Headers/hlsl/hlsl_intrinsics.h | 45 ------------------------
 1 file changed, 45 deletions(-)

diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
index 0d6baccd7c0a..d9f27c0db57c 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
@@ -737,15 +737,6 @@ float3 log(float3);
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_log)
 float4 log(float4);
 
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_log)
-double log(double);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_log)
-double2 log(double2);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_log)
-double3 log(double3);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_log)
-double4 log(double4);
-
 //===----------------------------------------------------------------------===//
 // log10 builtins
 //===----------------------------------------------------------------------===//
@@ -779,15 +770,6 @@ float3 log10(float3);
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_log10)
 float4 log10(float4);
 
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_log10)
-double log10(double);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_log10)
-double2 log10(double2);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_log10)
-double3 log10(double3);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_log10)
-double4 log10(double4);
-
 //===----------------------------------------------------------------------===//
 // log2 builtins
 //===----------------------------------------------------------------------===//
@@ -821,15 +803,6 @@ float3 log2(float3);
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_log2)
 float4 log2(float4);
 
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_log2)
-double log2(double);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_log2)
-double2 log2(double2);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_log2)
-double3 log2(double3);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_log2)
-double4 log2(double4);
-
 //===----------------------------------------------------------------------===//
 // mad builtins
 //===----------------------------------------------------------------------===//
@@ -1384,15 +1357,6 @@ float3 sin(float3);
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_sin)
 float4 sin(float4);
 
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_sin)
-double sin(double);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_sin)
-double2 sin(double2);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_sin)
-double3 sin(double3);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_sin)
-double4 sin(double4);
-
 //===----------------------------------------------------------------------===//
 // sqrt builtins
 //===----------------------------------------------------------------------===//
@@ -1441,15 +1405,6 @@ float3 trunc(float3);
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_trunc)
 float4 trunc(float4);
 
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_trunc)
-double trunc(double);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_trunc)
-double2 trunc(double2);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_trunc)
-double3 trunc(double3);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_trunc)
-double4 trunc(double4);
-
 //===----------------------------------------------------------------------===//
 // Wave* builtins
 //===----------------------------------------------------------------------===//
-- 
GitLab


From a83ed0423dcafd4fecb15994a5c5c8b85a56f59b Mon Sep 17 00:00:00 2001
From: Justin Bogner <mail@justinbogner.com>
Date: Mon, 25 Mar 2024 10:49:32 -0600
Subject: [PATCH 142/404] [DirectX] Follow naming conventions for enumerators
 in DXILABI.h. NFC (#86237)

These all-caps names differ from the llvm naming conventions for no good
reason, and `VOID` in all caps can cause problems in windows
environments (see [1]). Rename them to UpperCamelCase.

[1]: https://github.com/clangd/clangd/issues/1983
---
 llvm/include/llvm/Support/DXILABI.h       | 18 ++++----
 llvm/lib/Target/DirectX/DXILOpBuilder.cpp | 18 ++++----
 llvm/utils/TableGen/DXILEmitter.cpp       | 50 +++++++++++------------
 3 files changed, 43 insertions(+), 43 deletions(-)

diff --git a/llvm/include/llvm/Support/DXILABI.h b/llvm/include/llvm/Support/DXILABI.h
index c1d81775b671..da4bea8fc46e 100644
--- a/llvm/include/llvm/Support/DXILABI.h
+++ b/llvm/include/llvm/Support/DXILABI.h
@@ -23,20 +23,20 @@ namespace llvm {
 namespace dxil {
 
 enum class ParameterKind : uint8_t {
-  INVALID = 0,
-  VOID,
-  HALF,
-  FLOAT,
-  DOUBLE,
+  Invalid = 0,
+  Void,
+  Half,
+  Float,
+  Double,
   I1,
   I8,
   I16,
   I32,
   I64,
-  OVERLOAD,
-  CBUFFER_RET,
-  RESOURCE_RET,
-  DXIL_HANDLE,
+  Overload,
+  CBufferRet,
+  ResourceRet,
+  DXILHandle,
 };
 
 /// The kind of resource for an SRV or UAV resource. Sometimes referred to as
diff --git a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
index a1eacc2d4800..0841ae95423c 100644
--- a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
@@ -190,13 +190,13 @@ static StructType *getHandleType(LLVMContext &Ctx) {
 static Type *getTypeFromParameterKind(ParameterKind Kind, Type *OverloadTy) {
   auto &Ctx = OverloadTy->getContext();
   switch (Kind) {
-  case ParameterKind::VOID:
+  case ParameterKind::Void:
     return Type::getVoidTy(Ctx);
-  case ParameterKind::HALF:
+  case ParameterKind::Half:
     return Type::getHalfTy(Ctx);
-  case ParameterKind::FLOAT:
+  case ParameterKind::Float:
     return Type::getFloatTy(Ctx);
-  case ParameterKind::DOUBLE:
+  case ParameterKind::Double:
     return Type::getDoubleTy(Ctx);
   case ParameterKind::I1:
     return Type::getInt1Ty(Ctx);
@@ -208,11 +208,11 @@ static Type *getTypeFromParameterKind(ParameterKind Kind, Type *OverloadTy) {
     return Type::getInt32Ty(Ctx);
   case ParameterKind::I64:
     return Type::getInt64Ty(Ctx);
-  case ParameterKind::OVERLOAD:
+  case ParameterKind::Overload:
     return OverloadTy;
-  case ParameterKind::RESOURCE_RET:
+  case ParameterKind::ResourceRet:
     return getResRetType(OverloadTy, Ctx);
-  case ParameterKind::DXIL_HANDLE:
+  case ParameterKind::DXILHandle:
     return getHandleType(Ctx);
   default:
     break;
@@ -320,8 +320,8 @@ Type *DXILOpBuilder::getOverloadTy(dxil::OpCode OpCode, FunctionType *FT) {
   auto ParamKinds = getOpCodeParameterKind(*Prop);
   auto Kind = ParamKinds[Prop->OverloadParamIndex];
   // For ResRet and CBufferRet, OverloadTy is in field of StructType.
-  if (Kind == ParameterKind::CBUFFER_RET ||
-      Kind == ParameterKind::RESOURCE_RET) {
+  if (Kind == ParameterKind::CBufferRet ||
+      Kind == ParameterKind::ResourceRet) {
     auto *ST = cast<StructType>(OverloadType);
     OverloadType = ST->getElementType(0);
   }
diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp
index 47c89df35e19..f2504775d557 100644
--- a/llvm/utils/TableGen/DXILEmitter.cpp
+++ b/llvm/utils/TableGen/DXILEmitter.cpp
@@ -74,13 +74,13 @@ static ParameterKind getParameterKind(const Record *R) {
   auto VTRec = R->getValueAsDef("VT");
   switch (getValueType(VTRec)) {
   case MVT::isVoid:
-    return ParameterKind::VOID;
+    return ParameterKind::Void;
   case MVT::f16:
-    return ParameterKind::HALF;
+    return ParameterKind::Half;
   case MVT::f32:
-    return ParameterKind::FLOAT;
+    return ParameterKind::Float;
   case MVT::f64:
-    return ParameterKind::DOUBLE;
+    return ParameterKind::Double;
   case MVT::i1:
     return ParameterKind::I1;
   case MVT::i8:
@@ -91,11 +91,11 @@ static ParameterKind getParameterKind(const Record *R) {
     return ParameterKind::I32;
   case MVT::fAny:
   case MVT::iAny:
-    return ParameterKind::OVERLOAD;
+    return ParameterKind::Overload;
   case MVT::Other:
     // Handle DXIL-specific overload types
     if (R->getValueAsInt("isHalfOrFloat") || R->getValueAsInt("isI16OrI32")) {
-      return ParameterKind::OVERLOAD;
+      return ParameterKind::Overload;
     }
     LLVM_FALLTHROUGH;
   default:
@@ -201,16 +201,16 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
 /// \return std::string string representation of input Kind
 static std::string getParameterKindStr(ParameterKind Kind) {
   switch (Kind) {
-  case ParameterKind::INVALID:
-    return "INVALID";
-  case ParameterKind::VOID:
-    return "VOID";
-  case ParameterKind::HALF:
-    return "HALF";
-  case ParameterKind::FLOAT:
-    return "FLOAT";
-  case ParameterKind::DOUBLE:
-    return "DOUBLE";
+  case ParameterKind::Invalid:
+    return "Invalid";
+  case ParameterKind::Void:
+    return "Void";
+  case ParameterKind::Half:
+    return "Half";
+  case ParameterKind::Float:
+    return "Float";
+  case ParameterKind::Double:
+    return "Double";
   case ParameterKind::I1:
     return "I1";
   case ParameterKind::I8:
@@ -221,14 +221,14 @@ static std::string getParameterKindStr(ParameterKind Kind) {
     return "I32";
   case ParameterKind::I64:
     return "I64";
-  case ParameterKind::OVERLOAD:
-    return "OVERLOAD";
-  case ParameterKind::CBUFFER_RET:
-    return "CBUFFER_RET";
-  case ParameterKind::RESOURCE_RET:
-    return "RESOURCE_RET";
-  case ParameterKind::DXIL_HANDLE:
-    return "DXIL_HANDLE";
+  case ParameterKind::Overload:
+    return "Overload";
+  case ParameterKind::CBufferRet:
+    return "CBufferRet";
+  case ParameterKind::ResourceRet:
+    return "ResourceRet";
+  case ParameterKind::DXILHandle:
+    return "DXILHandle";
   }
   llvm_unreachable("Unknown llvm::dxil::ParameterKind enum");
 }
@@ -462,7 +462,7 @@ static void emitDXILOperationTable(std::vector<DXILOperationDesc> &Ops,
       [](raw_ostream &ParamOS, ParameterKind Kind) {
         ParamOS << "ParameterKind::" << getParameterKindStr(Kind);
       },
-      "ParameterKind::INVALID");
+      "ParameterKind::Invalid");
   OS << "  };\n\n";
   OS << "  unsigned Index = Prop.ParameterTableOffset;\n";
   OS << "  return DXILOpParameterKindTable + Index;\n";
-- 
GitLab


From 7b3e943dfc6355488a57a7feaa125f4879ba7537 Mon Sep 17 00:00:00 2001
From: AtariDreams <83477269+AtariDreams@users.noreply.github.com>
Date: Mon, 25 Mar 2024 12:53:38 -0400
Subject: [PATCH 143/404] Reapply: [Float2Int] Resolve FIXME: Pick the smallest
 legal type that fits (#86337)

Originally reverted because of a bug in Range that is now fixed
(#86041), we can reland this commit. Tests have been added to ensure the
miscompile that caused the revert does not happen again.
---
 .../llvm/Transforms/Scalar/Float2Int.h        |   2 +-
 llvm/lib/Transforms/Scalar/Float2Int.cpp      |  29 +-
 llvm/test/Transforms/Float2Int/basic.ll       | 317 ++++++++++++++----
 llvm/test/Transforms/Float2Int/pr79158.ll     |  55 ++-
 4 files changed, 334 insertions(+), 69 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Scalar/Float2Int.h b/llvm/include/llvm/Transforms/Scalar/Float2Int.h
index 83be329bed60..337e229efcf3 100644
--- a/llvm/include/llvm/Transforms/Scalar/Float2Int.h
+++ b/llvm/include/llvm/Transforms/Scalar/Float2Int.h
@@ -44,7 +44,7 @@ private:
   std::optional<ConstantRange> calcRange(Instruction *I);
   void walkBackwards();
   void walkForwards();
-  bool validateAndTransform();
+  bool validateAndTransform(const DataLayout &DL);
   Value *convert(Instruction *I, Type *ToTy);
   void cleanup();
 
diff --git a/llvm/lib/Transforms/Scalar/Float2Int.cpp b/llvm/lib/Transforms/Scalar/Float2Int.cpp
index de8c05d5689f..da4d39b4e3ed 100644
--- a/llvm/lib/Transforms/Scalar/Float2Int.cpp
+++ b/llvm/lib/Transforms/Scalar/Float2Int.cpp
@@ -311,7 +311,7 @@ void Float2IntPass::walkForwards() {
 }
 
 // If there is a valid transform to be done, do it.
-bool Float2IntPass::validateAndTransform() {
+bool Float2IntPass::validateAndTransform(const DataLayout &DL) {
   bool MadeChange = false;
 
   // Iterate over every disjoint partition of the def-use graph.
@@ -374,15 +374,23 @@ bool Float2IntPass::validateAndTransform() {
       LLVM_DEBUG(dbgs() << "F2I: Value not guaranteed to be representable!\n");
       continue;
     }
-    if (MinBW > 64) {
-      LLVM_DEBUG(
-          dbgs() << "F2I: Value requires more than 64 bits to represent!\n");
-      continue;
-    }
 
-    // OK, R is known to be representable. Now pick a type for it.
-    // FIXME: Pick the smallest legal type that will fit.
-    Type *Ty = (MinBW > 32) ? Type::getInt64Ty(*Ctx) : Type::getInt32Ty(*Ctx);
+    // OK, R is known to be representable.
+    // Pick the smallest legal type that will fit.
+    Type *Ty = DL.getSmallestLegalIntType(*Ctx, MinBW);
+    if (!Ty) {
+      // Every supported target supports 64-bit and 32-bit integers,
+      // so fallback to a 32 or 64-bit integer if the value fits.
+      if (MinBW <= 32) {
+        Ty = Type::getInt32Ty(*Ctx);
+      } else if (MinBW <= 64) {
+        Ty = Type::getInt64Ty(*Ctx);
+      } else {
+        LLVM_DEBUG(dbgs() << "F2I: Value requires more bits to represent than "
+                             "the target supports!\n");
+        continue;
+      }
+    }
 
     for (auto MI = ECs.member_begin(It), ME = ECs.member_end();
          MI != ME; ++MI)
@@ -489,7 +497,8 @@ bool Float2IntPass::runImpl(Function &F, const DominatorTree &DT) {
   walkBackwards();
   walkForwards();
 
-  bool Modified = validateAndTransform();
+  const DataLayout &DL = F.getParent()->getDataLayout();
+  bool Modified = validateAndTransform(DL);
   if (Modified)
     cleanup();
   return Modified;
diff --git a/llvm/test/Transforms/Float2Int/basic.ll b/llvm/test/Transforms/Float2Int/basic.ll
index 2854a83179b7..32f5ca2a053c 100644
--- a/llvm/test/Transforms/Float2Int/basic.ll
+++ b/llvm/test/Transforms/Float2Int/basic.ll
@@ -1,16 +1,35 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes='float2int' -S | FileCheck %s
+; RUN: opt < %s -passes='float2int' -S | FileCheck %s -check-prefixes=CHECK,NONE
+; RUN: opt < %s -passes='float2int' -S --data-layout="n64" | FileCheck %s -check-prefixes=CHECK,ONLY64
+; RUN: opt < %s -passes='float2int' -S --data-layout="n8:16:32:64"| FileCheck %s -check-prefixes=CHECK,MULTIPLE
+; RUN: opt < %s -passes=float2int -S --data-layout="e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"| FileCheck %s -check-prefixes=CHECK,PR-79158
 
 ;
 ; Positive tests
 ;
 
 define i16 @simple1(i8 %a) {
-; CHECK-LABEL: @simple1(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
-; CHECK-NEXT:    [[T21:%.*]] = add i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[T21]] to i16
-; CHECK-NEXT:    ret i16 [[TMP2]]
+; NONE-LABEL: @simple1(
+; NONE-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; NONE-NEXT:    [[T21:%.*]] = add i32 [[TMP1]], 1
+; NONE-NEXT:    [[TMP2:%.*]] = trunc i32 [[T21]] to i16
+; NONE-NEXT:    ret i16 [[TMP2]]
+;
+; ONLY64-LABEL: @simple1(
+; ONLY64-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i64
+; ONLY64-NEXT:    [[T21:%.*]] = add i64 [[TMP1]], 1
+; ONLY64-NEXT:    [[TMP2:%.*]] = trunc i64 [[T21]] to i16
+; ONLY64-NEXT:    ret i16 [[TMP2]]
+;
+; MULTIPLE-LABEL: @simple1(
+; MULTIPLE-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16
+; MULTIPLE-NEXT:    [[T21:%.*]] = add i16 [[TMP1]], 1
+; MULTIPLE-NEXT:    ret i16 [[T21]]
+;
+; PR-79158-LABEL: @simple1(
+; PR-79158-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16
+; PR-79158-NEXT:    [[T21:%.*]] = add i16 [[TMP1]], 1
+; PR-79158-NEXT:    ret i16 [[T21]]
 ;
   %t1 = uitofp i8 %a to float
   %t2 = fadd float %t1, 1.0
@@ -19,11 +38,29 @@ define i16 @simple1(i8 %a) {
 }
 
 define i8 @simple2(i8 %a) {
-; CHECK-LABEL: @simple2(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
-; CHECK-NEXT:    [[T21:%.*]] = sub i32 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[T21]] to i8
-; CHECK-NEXT:    ret i8 [[TMP2]]
+; NONE-LABEL: @simple2(
+; NONE-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; NONE-NEXT:    [[T21:%.*]] = sub i32 [[TMP1]], 1
+; NONE-NEXT:    [[TMP2:%.*]] = trunc i32 [[T21]] to i8
+; NONE-NEXT:    ret i8 [[TMP2]]
+;
+; ONLY64-LABEL: @simple2(
+; ONLY64-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i64
+; ONLY64-NEXT:    [[T21:%.*]] = sub i64 [[TMP1]], 1
+; ONLY64-NEXT:    [[TMP2:%.*]] = trunc i64 [[T21]] to i8
+; ONLY64-NEXT:    ret i8 [[TMP2]]
+;
+; MULTIPLE-LABEL: @simple2(
+; MULTIPLE-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16
+; MULTIPLE-NEXT:    [[T21:%.*]] = sub i16 [[TMP1]], 1
+; MULTIPLE-NEXT:    [[TMP2:%.*]] = trunc i16 [[T21]] to i8
+; MULTIPLE-NEXT:    ret i8 [[TMP2]]
+;
+; PR-79158-LABEL: @simple2(
+; PR-79158-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16
+; PR-79158-NEXT:    [[T21:%.*]] = sub i16 [[TMP1]], 1
+; PR-79158-NEXT:    [[TMP2:%.*]] = trunc i16 [[T21]] to i8
+; PR-79158-NEXT:    ret i8 [[TMP2]]
 ;
   %t1 = uitofp i8 %a to float
   %t2 = fsub float %t1, 1.0
@@ -32,10 +69,28 @@ define i8 @simple2(i8 %a) {
 }
 
 define i32 @simple3(i8 %a) {
-; CHECK-LABEL: @simple3(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
-; CHECK-NEXT:    [[T21:%.*]] = sub i32 [[TMP1]], 1
-; CHECK-NEXT:    ret i32 [[T21]]
+; NONE-LABEL: @simple3(
+; NONE-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; NONE-NEXT:    [[T21:%.*]] = sub i32 [[TMP1]], 1
+; NONE-NEXT:    ret i32 [[T21]]
+;
+; ONLY64-LABEL: @simple3(
+; ONLY64-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i64
+; ONLY64-NEXT:    [[T21:%.*]] = sub i64 [[TMP1]], 1
+; ONLY64-NEXT:    [[TMP2:%.*]] = trunc i64 [[T21]] to i32
+; ONLY64-NEXT:    ret i32 [[TMP2]]
+;
+; MULTIPLE-LABEL: @simple3(
+; MULTIPLE-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16
+; MULTIPLE-NEXT:    [[T21:%.*]] = sub i16 [[TMP1]], 1
+; MULTIPLE-NEXT:    [[TMP2:%.*]] = zext i16 [[T21]] to i32
+; MULTIPLE-NEXT:    ret i32 [[TMP2]]
+;
+; PR-79158-LABEL: @simple3(
+; PR-79158-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16
+; PR-79158-NEXT:    [[T21:%.*]] = sub i16 [[TMP1]], 1
+; PR-79158-NEXT:    [[TMP2:%.*]] = zext i16 [[T21]] to i32
+; PR-79158-NEXT:    ret i32 [[TMP2]]
 ;
   %t1 = uitofp i8 %a to float
   %t2 = fsub float %t1, 1.0
@@ -44,11 +99,29 @@ define i32 @simple3(i8 %a) {
 }
 
 define i1 @cmp(i8 %a, i8 %b) {
-; CHECK-LABEL: @cmp(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
-; CHECK-NEXT:    [[T31:%.*]] = icmp slt i32 [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    ret i1 [[T31]]
+; NONE-LABEL: @cmp(
+; NONE-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; NONE-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
+; NONE-NEXT:    [[T31:%.*]] = icmp slt i32 [[TMP1]], [[TMP2]]
+; NONE-NEXT:    ret i1 [[T31]]
+;
+; ONLY64-LABEL: @cmp(
+; ONLY64-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i64
+; ONLY64-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i64
+; ONLY64-NEXT:    [[T31:%.*]] = icmp slt i64 [[TMP1]], [[TMP2]]
+; ONLY64-NEXT:    ret i1 [[T31]]
+;
+; MULTIPLE-LABEL: @cmp(
+; MULTIPLE-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16
+; MULTIPLE-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i16
+; MULTIPLE-NEXT:    [[T31:%.*]] = icmp slt i16 [[TMP1]], [[TMP2]]
+; MULTIPLE-NEXT:    ret i1 [[T31]]
+;
+; PR-79158-LABEL: @cmp(
+; PR-79158-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16
+; PR-79158-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i16
+; PR-79158-NEXT:    [[T31:%.*]] = icmp slt i16 [[TMP1]], [[TMP2]]
+; PR-79158-NEXT:    ret i1 [[T31]]
 ;
   %t1 = uitofp i8 %a to float
   %t2 = uitofp i8 %b to float
@@ -70,12 +143,34 @@ define i32 @simple4(i32 %a) {
 }
 
 define i32 @simple5(i8 %a, i8 %b) {
-; CHECK-LABEL: @simple5(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
-; CHECK-NEXT:    [[T31:%.*]] = add i32 [[TMP1]], 1
-; CHECK-NEXT:    [[T42:%.*]] = mul i32 [[T31]], [[TMP2]]
-; CHECK-NEXT:    ret i32 [[T42]]
+; NONE-LABEL: @simple5(
+; NONE-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; NONE-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
+; NONE-NEXT:    [[T31:%.*]] = add i32 [[TMP1]], 1
+; NONE-NEXT:    [[T42:%.*]] = mul i32 [[T31]], [[TMP2]]
+; NONE-NEXT:    ret i32 [[T42]]
+;
+; ONLY64-LABEL: @simple5(
+; ONLY64-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i64
+; ONLY64-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i64
+; ONLY64-NEXT:    [[T31:%.*]] = add i64 [[TMP1]], 1
+; ONLY64-NEXT:    [[T42:%.*]] = mul i64 [[T31]], [[TMP2]]
+; ONLY64-NEXT:    [[TMP3:%.*]] = trunc i64 [[T42]] to i32
+; ONLY64-NEXT:    ret i32 [[TMP3]]
+;
+; MULTIPLE-LABEL: @simple5(
+; MULTIPLE-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; MULTIPLE-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
+; MULTIPLE-NEXT:    [[T31:%.*]] = add i32 [[TMP1]], 1
+; MULTIPLE-NEXT:    [[T42:%.*]] = mul i32 [[T31]], [[TMP2]]
+; MULTIPLE-NEXT:    ret i32 [[T42]]
+;
+; PR-79158-LABEL: @simple5(
+; PR-79158-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; PR-79158-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
+; PR-79158-NEXT:    [[T31:%.*]] = add i32 [[TMP1]], 1
+; PR-79158-NEXT:    [[T42:%.*]] = mul i32 [[T31]], [[TMP2]]
+; PR-79158-NEXT:    ret i32 [[T42]]
 ;
   %t1 = uitofp i8 %a to float
   %t2 = uitofp i8 %b to float
@@ -86,12 +181,34 @@ define i32 @simple5(i8 %a, i8 %b) {
 }
 
 define i32 @simple6(i8 %a, i8 %b) {
-; CHECK-LABEL: @simple6(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
-; CHECK-NEXT:    [[T31:%.*]] = sub i32 0, [[TMP1]]
-; CHECK-NEXT:    [[T42:%.*]] = mul i32 [[T31]], [[TMP2]]
-; CHECK-NEXT:    ret i32 [[T42]]
+; NONE-LABEL: @simple6(
+; NONE-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; NONE-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
+; NONE-NEXT:    [[T31:%.*]] = sub i32 0, [[TMP1]]
+; NONE-NEXT:    [[T42:%.*]] = mul i32 [[T31]], [[TMP2]]
+; NONE-NEXT:    ret i32 [[T42]]
+;
+; ONLY64-LABEL: @simple6(
+; ONLY64-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i64
+; ONLY64-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i64
+; ONLY64-NEXT:    [[T31:%.*]] = sub i64 0, [[TMP1]]
+; ONLY64-NEXT:    [[T42:%.*]] = mul i64 [[T31]], [[TMP2]]
+; ONLY64-NEXT:    [[TMP3:%.*]] = trunc i64 [[T42]] to i32
+; ONLY64-NEXT:    ret i32 [[TMP3]]
+;
+; MULTIPLE-LABEL: @simple6(
+; MULTIPLE-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; MULTIPLE-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
+; MULTIPLE-NEXT:    [[T31:%.*]] = sub i32 0, [[TMP1]]
+; MULTIPLE-NEXT:    [[T42:%.*]] = mul i32 [[T31]], [[TMP2]]
+; MULTIPLE-NEXT:    ret i32 [[T42]]
+;
+; PR-79158-LABEL: @simple6(
+; PR-79158-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; PR-79158-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
+; PR-79158-NEXT:    [[T31:%.*]] = sub i32 0, [[TMP1]]
+; PR-79158-NEXT:    [[T42:%.*]] = mul i32 [[T31]], [[TMP2]]
+; PR-79158-NEXT:    ret i32 [[T42]]
 ;
   %t1 = uitofp i8 %a to float
   %t2 = uitofp i8 %b to float
@@ -105,15 +222,48 @@ define i32 @simple6(i8 %a, i8 %b) {
 ; cause failure of the other.
 
 define i32 @multi1(i8 %a, i8 %b, i8 %c, float %d) {
-; CHECK-LABEL: @multi1(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
-; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
-; CHECK-NEXT:    [[FC:%.*]] = uitofp i8 [[C:%.*]] to float
-; CHECK-NEXT:    [[X1:%.*]] = add i32 [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[Z:%.*]] = fadd float [[FC]], [[D:%.*]]
-; CHECK-NEXT:    [[W:%.*]] = fptoui float [[Z]] to i32
-; CHECK-NEXT:    [[R:%.*]] = add i32 [[X1]], [[W]]
-; CHECK-NEXT:    ret i32 [[R]]
+; NONE-LABEL: @multi1(
+; NONE-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; NONE-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
+; NONE-NEXT:    [[FC:%.*]] = uitofp i8 [[C:%.*]] to float
+; NONE-NEXT:    [[X1:%.*]] = add i32 [[TMP1]], [[TMP2]]
+; NONE-NEXT:    [[Z:%.*]] = fadd float [[FC]], [[D:%.*]]
+; NONE-NEXT:    [[W:%.*]] = fptoui float [[Z]] to i32
+; NONE-NEXT:    [[R:%.*]] = add i32 [[X1]], [[W]]
+; NONE-NEXT:    ret i32 [[R]]
+;
+; ONLY64-LABEL: @multi1(
+; ONLY64-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i64
+; ONLY64-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i64
+; ONLY64-NEXT:    [[FC:%.*]] = uitofp i8 [[C:%.*]] to float
+; ONLY64-NEXT:    [[X1:%.*]] = add i64 [[TMP1]], [[TMP2]]
+; ONLY64-NEXT:    [[TMP3:%.*]] = trunc i64 [[X1]] to i32
+; ONLY64-NEXT:    [[Z:%.*]] = fadd float [[FC]], [[D:%.*]]
+; ONLY64-NEXT:    [[W:%.*]] = fptoui float [[Z]] to i32
+; ONLY64-NEXT:    [[R:%.*]] = add i32 [[TMP3]], [[W]]
+; ONLY64-NEXT:    ret i32 [[R]]
+;
+; MULTIPLE-LABEL: @multi1(
+; MULTIPLE-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16
+; MULTIPLE-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i16
+; MULTIPLE-NEXT:    [[FC:%.*]] = uitofp i8 [[C:%.*]] to float
+; MULTIPLE-NEXT:    [[X1:%.*]] = add i16 [[TMP1]], [[TMP2]]
+; MULTIPLE-NEXT:    [[TMP3:%.*]] = zext i16 [[X1]] to i32
+; MULTIPLE-NEXT:    [[Z:%.*]] = fadd float [[FC]], [[D:%.*]]
+; MULTIPLE-NEXT:    [[W:%.*]] = fptoui float [[Z]] to i32
+; MULTIPLE-NEXT:    [[R:%.*]] = add i32 [[TMP3]], [[W]]
+; MULTIPLE-NEXT:    ret i32 [[R]]
+;
+; PR-79158-LABEL: @multi1(
+; PR-79158-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16
+; PR-79158-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i16
+; PR-79158-NEXT:    [[FC:%.*]] = uitofp i8 [[C:%.*]] to float
+; PR-79158-NEXT:    [[X1:%.*]] = add i16 [[TMP1]], [[TMP2]]
+; PR-79158-NEXT:    [[TMP3:%.*]] = zext i16 [[X1]] to i32
+; PR-79158-NEXT:    [[Z:%.*]] = fadd float [[FC]], [[D:%.*]]
+; PR-79158-NEXT:    [[W:%.*]] = fptoui float [[Z]] to i32
+; PR-79158-NEXT:    [[R:%.*]] = add i32 [[TMP3]], [[W]]
+; PR-79158-NEXT:    ret i32 [[R]]
 ;
   %fa = uitofp i8 %a to float
   %fb = uitofp i8 %b to float
@@ -127,11 +277,27 @@ define i32 @multi1(i8 %a, i8 %b, i8 %c, float %d) {
 }
 
 define i16 @simple_negzero(i8 %a) {
-; CHECK-LABEL: @simple_negzero(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
-; CHECK-NEXT:    [[T21:%.*]] = add i32 [[TMP1]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[T21]] to i16
-; CHECK-NEXT:    ret i16 [[TMP2]]
+; NONE-LABEL: @simple_negzero(
+; NONE-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; NONE-NEXT:    [[T21:%.*]] = add i32 [[TMP1]], 0
+; NONE-NEXT:    [[TMP2:%.*]] = trunc i32 [[T21]] to i16
+; NONE-NEXT:    ret i16 [[TMP2]]
+;
+; ONLY64-LABEL: @simple_negzero(
+; ONLY64-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i64
+; ONLY64-NEXT:    [[T21:%.*]] = add i64 [[TMP1]], 0
+; ONLY64-NEXT:    [[TMP2:%.*]] = trunc i64 [[T21]] to i16
+; ONLY64-NEXT:    ret i16 [[TMP2]]
+;
+; MULTIPLE-LABEL: @simple_negzero(
+; MULTIPLE-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16
+; MULTIPLE-NEXT:    [[T21:%.*]] = add i16 [[TMP1]], 0
+; MULTIPLE-NEXT:    ret i16 [[T21]]
+;
+; PR-79158-LABEL: @simple_negzero(
+; PR-79158-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16
+; PR-79158-NEXT:    [[T21:%.*]] = add i16 [[TMP1]], 0
+; PR-79158-NEXT:    ret i16 [[T21]]
 ;
   %t1 = uitofp i8 %a to float
   %t2 = fadd fast float %t1, -0.0
@@ -140,12 +306,33 @@ define i16 @simple_negzero(i8 %a) {
 }
 
 define i32 @simple_negative(i8 %call) {
-; CHECK-LABEL: @simple_negative(
-; CHECK-NEXT:    [[TMP1:%.*]] = sext i8 [[CALL:%.*]] to i32
-; CHECK-NEXT:    [[MUL1:%.*]] = mul i32 [[TMP1]], -3
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[MUL1]] to i8
-; CHECK-NEXT:    [[CONV3:%.*]] = sext i8 [[TMP2]] to i32
-; CHECK-NEXT:    ret i32 [[CONV3]]
+; NONE-LABEL: @simple_negative(
+; NONE-NEXT:    [[TMP1:%.*]] = sext i8 [[CALL:%.*]] to i32
+; NONE-NEXT:    [[MUL1:%.*]] = mul i32 [[TMP1]], -3
+; NONE-NEXT:    [[TMP2:%.*]] = trunc i32 [[MUL1]] to i8
+; NONE-NEXT:    [[CONV3:%.*]] = sext i8 [[TMP2]] to i32
+; NONE-NEXT:    ret i32 [[CONV3]]
+;
+; ONLY64-LABEL: @simple_negative(
+; ONLY64-NEXT:    [[TMP1:%.*]] = sext i8 [[CALL:%.*]] to i64
+; ONLY64-NEXT:    [[MUL1:%.*]] = mul i64 [[TMP1]], -3
+; ONLY64-NEXT:    [[TMP2:%.*]] = trunc i64 [[MUL1]] to i8
+; ONLY64-NEXT:    [[CONV3:%.*]] = sext i8 [[TMP2]] to i32
+; ONLY64-NEXT:    ret i32 [[CONV3]]
+;
+; MULTIPLE-LABEL: @simple_negative(
+; MULTIPLE-NEXT:    [[TMP1:%.*]] = sext i8 [[CALL:%.*]] to i16
+; MULTIPLE-NEXT:    [[MUL1:%.*]] = mul i16 [[TMP1]], -3
+; MULTIPLE-NEXT:    [[TMP2:%.*]] = trunc i16 [[MUL1]] to i8
+; MULTIPLE-NEXT:    [[CONV3:%.*]] = sext i8 [[TMP2]] to i32
+; MULTIPLE-NEXT:    ret i32 [[CONV3]]
+;
+; PR-79158-LABEL: @simple_negative(
+; PR-79158-NEXT:    [[TMP1:%.*]] = sext i8 [[CALL:%.*]] to i16
+; PR-79158-NEXT:    [[MUL1:%.*]] = mul i16 [[TMP1]], -3
+; PR-79158-NEXT:    [[TMP2:%.*]] = trunc i16 [[MUL1]] to i8
+; PR-79158-NEXT:    [[CONV3:%.*]] = sext i8 [[TMP2]] to i32
+; PR-79158-NEXT:    ret i32 [[CONV3]]
 ;
   %conv1 = sitofp i8 %call to float
   %mul = fmul float %conv1, -3.000000e+00
@@ -155,11 +342,27 @@ define i32 @simple_negative(i8 %call) {
 }
 
 define i16 @simple_fneg(i8 %a) {
-; CHECK-LABEL: @simple_fneg(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
-; CHECK-NEXT:    [[T21:%.*]] = sub i32 0, [[TMP1]]
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[T21]] to i16
-; CHECK-NEXT:    ret i16 [[TMP2]]
+; NONE-LABEL: @simple_fneg(
+; NONE-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; NONE-NEXT:    [[T21:%.*]] = sub i32 0, [[TMP1]]
+; NONE-NEXT:    [[TMP2:%.*]] = trunc i32 [[T21]] to i16
+; NONE-NEXT:    ret i16 [[TMP2]]
+;
+; ONLY64-LABEL: @simple_fneg(
+; ONLY64-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i64
+; ONLY64-NEXT:    [[T21:%.*]] = sub i64 0, [[TMP1]]
+; ONLY64-NEXT:    [[TMP2:%.*]] = trunc i64 [[T21]] to i16
+; ONLY64-NEXT:    ret i16 [[TMP2]]
+;
+; MULTIPLE-LABEL: @simple_fneg(
+; MULTIPLE-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16
+; MULTIPLE-NEXT:    [[T21:%.*]] = sub i16 0, [[TMP1]]
+; MULTIPLE-NEXT:    ret i16 [[T21]]
+;
+; PR-79158-LABEL: @simple_fneg(
+; PR-79158-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16
+; PR-79158-NEXT:    [[T21:%.*]] = sub i16 0, [[TMP1]]
+; PR-79158-NEXT:    ret i16 [[T21]]
 ;
   %t1 = uitofp i8 %a to float
   %t2 = fneg fast float %t1
diff --git a/llvm/test/Transforms/Float2Int/pr79158.ll b/llvm/test/Transforms/Float2Int/pr79158.ll
index 5e78cc0bc66f..639a8ac9934f 100644
--- a/llvm/test/Transforms/Float2Int/pr79158.ll
+++ b/llvm/test/Transforms/Float2Int/pr79158.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt < %s -passes=float2int -S | FileCheck %s
+; RUN: opt < %s -passes=float2int -S | FileCheck %s -check-prefixes=CHECK,NONE
+; RUN: opt < %s -passes=float2int -S --data-layout="n64" | FileCheck %s -check-prefixes=CHECK,ONLY64
+; RUN: opt < %s -passes=float2int -S --data-layout="n8:16:32:64"| FileCheck %s -check-prefixes=CHECK,MULTIPLE
+; RUN: opt < %s -passes=float2int -S --data-layout="e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"| FileCheck %s -check-prefixes=CHECK,PR-79158
 
 define i32 @pr79158(i32 %x) {
 ; CHECK-LABEL: define i32 @pr79158(
@@ -18,3 +21,53 @@ entry:
   %conv1 = fptoui double %mul to i32
   ret i32 %conv1
 }
+
+define i32 @pr79158_2(i32 %x) {
+; NONE-LABEL: define i32 @pr79158_2(
+; NONE-SAME: i32 [[X:%.*]]) {
+; NONE-NEXT:  entry:
+; NONE-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X]], 0
+; NONE-NEXT:    [[TMP0:%.*]] = zext i1 [[CMP]] to i32
+; NONE-NEXT:    [[MUL1:%.*]] = mul i32 [[TMP0]], 255
+; NONE-NEXT:    [[TMP1:%.*]] = trunc i32 [[MUL1]] to i8
+; NONE-NEXT:    [[CONV2:%.*]] = zext i8 [[TMP1]] to i32
+; NONE-NEXT:    ret i32 [[CONV2]]
+;
+; ONLY64-LABEL: define i32 @pr79158_2(
+; ONLY64-SAME: i32 [[X:%.*]]) {
+; ONLY64-NEXT:  entry:
+; ONLY64-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X]], 0
+; ONLY64-NEXT:    [[TMP0:%.*]] = zext i1 [[CMP]] to i64
+; ONLY64-NEXT:    [[MUL1:%.*]] = mul i64 [[TMP0]], 255
+; ONLY64-NEXT:    [[TMP1:%.*]] = trunc i64 [[MUL1]] to i8
+; ONLY64-NEXT:    [[CONV2:%.*]] = zext i8 [[TMP1]] to i32
+; ONLY64-NEXT:    ret i32 [[CONV2]]
+;
+; MULTIPLE-LABEL: define i32 @pr79158_2(
+; MULTIPLE-SAME: i32 [[X:%.*]]) {
+; MULTIPLE-NEXT:  entry:
+; MULTIPLE-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X]], 0
+; MULTIPLE-NEXT:    [[TMP0:%.*]] = zext i1 [[CMP]] to i16
+; MULTIPLE-NEXT:    [[MUL1:%.*]] = mul i16 [[TMP0]], 255
+; MULTIPLE-NEXT:    [[TMP1:%.*]] = trunc i16 [[MUL1]] to i8
+; MULTIPLE-NEXT:    [[CONV2:%.*]] = zext i8 [[TMP1]] to i32
+; MULTIPLE-NEXT:    ret i32 [[CONV2]]
+;
+; PR-79158-LABEL: define i32 @pr79158_2(
+; PR-79158-SAME: i32 [[X:%.*]]) {
+; PR-79158-NEXT:  entry:
+; PR-79158-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X]], 0
+; PR-79158-NEXT:    [[TMP0:%.*]] = zext i1 [[CMP]] to i16
+; PR-79158-NEXT:    [[MUL1:%.*]] = mul i16 [[TMP0]], 255
+; PR-79158-NEXT:    [[TMP1:%.*]] = trunc i16 [[MUL1]] to i8
+; PR-79158-NEXT:    [[CONV2:%.*]] = zext i8 [[TMP1]] to i32
+; PR-79158-NEXT:    ret i32 [[CONV2]]
+;
+entry:
+  %cmp = icmp sgt i32 %x, 0
+  %conv = uitofp i1 %cmp to float
+  %mul = fmul float %conv, 2.550000e+02
+  %conv1 = fptoui float %mul to i8
+  %conv2 = zext i8 %conv1 to i32
+  ret i32 %conv2
+}
-- 
GitLab


From a9d1fead961440d415f931bc22c160dec88e03fd Mon Sep 17 00:00:00 2001
From: Vivian <zhyuhang88@gmail.com>
Date: Mon, 25 Mar 2024 09:53:57 -0700
Subject: [PATCH 144/404] Fix the condition for peeling the first iteration
 (#86350)

This PR fixes the condition used in loop peeling of the first iteration.
Using ceilDiv instead of floorDiv when calculating the loop counts, so
that the first iteration gets peeled as needed.
---
 .../SCF/Transforms/LoopSpecialization.cpp     |  2 +-
 .../Dialect/SCF/for-loop-peeling-front.mlir   | 47 +++++++++++++++++--
 2 files changed, 44 insertions(+), 5 deletions(-)

diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
index a5bff0a892c3..a30e349d4913 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
@@ -220,7 +220,7 @@ LogicalResult mlir::scf::peelForLoopFirstIteration(RewriterBase &b, ForOp forOp,
   auto stepInt = getConstantIntValue(forOp.getStep());
 
   // Peeling is not needed if there is one or less iteration.
-  if (lbInt && ubInt && stepInt && (*ubInt - *lbInt) / *stepInt <= 1)
+  if (lbInt && ubInt && stepInt && ceil(float(*ubInt - *lbInt) / *stepInt) <= 1)
     return failure();
 
   AffineExpr lbSymbol, stepSymbol;
diff --git a/mlir/test/Dialect/SCF/for-loop-peeling-front.mlir b/mlir/test/Dialect/SCF/for-loop-peeling-front.mlir
index 65141ff7623f..fe3b3e686a3e 100644
--- a/mlir/test/Dialect/SCF/for-loop-peeling-front.mlir
+++ b/mlir/test/Dialect/SCF/for-loop-peeling-front.mlir
@@ -13,11 +13,11 @@
 //      CHECK:     %[[INIT:.*]] = arith.addi %[[ACC]], %[[CAST]] : i32
 //      CHECK:     scf.yield %[[INIT]]
 //      CHECK:   }
-//      CHECK:   %[[RESULT:.*]] = scf.for %[[IV:.*]] = %[[C4]] to %[[C17]]
-// CHECK-SAME:       step %[[C4]] iter_args(%[[ACC:.*]] = %[[FIRST]]) -> (i32) {
-//      CHECK:     %[[MIN2:.*]] = affine.min #[[MAP]](%[[C17]], %[[IV]])[%[[C4]]]
+//      CHECK:   %[[RESULT:.*]] = scf.for %[[IV2:.*]] = %[[C4]] to %[[C17]]
+// CHECK-SAME:       step %[[C4]] iter_args(%[[ACC2:.*]] = %[[FIRST]]) -> (i32) {
+//      CHECK:     %[[MIN2:.*]] = affine.min #[[MAP]](%[[C17]], %[[IV2]])[%[[C4]]]
 //      CHECK:     %[[CAST2:.*]] = arith.index_cast %[[MIN2]] : index to i32
-//      CHECK:     %[[ADD:.*]] = arith.addi %[[ACC]], %[[CAST2]] : i32
+//      CHECK:     %[[ADD:.*]] = arith.addi %[[ACC2]], %[[CAST2]] : i32
 //      CHECK:     scf.yield %[[ADD]]
 //      CHECK:   }
 //      CHECK:   return %[[RESULT]]
@@ -110,6 +110,45 @@ func.func @fully_dynamic_bounds(%lb : index, %ub: index, %step: index) -> i32 {
 
 // -----
 
+//  CHECK-DAG: #[[MAP:.*]] = affine_map<(d0, d1)[s0] -> (4, d0 - d1)>
+//      CHECK: func @two_iteration_example(
+//  CHECK-DAG:   %[[C0_I32:.*]] = arith.constant 0 : i32
+//  CHECK-DAG:   %[[C2:.*]] = arith.constant 2 : index
+//  CHECK-DAG:   %[[C4:.*]] = arith.constant 4 : index
+//  CHECK-DAG:   %[[C8:.*]] = arith.constant 8 : index
+//  CHECK-DAG:   %[[C6:.*]] = arith.constant 6 : index
+//      CHECK:   %[[FIRST:.*]] = scf.for %[[IV:.*]] = %[[C2]] to %[[C6]]
+// CHECK-SAME:       step %[[C4]] iter_args(%[[ACC:.*]] = %[[C0_I32]]) -> (i32) {
+//      CHECK:     %[[MIN:.*]] = affine.min #[[MAP]](%[[C6]], %[[IV]])[%[[C4]]]
+//      CHECK:     %[[CAST:.*]] = arith.index_cast %[[MIN]] : index to i32
+//      CHECK:     %[[INIT:.*]] = arith.addi %[[ACC]], %[[CAST]] : i32
+//      CHECK:     scf.yield %[[INIT]]
+//      CHECK:   }
+//      CHECK:   %[[RESULT:.*]] = scf.for %[[IV2:.*]] = %[[C6]] to %[[C8]]
+// CHECK-SAME:       step %[[C4]] iter_args(%[[ACC2:.*]] = %[[FIRST]]) -> (i32) {
+//      CHECK:     %[[MIN2:.*]] = affine.min #[[MAP]](%[[C8]], %[[IV2]])[%[[C4]]]
+//      CHECK:     %[[CAST2:.*]] = arith.index_cast %[[MIN2]] : index to i32
+//      CHECK:     %[[ADD:.*]] = arith.addi %[[ACC2]], %[[CAST2]] : i32
+//      CHECK:     scf.yield %[[ADD]]
+//      CHECK:   }
+//      CHECK:   return %[[RESULT]]
+#map = affine_map<(d0, d1)[s0] -> (s0, d0 - d1)>
+func.func @two_iteration_example() -> i32 {
+  %c0_i32 = arith.constant 0 : i32
+  %lb = arith.constant 2 : index
+  %step = arith.constant 4 : index
+  %ub = arith.constant 8 : index
+  %r = scf.for %iv = %lb to %ub step %step iter_args(%arg = %c0_i32) -> i32 {
+    %s = affine.min #map(%ub, %iv)[%step]
+    %casted = arith.index_cast %s : index to i32
+    %0 = arith.addi %arg, %casted : i32
+    scf.yield %0 : i32
+  }
+  return %r : i32
+}
+
+// -----
+
 //  CHECK-DAG: #[[MAP:.*]] = affine_map<(d0, d1)[s0] -> (4, d0 - d1)>
 //      CHECK: func @no_peeling_front(
 //  CHECK-DAG:   %[[C0_I32:.*]] = arith.constant 0 : i32
-- 
GitLab


From d99cfa053998483e7283e91ff204fbe117345685 Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Mon, 25 Mar 2024 12:54:11 -0400
Subject: [PATCH 145/404] [C99] Claim conformance to WG14 N631 and WG14 N696

Both of these relate to handling of standard pragmas. N631 is about
various STDC pragmas being included in the standard at all, and N696
is about whether macros are expanded in standard pragmas (they're not).
---
 clang/test/C/C99/n696.c | 22 ++++++++++++++++++++++
 clang/www/c_status.html |  4 ++--
 2 files changed, 24 insertions(+), 2 deletions(-)
 create mode 100644 clang/test/C/C99/n696.c

diff --git a/clang/test/C/C99/n696.c b/clang/test/C/C99/n696.c
new file mode 100644
index 000000000000..c84ef2c7159e
--- /dev/null
+++ b/clang/test/C/C99/n696.c
@@ -0,0 +1,22 @@
+// RUN: %clang_cc1 -verify %s
+
+/* WG14 N696: yes
+ * Standard pragmas - improved wording
+ *
+ * NB: this also covers N631 which changed these features into pragmas rather
+ * than macros.
+ */
+
+// Verify that we do not expand macros in STDC pragmas. If we expanded them,
+// this code would issue diagnostics.
+#define ON 12
+#pragma STDC FENV_ACCESS ON
+#pragma STDC CX_LIMITED_RANGE ON
+#pragma STDC FP_CONTRACT ON
+
+// If we expanded macros, this code would not issue diagnostics.
+#define BLERP OFF
+#pragma STDC FENV_ACCESS BLERP      // expected-warning {{expected 'ON' or 'OFF' or 'DEFAULT' in pragma}}
+#pragma STDC CX_LIMITED_RANGE BLERP // expected-warning {{expected 'ON' or 'OFF' or 'DEFAULT' in pragma}}
+#pragma STDC FP_CONTRACT BLERP      // expected-warning {{expected 'ON' or 'OFF' or 'DEFAULT' in pragma}}
+
diff --git a/clang/www/c_status.html b/clang/www/c_status.html
index e97275e02033..ddfdc2f21c87 100644
--- a/clang/www/c_status.html
+++ b/clang/www/c_status.html
@@ -330,11 +330,11 @@ conformance.</p>
     </tr>
       <tr>
         <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n631.htm">N631</a></td>
-        <td class="unknown" align="center">Unknown</td>
+        <td class="full" align="center">Yes</td>
       </tr>
       <tr>
         <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n696.ps">N696</a></td>
-        <td class="unknown" align="center">Unknown</td>
+        <td class="full" align="center">Yes</td>
       </tr>
     <tr>
       <td>__func__ predefined identifier</td>
-- 
GitLab


From 96819daa3d095cf9f662e0229dc82eaaa25480e8 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Mon, 25 Mar 2024 17:10:23 +0000
Subject: [PATCH 146/404] [AArch64] Handle v2i16 and v2i8 in concat load
 combine. (#86264)

This extends the concat load patch from
https://reviews.llvm.org/D121400, which was later moved to a combine, to
handle v2i8 and v2i16 concat loads too.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    | 21 +++--
 llvm/test/CodeGen/AArch64/insert-subvector.ll | 85 ++++---------------
 2 files changed, 28 insertions(+), 78 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 7fab274ab957..043a617898a6 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -18629,12 +18629,16 @@ static SDValue performConcatVectorsCombine(SDNode *N,
     }
   }
 
-  if (N->getOperand(0).getValueType() == MVT::v4i8) {
+  if (N->getOperand(0).getValueType() == MVT::v4i8 ||
+      N->getOperand(0).getValueType() == MVT::v2i16 ||
+      N->getOperand(0).getValueType() == MVT::v2i8) {
+    EVT SrcVT = N->getOperand(0).getValueType();
     // If we have a concat of v4i8 loads, convert them to a buildvector of f32
     // loads to prevent having to go through the v4i8 load legalization that
     // needs to extend each element into a larger type.
-    if (N->getNumOperands() % 2 == 0 && all_of(N->op_values(), [](SDValue V) {
-          if (V.getValueType() != MVT::v4i8)
+    if (N->getNumOperands() % 2 == 0 &&
+        all_of(N->op_values(), [SrcVT](SDValue V) {
+          if (V.getValueType() != SrcVT)
             return false;
           if (V.isUndef())
             return true;
@@ -18642,19 +18646,18 @@ static SDValue performConcatVectorsCombine(SDNode *N,
           return LD && V.hasOneUse() && LD->isSimple() && !LD->isIndexed() &&
                  LD->getExtensionType() == ISD::NON_EXTLOAD;
         })) {
-      EVT NVT =
-          EVT::getVectorVT(*DAG.getContext(), MVT::f32, N->getNumOperands());
+      EVT FVT = SrcVT == MVT::v2i8 ? MVT::f16 : MVT::f32;
+      EVT NVT = EVT::getVectorVT(*DAG.getContext(), FVT, N->getNumOperands());
       SmallVector<SDValue> Ops;
 
       for (unsigned i = 0; i < N->getNumOperands(); i++) {
         SDValue V = N->getOperand(i);
         if (V.isUndef())
-          Ops.push_back(DAG.getUNDEF(MVT::f32));
+          Ops.push_back(DAG.getUNDEF(FVT));
         else {
           LoadSDNode *LD = cast<LoadSDNode>(V);
-          SDValue NewLoad =
-              DAG.getLoad(MVT::f32, dl, LD->getChain(), LD->getBasePtr(),
-                          LD->getMemOperand());
+          SDValue NewLoad = DAG.getLoad(FVT, dl, LD->getChain(),
+                                        LD->getBasePtr(), LD->getMemOperand());
           DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewLoad.getValue(1));
           Ops.push_back(NewLoad);
         }
diff --git a/llvm/test/CodeGen/AArch64/insert-subvector.ll b/llvm/test/CodeGen/AArch64/insert-subvector.ll
index 95ad9807ed63..6828fa9f1508 100644
--- a/llvm/test/CodeGen/AArch64/insert-subvector.ll
+++ b/llvm/test/CodeGen/AArch64/insert-subvector.ll
@@ -377,12 +377,8 @@ define <16 x i8> @load_v16i8_8_2(float %tmp, <16 x i8> %b, ptr %a) {
 define <8 x i8> @load_v8i8_2_1(float %tmp, <8 x i8> %b, ptr %a) {
 ; CHECK-LABEL: load_v8i8_2_1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1 { v2.b }[0], [x0]
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    mov v0.16b, v2.16b
-; CHECK-NEXT:    ld1 { v0.b }[4], [x8]
-; CHECK-NEXT:    mov v2.b[1], v0.b[4]
 ; CHECK-NEXT:    fmov d0, d1
+; CHECK-NEXT:    ldr h2, [x0]
 ; CHECK-NEXT:    mov v0.h[0], v2.h[0]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
@@ -395,12 +391,9 @@ define <8 x i8> @load_v8i8_2_1(float %tmp, <8 x i8> %b, ptr %a) {
 define <8 x i8> @load_v8i8_2_15(float %tmp, <8 x i8> %b, ptr %a) {
 ; CHECK-LABEL: load_v8i8_2_15:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1 { v0.b }[0], [x0]
-; CHECK-NEXT:    add x8, x0, #1
+; CHECK-NEXT:    ldr h0, [x0]
 ; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT:    ld1 { v0.b }[4], [x8]
 ; CHECK-NEXT:    adrp x8, .LCPI33_0
-; CHECK-NEXT:    mov v0.b[1], v0.b[4]
 ; CHECK-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-NEXT:    ldr d1, [x8, :lo12:.LCPI33_0]
 ; CHECK-NEXT:    tbl v0.8b, { v0.16b }, v1.8b
@@ -414,12 +407,8 @@ define <8 x i8> @load_v8i8_2_15(float %tmp, <8 x i8> %b, ptr %a) {
 define <8 x i8> @load_v8i8_2_2(float %tmp, <8 x i8> %b, ptr %a) {
 ; CHECK-LABEL: load_v8i8_2_2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1 { v2.b }[0], [x0]
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    mov v0.16b, v2.16b
-; CHECK-NEXT:    ld1 { v0.b }[4], [x8]
-; CHECK-NEXT:    mov v2.b[1], v0.b[4]
 ; CHECK-NEXT:    fmov d0, d1
+; CHECK-NEXT:    ldr h2, [x0]
 ; CHECK-NEXT:    mov v0.h[1], v2.h[0]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
@@ -432,12 +421,8 @@ define <8 x i8> @load_v8i8_2_2(float %tmp, <8 x i8> %b, ptr %a) {
 define <8 x i8> @load_v8i8_2_3(float %tmp, <8 x i8> %b, ptr %a) {
 ; CHECK-LABEL: load_v8i8_2_3:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1 { v2.b }[0], [x0]
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    mov v0.16b, v2.16b
-; CHECK-NEXT:    ld1 { v0.b }[4], [x8]
-; CHECK-NEXT:    mov v2.b[1], v0.b[4]
 ; CHECK-NEXT:    fmov d0, d1
+; CHECK-NEXT:    ldr h2, [x0]
 ; CHECK-NEXT:    mov v0.h[2], v2.h[0]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
@@ -450,12 +435,8 @@ define <8 x i8> @load_v8i8_2_3(float %tmp, <8 x i8> %b, ptr %a) {
 define <8 x i8> @load_v8i8_2_4(float %tmp, <8 x i8> %b, ptr %a) {
 ; CHECK-LABEL: load_v8i8_2_4:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1 { v2.b }[0], [x0]
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    mov v0.16b, v2.16b
-; CHECK-NEXT:    ld1 { v0.b }[4], [x8]
-; CHECK-NEXT:    mov v2.b[1], v0.b[4]
 ; CHECK-NEXT:    fmov d0, d1
+; CHECK-NEXT:    ldr h2, [x0]
 ; CHECK-NEXT:    mov v0.h[3], v2.h[0]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
@@ -468,11 +449,9 @@ define <8 x i8> @load_v8i8_2_4(float %tmp, <8 x i8> %b, ptr %a) {
 define <4 x i8> @load_v4i8_2_1(float %tmp, <4 x i8> %b, ptr %a) {
 ; CHECK-LABEL: load_v4i8_2_1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1 { v0.b }[0], [x0]
-; CHECK-NEXT:    add x8, x0, #1
+; CHECK-NEXT:    ldr h0, [x0]
 ; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT:    ld1 { v0.b }[4], [x8]
-; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-NEXT:    zip1 v0.8b, v0.8b, v0.8b
 ; CHECK-NEXT:    mov v0.s[1], v1.s[1]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
@@ -485,10 +464,8 @@ define <4 x i8> @load_v4i8_2_1(float %tmp, <4 x i8> %b, ptr %a) {
 define <4 x i8> @load_v4i8_2_2(float %tmp, <4 x i8> %b, ptr %a) {
 ; CHECK-LABEL: load_v4i8_2_2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1 { v0.b }[0], [x0]
-; CHECK-NEXT:    add x8, x0, #1
-; CHECK-NEXT:    ld1 { v0.b }[4], [x8]
-; CHECK-NEXT:    uzp1 v2.4h, v0.4h, v0.4h
+; CHECK-NEXT:    ldr h0, [x0]
+; CHECK-NEXT:    zip1 v2.8b, v0.8b, v0.8b
 ; CHECK-NEXT:    fmov d0, d1
 ; CHECK-NEXT:    mov v0.s[1], v2.s[0]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -504,13 +481,8 @@ define <4 x i8> @load_v4i8_2_2(float %tmp, <4 x i8> %b, ptr %a) {
 define <8 x i16> @load_v8i16_2_1(float %tmp, <8 x i16> %b, ptr %a) {
 ; CHECK-LABEL: load_v8i16_2_1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    add x9, x0, #2
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    ld1 { v0.h }[2], [x9]
-; CHECK-NEXT:    xtn v2.4h, v0.4s
 ; CHECK-NEXT:    mov v0.16b, v1.16b
-; CHECK-NEXT:    mov v0.s[0], v2.s[0]
+; CHECK-NEXT:    ld1 { v0.s }[0], [x0]
 ; CHECK-NEXT:    ret
   %l = load <2 x i16>, ptr %a
   %s1 = shufflevector <2 x i16> %l, <2 x i16> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -521,13 +493,9 @@ define <8 x i16> @load_v8i16_2_1(float %tmp, <8 x i16> %b, ptr %a) {
 define <8 x i16> @load_v8i16_2_15(float %tmp, <8 x i16> %b, ptr %a) {
 ; CHECK-LABEL: load_v8i16_2_15:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    add x9, x0, #2
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 def $q0_q1
-; CHECK-NEXT:    fmov s2, w8
 ; CHECK-NEXT:    adrp x8, .LCPI40_0
-; CHECK-NEXT:    ld1 { v2.h }[2], [x9]
-; CHECK-NEXT:    xtn v0.4h, v2.4s
+; CHECK-NEXT:    ldr s0, [x0]
 ; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI40_0]
 ; CHECK-NEXT:    tbl v0.16b, { v0.16b, v1.16b }, v2.16b
 ; CHECK-NEXT:    ret
@@ -540,13 +508,8 @@ define <8 x i16> @load_v8i16_2_15(float %tmp, <8 x i16> %b, ptr %a) {
 define <8 x i16> @load_v8i16_2_2(float %tmp, <8 x i16> %b, ptr %a) {
 ; CHECK-LABEL: load_v8i16_2_2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    add x9, x0, #2
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    ld1 { v0.h }[2], [x9]
-; CHECK-NEXT:    xtn v2.4h, v0.4s
 ; CHECK-NEXT:    mov v0.16b, v1.16b
-; CHECK-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-NEXT:    ld1 { v0.s }[1], [x0]
 ; CHECK-NEXT:    ret
   %l = load <2 x i16>, ptr %a
   %s1 = shufflevector <2 x i16> %l, <2 x i16> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -557,13 +520,8 @@ define <8 x i16> @load_v8i16_2_2(float %tmp, <8 x i16> %b, ptr %a) {
 define <8 x i16> @load_v8i16_2_3(float %tmp, <8 x i16> %b, ptr %a) {
 ; CHECK-LABEL: load_v8i16_2_3:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    add x9, x0, #2
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    ld1 { v0.h }[2], [x9]
-; CHECK-NEXT:    xtn v2.4h, v0.4s
 ; CHECK-NEXT:    mov v0.16b, v1.16b
-; CHECK-NEXT:    mov v0.s[2], v2.s[0]
+; CHECK-NEXT:    ld1 { v0.s }[2], [x0]
 ; CHECK-NEXT:    ret
   %l = load <2 x i16>, ptr %a
   %s1 = shufflevector <2 x i16> %l, <2 x i16> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -574,13 +532,8 @@ define <8 x i16> @load_v8i16_2_3(float %tmp, <8 x i16> %b, ptr %a) {
 define <8 x i16> @load_v8i16_2_4(float %tmp, <8 x i16> %b, ptr %a) {
 ; CHECK-LABEL: load_v8i16_2_4:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldrh w8, [x0]
-; CHECK-NEXT:    add x9, x0, #2
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    ld1 { v0.h }[2], [x9]
-; CHECK-NEXT:    xtn v2.4h, v0.4s
 ; CHECK-NEXT:    mov v0.16b, v1.16b
-; CHECK-NEXT:    mov v0.s[3], v2.s[0]
+; CHECK-NEXT:    ld1 { v0.s }[3], [x0]
 ; CHECK-NEXT:    ret
   %l = load <2 x i16>, ptr %a
   %s1 = shufflevector <2 x i16> %l, <2 x i16> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -591,11 +544,8 @@ define <8 x i16> @load_v8i16_2_4(float %tmp, <8 x i16> %b, ptr %a) {
 define <4 x i16> @load_v4i16_2_1(float %tmp, <4 x i16> %b, ptr %a) {
 ; CHECK-LABEL: load_v4i16_2_1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1 { v0.h }[0], [x0]
-; CHECK-NEXT:    add x8, x0, #2
+; CHECK-NEXT:    ldr s0, [x0]
 ; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT:    ld1 { v0.h }[2], [x8]
-; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
 ; CHECK-NEXT:    mov v0.s[1], v1.s[1]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
@@ -608,11 +558,8 @@ define <4 x i16> @load_v4i16_2_1(float %tmp, <4 x i16> %b, ptr %a) {
 define <4 x i16> @load_v4i16_2_2(float %tmp, <4 x i16> %b, ptr %a) {
 ; CHECK-LABEL: load_v4i16_2_2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ld1 { v0.h }[0], [x0]
-; CHECK-NEXT:    add x8, x0, #2
-; CHECK-NEXT:    ld1 { v0.h }[2], [x8]
-; CHECK-NEXT:    uzp1 v2.4h, v0.4h, v0.4h
 ; CHECK-NEXT:    fmov d0, d1
+; CHECK-NEXT:    ldr s2, [x0]
 ; CHECK-NEXT:    mov v0.s[1], v2.s[0]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
-- 
GitLab


From 5ef09540e16dfc81643eaf866d4ada0b52cf27b8 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Mon, 25 Mar 2024 13:10:44 -0400
Subject: [PATCH 147/404] [gn] port 4fb0805c6525b

That's an old commit, but it suddenly became load-bearing, maybe
due to 0cf4788d9d0df609
---
 llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn
index c2999a67f58a..103954e5756d 100644
--- a/llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/Interpreter/BUILD.gn
@@ -8,6 +8,7 @@ unittest("ClangReplInterpreterTests") {
     "//clang/lib/Frontend",
     "//clang/lib/Interpreter",
     "//llvm/lib/IR",
+    "//llvm/lib/Target:TargetsToBuild",
     "//llvm/lib/TargetParser",
     "//llvm/lib/Testing/Support",
   ]
-- 
GitLab


From c62c74639ad6eb0988861077b7123fa2b2c17cc5 Mon Sep 17 00:00:00 2001
From: Cooper Partin <coopp@microsoft.com>
Date: Mon, 25 Mar 2024 10:18:53 -0700
Subject: [PATCH 148/404] Add support for PSV EntryFunctionName (#86296)

This change introduces a version 3 of the PSV data that includes support
for the name of the entry function as an offset into StringTable data to
a null-terminated utf-8 string.

Additional tests were added to ensure that the new value was properly
serialized/deserialized from object data.

Fixes #80175

---------

Co-authored-by: Cooper Partin <coopp@ntdev.microsoft.com>
---
 llvm/include/llvm/BinaryFormat/DXContainer.h  |  16 +++
 llvm/include/llvm/MC/DXContainerPSVInfo.h     |  25 ++--
 llvm/include/llvm/MC/StringTableBuilder.h     |   8 +-
 llvm/include/llvm/Object/DXContainer.h        |  16 ++-
 .../include/llvm/ObjectYAML/DXContainerYAML.h |   5 +-
 llvm/lib/MC/DXContainerPSVInfo.cpp            |  75 ++++++++----
 llvm/lib/Object/DXContainer.cpp               |  15 ++-
 llvm/lib/ObjectYAML/DXContainerEmitter.cpp    |   3 +-
 llvm/lib/ObjectYAML/DXContainerYAML.cpp       |  15 +++
 .../DXContainer/PSVv3-amplification.yaml      |  97 ++++++++++++++++
 .../ObjectYAML/DXContainer/PSVv3-compute.yaml |  95 +++++++++++++++
 .../ObjectYAML/DXContainer/PSVv3-domain.yaml  | 105 +++++++++++++++++
 .../DXContainer/PSVv3-geometry.yaml           | 105 +++++++++++++++++
 .../ObjectYAML/DXContainer/PSVv3-hull.yaml    | 107 +++++++++++++++++
 .../ObjectYAML/DXContainer/PSVv3-mesh.yaml    | 109 ++++++++++++++++++
 .../ObjectYAML/DXContainer/PSVv3-pixel.yaml   |  99 ++++++++++++++++
 .../ObjectYAML/DXContainer/PSVv3-vertex.yaml  |  97 ++++++++++++++++
 llvm/tools/obj2yaml/dxcontainer2yaml.cpp      |   3 +
 18 files changed, 943 insertions(+), 52 deletions(-)
 create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml
 create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml
 create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml
 create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml
 create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml
 create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml
 create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml
 create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml

diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h
index 532f9481766a..e8d03f806715 100644
--- a/llvm/include/llvm/BinaryFormat/DXContainer.h
+++ b/llvm/include/llvm/BinaryFormat/DXContainer.h
@@ -424,6 +424,22 @@ struct ResourceBindInfo : public v0::ResourceBindInfo {
 };
 
 } // namespace v2
+
+namespace v3 {
+struct RuntimeInfo : public v2::RuntimeInfo {
+  uint32_t EntryNameOffset;
+
+  void swapBytes() {
+    v2::RuntimeInfo::swapBytes();
+    sys::swapByteOrder(EntryNameOffset);
+  }
+
+  void swapBytes(Triple::EnvironmentType Stage) {
+    v2::RuntimeInfo::swapBytes(Stage);
+  }
+};
+
+} // namespace v3
 } // namespace PSV
 
 #define COMPONENT_PRECISION(Val, Enum) Enum = Val,
diff --git a/llvm/include/llvm/MC/DXContainerPSVInfo.h b/llvm/include/llvm/MC/DXContainerPSVInfo.h
index 7d21c18d252f..bad2fe78eb8f 100644
--- a/llvm/include/llvm/MC/DXContainerPSVInfo.h
+++ b/llvm/include/llvm/MC/DXContainerPSVInfo.h
@@ -9,9 +9,11 @@
 #ifndef LLVM_MC_DXCONTAINERPSVINFO_H
 #define LLVM_MC_DXCONTAINERPSVINFO_H
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/DXContainer.h"
+#include "llvm/MC/StringTableBuilder.h"
 #include "llvm/TargetParser/Triple.h"
 
 #include <array>
@@ -45,8 +47,9 @@ struct PSVSignatureElement {
 // modifiable format, and can be used to serialize the data back into valid PSV
 // RuntimeInfo.
 struct PSVRuntimeInfo {
+  PSVRuntimeInfo() : DXConStrTabBuilder(StringTableBuilder::DXContainer) {}
   bool IsFinalized = false;
-  dxbc::PSV::v2::RuntimeInfo BaseData;
+  dxbc::PSV::v3::RuntimeInfo BaseData;
   SmallVector<dxbc::PSV::v2::ResourceBindInfo> Resources;
   SmallVector<PSVSignatureElement> InputElements;
   SmallVector<PSVSignatureElement> OutputElements;
@@ -64,6 +67,7 @@ struct PSVRuntimeInfo {
   std::array<SmallVector<uint32_t>, 4> InputOutputMap;
   SmallVector<uint32_t> InputPatchMap;
   SmallVector<uint32_t> PatchOutputMap;
+  llvm::StringRef EntryName;
 
   // Serialize PSVInfo into the provided raw_ostream. The version field
   // specifies the data version to encode, the default value specifies encoding
@@ -71,19 +75,12 @@ struct PSVRuntimeInfo {
   void write(raw_ostream &OS,
              uint32_t Version = std::numeric_limits<uint32_t>::max()) const;
 
-  void finalize(Triple::EnvironmentType Stage) {
-    IsFinalized = true;
-    BaseData.SigInputElements = static_cast<uint32_t>(InputElements.size());
-    BaseData.SigOutputElements = static_cast<uint32_t>(OutputElements.size());
-    BaseData.SigPatchOrPrimElements =
-        static_cast<uint32_t>(PatchOrPrimElements.size());
-    if (!sys::IsBigEndianHost)
-      return;
-    BaseData.swapBytes();
-    BaseData.swapBytes(Stage);
-    for (auto &Res : Resources)
-      Res.swapBytes();
-  }
+  void finalize(Triple::EnvironmentType Stage);
+
+private:
+  SmallVector<uint32_t, 64> IndexBuffer;
+  SmallVector<llvm::dxbc::PSV::v0::SignatureElement, 32> SignatureElements;
+  StringTableBuilder DXConStrTabBuilder;
 };
 
 class Signature {
diff --git a/llvm/include/llvm/MC/StringTableBuilder.h b/llvm/include/llvm/MC/StringTableBuilder.h
index 4ee421e22c17..a738683548cf 100644
--- a/llvm/include/llvm/MC/StringTableBuilder.h
+++ b/llvm/include/llvm/MC/StringTableBuilder.h
@@ -74,12 +74,8 @@ public:
   /// Check if a string is contained in the string table. Since this class
   /// doesn't store the string values, this function can be used to check if
   /// storage needs to be done prior to adding the string.
-  bool contains(StringRef S) const {
-    return contains(CachedHashStringRef(S));
-  }
-  bool contains(CachedHashStringRef S) const {
-    return StringIndexMap.count(S);
-  }
+  bool contains(StringRef S) const { return contains(CachedHashStringRef(S)); }
+  bool contains(CachedHashStringRef S) const { return StringIndexMap.count(S); }
 
   size_t getSize() const { return Size; }
   void clear();
diff --git a/llvm/include/llvm/Object/DXContainer.h b/llvm/include/llvm/Object/DXContainer.h
index b6e3d321da24..19c83ba6c6e8 100644
--- a/llvm/include/llvm/Object/DXContainer.h
+++ b/llvm/include/llvm/Object/DXContainer.h
@@ -125,7 +125,8 @@ class PSVRuntimeInfo {
   uint32_t Size;
   using InfoStruct =
       std::variant<std::monostate, dxbc::PSV::v0::RuntimeInfo,
-                   dxbc::PSV::v1::RuntimeInfo, dxbc::PSV::v2::RuntimeInfo>;
+                   dxbc::PSV::v1::RuntimeInfo, dxbc::PSV::v2::RuntimeInfo,
+                   dxbc::PSV::v3::RuntimeInfo>;
   InfoStruct BasicInfo;
   ResourceArray Resources;
   StringRef StringTable;
@@ -151,9 +152,11 @@ public:
   ResourceArray getResources() const { return Resources; }
 
   uint32_t getVersion() const {
-    return Size >= sizeof(dxbc::PSV::v2::RuntimeInfo)
-               ? 2
-               : (Size >= sizeof(dxbc::PSV::v1::RuntimeInfo) ? 1 : 0);
+    return Size >= sizeof(dxbc::PSV::v3::RuntimeInfo)
+               ? 3
+               : (Size >= sizeof(dxbc::PSV::v2::RuntimeInfo)     ? 2
+                  : (Size >= sizeof(dxbc::PSV::v1::RuntimeInfo)) ? 1
+                                                                 : 0);
   }
 
   uint32_t getResourceStride() const { return Resources.Stride; }
@@ -161,6 +164,11 @@ public:
   const InfoStruct &getInfo() const { return BasicInfo; }
 
   template <typename T> const T *getInfoAs() const {
+    if (const auto *P = std::get_if<dxbc::PSV::v3::RuntimeInfo>(&BasicInfo))
+      return static_cast<const T *>(P);
+    if (std::is_same<T, dxbc::PSV::v3::RuntimeInfo>::value)
+      return nullptr;
+
     if (const auto *P = std::get_if<dxbc::PSV::v2::RuntimeInfo>(&BasicInfo))
       return static_cast<const T *>(P);
     if (std::is_same<T, dxbc::PSV::v2::RuntimeInfo>::value)
diff --git a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
index f7f8d5e6bf47..9c4d9e19f11b 100644
--- a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
+++ b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
@@ -107,7 +107,7 @@ struct PSVInfo {
   // the format.
   uint32_t Version;
 
-  dxbc::PSV::v2::RuntimeInfo Info;
+  dxbc::PSV::v3::RuntimeInfo Info;
   uint32_t ResourceStride;
   SmallVector<ResourceBindInfo> Resources;
   SmallVector<SignatureElement> SigInputElements;
@@ -121,12 +121,15 @@ struct PSVInfo {
   MaskVector InputPatchMap;
   MaskVector PatchOutputMap;
 
+  StringRef EntryName;
+
   void mapInfoForVersion(yaml::IO &IO);
 
   PSVInfo();
   PSVInfo(const dxbc::PSV::v0::RuntimeInfo *P, uint16_t Stage);
   PSVInfo(const dxbc::PSV::v1::RuntimeInfo *P);
   PSVInfo(const dxbc::PSV::v2::RuntimeInfo *P);
+  PSVInfo(const dxbc::PSV::v3::RuntimeInfo *P, StringRef StringTable);
 };
 
 struct SignatureParameter {
diff --git a/llvm/lib/MC/DXContainerPSVInfo.cpp b/llvm/lib/MC/DXContainerPSVInfo.cpp
index 48182fcd31df..aeff69380139 100644
--- a/llvm/lib/MC/DXContainerPSVInfo.cpp
+++ b/llvm/lib/MC/DXContainerPSVInfo.cpp
@@ -81,13 +81,18 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const {
     BindingSize = sizeof(dxbc::PSV::v0::ResourceBindInfo);
     break;
   case 2:
-  default:
     InfoSize = sizeof(dxbc::PSV::v2::RuntimeInfo);
     BindingSize = sizeof(dxbc::PSV::v2::ResourceBindInfo);
+    break;
+  case 3:
+  default:
+    InfoSize = sizeof(dxbc::PSV::v3::RuntimeInfo);
+    BindingSize = sizeof(dxbc::PSV::v2::ResourceBindInfo);
   }
-  // Write the size of the info.
 
+  // Write the size of the info.
   support::endian::write(OS, InfoSize, llvm::endianness::little);
+
   // Write the info itself.
   OS.write(reinterpret_cast<const char *>(&BaseData), InfoSize);
 
@@ -104,32 +109,12 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const {
   if (Version == 0)
     return;
 
-  StringTableBuilder StrTabBuilder((StringTableBuilder::DXContainer));
-  SmallVector<uint32_t, 64> IndexBuffer;
-  SmallVector<v0::SignatureElement, 32> SignatureElements;
-  SmallVector<StringRef, 32> SemanticNames;
-
-  ProcessElementList(StrTabBuilder, IndexBuffer, SignatureElements,
-                     SemanticNames, InputElements);
-  ProcessElementList(StrTabBuilder, IndexBuffer, SignatureElements,
-                     SemanticNames, OutputElements);
-  ProcessElementList(StrTabBuilder, IndexBuffer, SignatureElements,
-                     SemanticNames, PatchOrPrimElements);
-
-  StrTabBuilder.finalize();
-  for (auto ElAndName : zip(SignatureElements, SemanticNames)) {
-    v0::SignatureElement &El = std::get<0>(ElAndName);
-    StringRef Name = std::get<1>(ElAndName);
-    El.NameOffset = static_cast<uint32_t>(StrTabBuilder.getOffset(Name));
-    if (sys::IsBigEndianHost)
-      El.swapBytes();
-  }
-
-  support::endian::write(OS, static_cast<uint32_t>(StrTabBuilder.getSize()),
+  support::endian::write(OS,
+                         static_cast<uint32_t>(DXConStrTabBuilder.getSize()),
                          llvm::endianness::little);
 
   // Write the string table.
-  StrTabBuilder.write(OS);
+  DXConStrTabBuilder.write(OS);
 
   // Write the index table size, then table.
   support::endian::write(OS, static_cast<uint32_t>(IndexBuffer.size()),
@@ -162,6 +147,46 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const {
                                llvm::endianness::little);
 }
 
+void PSVRuntimeInfo::finalize(Triple::EnvironmentType Stage) {
+  IsFinalized = true;
+  BaseData.SigInputElements = static_cast<uint32_t>(InputElements.size());
+  BaseData.SigOutputElements = static_cast<uint32_t>(OutputElements.size());
+  BaseData.SigPatchOrPrimElements =
+      static_cast<uint32_t>(PatchOrPrimElements.size());
+
+  SmallVector<StringRef, 32> SemanticNames;
+
+  // Build a string table and set associated offsets to be written when
+  // write() is called
+  ProcessElementList(DXConStrTabBuilder, IndexBuffer, SignatureElements,
+                     SemanticNames, InputElements);
+  ProcessElementList(DXConStrTabBuilder, IndexBuffer, SignatureElements,
+                     SemanticNames, OutputElements);
+  ProcessElementList(DXConStrTabBuilder, IndexBuffer, SignatureElements,
+                     SemanticNames, PatchOrPrimElements);
+
+  DXConStrTabBuilder.add(EntryName);
+
+  DXConStrTabBuilder.finalize();
+  for (auto ElAndName : zip(SignatureElements, SemanticNames)) {
+    llvm::dxbc::PSV::v0::SignatureElement &El = std::get<0>(ElAndName);
+    StringRef Name = std::get<1>(ElAndName);
+    El.NameOffset = static_cast<uint32_t>(DXConStrTabBuilder.getOffset(Name));
+    if (sys::IsBigEndianHost)
+      El.swapBytes();
+  }
+
+  BaseData.EntryNameOffset =
+      static_cast<uint32_t>(DXConStrTabBuilder.getOffset(EntryName));
+
+  if (!sys::IsBigEndianHost)
+    return;
+  BaseData.swapBytes();
+  BaseData.swapBytes(Stage);
+  for (auto &Res : Resources)
+    Res.swapBytes();
+}
+
 void Signature::write(raw_ostream &OS) {
   SmallVector<dxbc::ProgramSignatureElement> SigParams;
   SigParams.reserve(Params.size());
diff --git a/llvm/lib/Object/DXContainer.cpp b/llvm/lib/Object/DXContainer.cpp
index 935749afe338..3b1a6203a1f8 100644
--- a/llvm/lib/Object/DXContainer.cpp
+++ b/llvm/lib/Object/DXContainer.cpp
@@ -247,7 +247,14 @@ Error DirectX::PSVRuntimeInfo::parse(uint16_t ShaderKind) {
   const uint32_t PSVVersion = getVersion();
 
   // Detect the PSVVersion by looking at the size field.
-  if (PSVVersion == 2) {
+  if (PSVVersion == 3) {
+    v3::RuntimeInfo Info;
+    if (Error Err = readStruct(PSVInfoData, Current, Info))
+      return Err;
+    if (sys::IsBigEndianHost)
+      Info.swapBytes(ShaderStage);
+    BasicInfo = Info;
+  } else if (PSVVersion == 2) {
     v2::RuntimeInfo Info;
     if (Error Err = readStruct(PSVInfoData, Current, Info))
       return Err;
@@ -425,6 +432,8 @@ Error DirectX::PSVRuntimeInfo::parse(uint16_t ShaderKind) {
 }
 
 uint8_t DirectX::PSVRuntimeInfo::getSigInputCount() const {
+  if (const auto *P = std::get_if<dxbc::PSV::v3::RuntimeInfo>(&BasicInfo))
+    return P->SigInputElements;
   if (const auto *P = std::get_if<dxbc::PSV::v2::RuntimeInfo>(&BasicInfo))
     return P->SigInputElements;
   if (const auto *P = std::get_if<dxbc::PSV::v1::RuntimeInfo>(&BasicInfo))
@@ -433,6 +442,8 @@ uint8_t DirectX::PSVRuntimeInfo::getSigInputCount() const {
 }
 
 uint8_t DirectX::PSVRuntimeInfo::getSigOutputCount() const {
+  if (const auto *P = std::get_if<dxbc::PSV::v3::RuntimeInfo>(&BasicInfo))
+    return P->SigOutputElements;
   if (const auto *P = std::get_if<dxbc::PSV::v2::RuntimeInfo>(&BasicInfo))
     return P->SigOutputElements;
   if (const auto *P = std::get_if<dxbc::PSV::v1::RuntimeInfo>(&BasicInfo))
@@ -441,6 +452,8 @@ uint8_t DirectX::PSVRuntimeInfo::getSigOutputCount() const {
 }
 
 uint8_t DirectX::PSVRuntimeInfo::getSigPatchOrPrimCount() const {
+  if (const auto *P = std::get_if<dxbc::PSV::v3::RuntimeInfo>(&BasicInfo))
+    return P->SigPatchOrPrimElements;
   if (const auto *P = std::get_if<dxbc::PSV::v2::RuntimeInfo>(&BasicInfo))
     return P->SigPatchOrPrimElements;
   if (const auto *P = std::get_if<dxbc::PSV::v1::RuntimeInfo>(&BasicInfo))
diff --git a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp
index 09a5e41c7123..f3a518df3175 100644
--- a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp
+++ b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp
@@ -198,8 +198,9 @@ void DXContainerWriter::writeParts(raw_ostream &OS) {
       if (!P.Info.has_value())
         continue;
       mcdxbc::PSVRuntimeInfo PSV;
-      memcpy(&PSV.BaseData, &P.Info->Info, sizeof(dxbc::PSV::v2::RuntimeInfo));
+      memcpy(&PSV.BaseData, &P.Info->Info, sizeof(dxbc::PSV::v3::RuntimeInfo));
       PSV.Resources = P.Info->Resources;
+      PSV.EntryName = P.Info->EntryName;
 
       for (auto El : P.Info->SigInputElements)
         PSV.InputElements.push_back(mcdxbc::PSVSignatureElement{
diff --git a/llvm/lib/ObjectYAML/DXContainerYAML.cpp b/llvm/lib/ObjectYAML/DXContainerYAML.cpp
index a6871e7855e4..38063670aee6 100644
--- a/llvm/lib/ObjectYAML/DXContainerYAML.cpp
+++ b/llvm/lib/ObjectYAML/DXContainerYAML.cpp
@@ -74,6 +74,16 @@ DXContainerYAML::PSVInfo::PSVInfo(const dxbc::PSV::v2::RuntimeInfo *P)
   memcpy(&Info, P, sizeof(dxbc::PSV::v2::RuntimeInfo));
 }
 
+DXContainerYAML::PSVInfo::PSVInfo(const dxbc::PSV::v3::RuntimeInfo *P,
+                                  StringRef StringTable)
+    : Version(3),
+      EntryName(StringTable.substr(P->EntryNameOffset,
+                                   StringTable.find('\0', P->EntryNameOffset) -
+                                       P->EntryNameOffset)) {
+  memset(&Info, 0, sizeof(Info));
+  memcpy(&Info, P, sizeof(dxbc::PSV::v3::RuntimeInfo));
+}
+
 namespace yaml {
 
 void MappingTraits<DXContainerYAML::VersionTuple>::mapping(
@@ -348,6 +358,11 @@ void DXContainerYAML::PSVInfo::mapInfoForVersion(yaml::IO &IO) {
   IO.mapRequired("NumThreadsX", Info.NumThreadsX);
   IO.mapRequired("NumThreadsY", Info.NumThreadsY);
   IO.mapRequired("NumThreadsZ", Info.NumThreadsZ);
+
+  if (Version == 2)
+    return;
+
+  IO.mapRequired("EntryName", EntryName);
 }
 
 } // namespace llvm
diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml
new file mode 100644
index 000000000000..09885bd529f0
--- /dev/null
+++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml
@@ -0,0 +1,97 @@
+# RUN: yaml2obj %s | obj2yaml | FileCheck %s
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+  PartCount:       2
+Parts:
+  - Name:            PSV0
+    Size:            144
+    PSVInfo:
+      Version:         3
+      ShaderStage:     14
+      PayloadSizeInBytes: 4092
+      MinimumWaveLaneCount: 0
+      MaximumWaveLaneCount: 4294967295
+      UsesViewID:      0
+      SigInputVectors: 0
+      SigOutputVectors: [ 8, 16, 32, 64 ]
+      NumThreadsX:     512
+      NumThreadsY:     1024
+      NumThreadsZ:     2048
+      EntryName: ASEntry
+      ResourceStride:       24
+      Resources:
+        - Type:            1
+          Space:           2
+          LowerBound:      3
+          UpperBound:      4
+          Kind:            5
+          Flags:           6
+        - Type:            128
+          Space:           32768
+          LowerBound:      8388608
+          UpperBound:      2147483648
+          Kind:            65535
+          Flags:           16776960
+      SigInputElements: []
+      SigOutputElements: []
+      SigPatchOrPrimElements: []
+      InputOutputMap:
+        - [  ]
+        - [  ]
+        - [  ]
+        - [  ]
+  - Name:            DXIL
+    Size:            24
+    Program:
+      MajorVersion:    6
+      MinorVersion:    0
+      ShaderKind:      14
+      Size:            6
+      DXILMajorVersion: 0
+      DXILMinorVersion: 1
+      DXILSize:        0
+...
+
+# CHECK: Name:            PSV0
+# CHECK: PSVInfo:
+# CHECK-NEXT: Version:         3
+# CHECK-NEXT: ShaderStage:     14
+# CHECK-NEXT: PayloadSizeInBytes: 4092
+# CHECK-NEXT: MinimumWaveLaneCount: 0
+# CHECK-NEXT: MaximumWaveLaneCount: 4294967295
+# CHECK-NEXT: UsesViewID:      0
+# CHECK-NEXT: SigInputVectors: 0
+# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ]
+# CHECK-NEXT: NumThreadsX:     512
+# CHECK-NEXT: NumThreadsY:     1024
+# CHECK-NEXT: NumThreadsZ:     2048
+# CHECK-NEXT: EntryName: ASEntry
+# CHECK-NEXT: ResourceStride: 24
+# CHECK-NEXT: Resources:
+# CHECK-NEXT: - Type:            1
+# CHECK-NEXT: Space:           2
+# CHECK-NEXT: LowerBound:      3
+# CHECK-NEXT: UpperBound:      4
+# CHECK-NEXT: Kind:            5
+# CHECK-NEXT: Flags:           6
+# CHECK-NEXT: - Type:            128
+# CHECK-NEXT: Space:           32768
+# CHECK-NEXT: LowerBound:      8388608
+# CHECK-NEXT: UpperBound:      2147483648
+# CHECK-NEXT: Kind:            65535
+# CHECK-NEXT: Flags:           16776960
+# CHECK-NEXT: SigInputElements: []
+# CHECK-NEXT: SigOutputElements: []
+# CHECK-NEXT: SigPatchOrPrimElements: []
+# CHECK-NEXT: InputOutputMap:
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT: Name
diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml
new file mode 100644
index 000000000000..ee6fb112c772
--- /dev/null
+++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml
@@ -0,0 +1,95 @@
+# RUN: yaml2obj %s | obj2yaml | FileCheck %s
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+  PartCount:       2
+Parts:
+  - Name:            PSV0
+    Size:            144
+    PSVInfo:
+      Version:         3
+      ShaderStage:     5
+      MinimumWaveLaneCount: 0
+      MaximumWaveLaneCount: 4294967295
+      UsesViewID:      0
+      SigInputVectors: 0
+      SigOutputVectors: [ 8, 16, 32, 64 ]
+      NumThreadsX:     512
+      NumThreadsY:     1024
+      NumThreadsZ:     2048
+      EntryName: CSEntry
+      ResourceStride:       24
+      Resources:
+        - Type:            1
+          Space:           2
+          LowerBound:      3
+          UpperBound:      4
+          Kind:            5
+          Flags:           6
+        - Type:            128
+          Space:           32768
+          LowerBound:      8388608
+          UpperBound:      2147483648
+          Kind:            65535
+          Flags:           16776960
+      SigInputElements: []
+      SigOutputElements: []
+      SigPatchOrPrimElements: []
+      InputOutputMap:
+        - [  ]
+        - [  ]
+        - [  ]
+        - [  ]
+  - Name:            DXIL
+    Size:            24
+    Program:
+      MajorVersion:    6
+      MinorVersion:    0
+      ShaderKind:      5
+      Size:            6
+      DXILMajorVersion: 0
+      DXILMinorVersion: 1
+      DXILSize:        0
+...
+
+# CHECK: Name:            PSV0
+# CHECK: PSVInfo:
+# CHECK-NEXT: Version:         3
+# CHECK-NEXT: ShaderStage:     5
+# CHECK-NEXT: MinimumWaveLaneCount: 0
+# CHECK-NEXT: MaximumWaveLaneCount: 4294967295
+# CHECK-NEXT: UsesViewID:      0
+# CHECK-NEXT: SigInputVectors: 0
+# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ]
+# CHECK-NEXT: NumThreadsX:     512
+# CHECK-NEXT: NumThreadsY:     1024
+# CHECK-NEXT: NumThreadsZ:     2048
+# CHECK-NEXT: EntryName: CSEntry
+# CHECK-NEXT: ResourceStride: 24
+# CHECK-NEXT: Resources:
+# CHECK-NEXT: - Type:            1
+# CHECK-NEXT: Space:           2
+# CHECK-NEXT: LowerBound:      3
+# CHECK-NEXT: UpperBound:      4
+# CHECK-NEXT: Kind:            5
+# CHECK-NEXT: Flags:           6
+# CHECK-NEXT: - Type:            128
+# CHECK-NEXT: Space:           32768
+# CHECK-NEXT: LowerBound:      8388608
+# CHECK-NEXT: UpperBound:      2147483648
+# CHECK-NEXT: Kind:            65535
+# CHECK-NEXT: Flags:           16776960
+# CHECK-NEXT: SigInputElements: []
+# CHECK-NEXT: SigOutputElements: []
+# CHECK-NEXT: SigPatchOrPrimElements: []
+# CHECK-NEXT: InputOutputMap:
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT: Name
diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml
new file mode 100644
index 000000000000..dd367deae88e
--- /dev/null
+++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml
@@ -0,0 +1,105 @@
+# RUN: yaml2obj %s | obj2yaml | FileCheck %s
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+  PartCount:       2
+Parts:
+  - Name:            PSV0
+    Size:            144
+    PSVInfo:
+      Version:         3
+      ShaderStage:     4
+      InputControlPointCount: 1024
+      OutputPositionPresent: 1
+      TessellatorDomain: 2056
+      MinimumWaveLaneCount: 0
+      MaximumWaveLaneCount: 4294967295
+      UsesViewID:      0
+      SigPatchConstOrPrimVectors:  0
+      SigInputVectors: 0
+      SigOutputVectors: [ 0, 16, 32, 64 ]
+      NumThreadsX:     512
+      NumThreadsY:     1024
+      NumThreadsZ:     2048
+      EntryName: DSEntry
+      ResourceStride:       24
+      Resources:
+        - Type:            1
+          Space:           2
+          LowerBound:      3
+          UpperBound:      4
+          Kind:            5
+          Flags:           6
+        - Type:            128
+          Space:           32768
+          LowerBound:      8388608
+          UpperBound:      2147483648
+          Kind:            65535
+          Flags:           16776960
+      SigInputElements: []
+      SigOutputElements: []
+      SigPatchOrPrimElements: []
+      InputOutputMap:
+        - [  ]
+        - [  ]
+        - [  ]
+        - [  ]
+      PatchOutputMap: []
+  - Name:            DXIL
+    Size:            24
+    Program:
+      MajorVersion:    6
+      MinorVersion:    0
+      ShaderKind:      4
+      Size:            6
+      DXILMajorVersion: 0
+      DXILMinorVersion: 1
+      DXILSize:        0
+...
+
+# CHECK: Name:            PSV0
+# CHECK: PSVInfo:
+# CHECK-NEXT: Version:         3
+# CHECK-NEXT: ShaderStage:     4
+# CHECK-NEXT: InputControlPointCount: 1024
+# CHECK-NEXT: OutputPositionPresent: 1
+# CHECK-NEXT: TessellatorDomain: 2056
+# CHECK-NEXT: MinimumWaveLaneCount: 0
+# CHECK-NEXT: MaximumWaveLaneCount: 4294967295
+# CHECK-NEXT: UsesViewID:      0
+# CHECK-NEXT: SigPatchConstOrPrimVectors:  0
+# CHECK-NEXT: SigInputVectors: 0
+# CHECK-NEXT: SigOutputVectors: [ 0, 16, 32, 64 ]
+# CHECK-NEXT: NumThreadsX:     512
+# CHECK-NEXT: NumThreadsY:     1024
+# CHECK-NEXT: NumThreadsZ:     2048
+# CHECK-NEXT: EntryName: DSEntry
+# CHECK-NEXT: ResourceStride: 24
+# CHECK-NEXT: Resources:
+# CHECK-NEXT: - Type:            1
+# CHECK-NEXT: Space:           2
+# CHECK-NEXT: LowerBound:      3
+# CHECK-NEXT: UpperBound:      4
+# CHECK-NEXT: Kind:            5
+# CHECK-NEXT: Flags:           6
+# CHECK-NEXT: - Type:            128
+# CHECK-NEXT: Space:           32768
+# CHECK-NEXT: LowerBound:      8388608
+# CHECK-NEXT: UpperBound:      2147483648
+# CHECK-NEXT: Kind:            65535
+# CHECK-NEXT: Flags:           16776960
+# CHECK-NEXT: SigInputElements: []
+# CHECK-NEXT: SigOutputElements: []
+# CHECK-NEXT: SigPatchOrPrimElements: []
+# CHECK-NEXT: InputOutputMap:
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT: PatchOutputMap: [  ]
+# CHECK-NEXT: Name
diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml
new file mode 100644
index 000000000000..4c7680b63b02
--- /dev/null
+++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml
@@ -0,0 +1,105 @@
+# RUN: yaml2obj %s | obj2yaml | FileCheck %s
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+  PartCount:       2
+Parts:
+  - Name:            PSV0
+    Size:            144
+    PSVInfo:
+      Version:         3
+      ShaderStage:     2
+      InputPrimitive: 1024
+      OutputTopology: 4096
+      OutputStreamMask: 2056
+      OutputPositionPresent: 1
+      MinimumWaveLaneCount: 0
+      MaximumWaveLaneCount: 4294967295
+      UsesViewID:      0
+      MaxVertexCount:  4096
+      SigInputVectors: 0
+      SigOutputVectors: [ 8, 16, 32, 64 ]
+      NumThreadsX:     512
+      NumThreadsY:     1024
+      NumThreadsZ:     2048
+      EntryName: GSEntry
+      ResourceStride:       24
+      Resources:
+        - Type:            1
+          Space:           2
+          LowerBound:      3
+          UpperBound:      4
+          Kind:            5
+          Flags:           6
+        - Type:            128
+          Space:           32768
+          LowerBound:      8388608
+          UpperBound:      2147483648
+          Kind:            65535
+          Flags:           16776960
+      SigInputElements: []
+      SigOutputElements: []
+      SigPatchOrPrimElements: []
+      InputOutputMap:
+        - [  ]
+        - [  ]
+        - [  ]
+        - [  ]
+  - Name:            DXIL
+    Size:            24
+    Program:
+      MajorVersion:    6
+      MinorVersion:    0
+      ShaderKind:      2
+      Size:            6
+      DXILMajorVersion: 0
+      DXILMinorVersion: 1
+      DXILSize:        0
+...
+
+# CHECK: Name:            PSV0
+# CHECK: PSVInfo:
+# CHECK-NEXT: Version:         3
+# CHECK-NEXT: ShaderStage:     2
+# CHECK-NEXT: InputPrimitive: 1024
+# CHECK-NEXT: OutputTopology: 4096
+# CHECK-NEXT: OutputStreamMask: 2056
+# CHECK-NEXT: OutputPositionPresent: 1
+# CHECK-NEXT: MinimumWaveLaneCount: 0
+# CHECK-NEXT: MaximumWaveLaneCount: 4294967295
+# CHECK-NEXT: UsesViewID:      0
+# CHECK-NEXT: MaxVertexCount:  4096
+# CHECK-NEXT: SigInputVectors: 0
+# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ]
+# CHECK-NEXT: NumThreadsX:     512
+# CHECK-NEXT: NumThreadsY:     1024
+# CHECK-NEXT: NumThreadsZ:     2048
+# CHECK-NEXT: EntryName: GSEntry
+# CHECK-NEXT: ResourceStride: 24
+# CHECK-NEXT: Resources:
+# CHECK-NEXT: - Type:            1
+# CHECK-NEXT: Space:           2
+# CHECK-NEXT: LowerBound:      3
+# CHECK-NEXT: UpperBound:      4
+# CHECK-NEXT: Kind:            5
+# CHECK-NEXT: Flags:           6
+# CHECK-NEXT: - Type:            128
+# CHECK-NEXT: Space:           32768
+# CHECK-NEXT: LowerBound:      8388608
+# CHECK-NEXT: UpperBound:      2147483648
+# CHECK-NEXT: Kind:            65535
+# CHECK-NEXT: Flags:           16776960
+# CHECK-NEXT: SigInputElements: []
+# CHECK-NEXT: SigOutputElements: []
+# CHECK-NEXT: SigPatchOrPrimElements: []
+# CHECK-NEXT: InputOutputMap:
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT: Name
diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml
new file mode 100644
index 000000000000..3bbad8a9b0ee
--- /dev/null
+++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml
@@ -0,0 +1,107 @@
+# RUN: yaml2obj %s | obj2yaml | FileCheck %s
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+  PartCount:       2
+Parts:
+  - Name:            PSV0
+    Size:            144
+    PSVInfo:
+      Version:         3
+      ShaderStage:     3
+      InputControlPointCount: 1024
+      OutputControlPointCount: 4096
+      TessellatorDomain: 2056
+      TessellatorOutputPrimitive: 8192
+      MinimumWaveLaneCount: 0
+      MaximumWaveLaneCount: 4294967295
+      UsesViewID:      0
+      SigPatchConstOrPrimVectors:  0
+      SigInputVectors: 0
+      SigOutputVectors: [ 0, 16, 32, 64 ]
+      NumThreadsX:     512
+      NumThreadsY:     1024
+      NumThreadsZ:     2048
+      EntryName: HSEntry
+      ResourceStride:       24
+      Resources:
+        - Type:            1
+          Space:           2
+          LowerBound:      3
+          UpperBound:      4
+          Kind:            5
+          Flags:           6
+        - Type:            128
+          Space:           32768
+          LowerBound:      8388608
+          UpperBound:      2147483648
+          Kind:            65535
+          Flags:           16776960
+      SigInputElements: []
+      SigOutputElements: []
+      SigPatchOrPrimElements: []
+      InputOutputMap:
+        - [  ]
+        - [  ]
+        - [  ]
+        - [  ]
+      InputPatchMap: []
+  - Name:            DXIL
+    Size:            24
+    Program:
+      MajorVersion:    6
+      MinorVersion:    0
+      ShaderKind:      3
+      Size:            6
+      DXILMajorVersion: 0
+      DXILMinorVersion: 1
+      DXILSize:        0
+...
+
+# CHECK: Name:            PSV0
+# CHECK: PSVInfo:
+# CHECK-NEXT: Version:         3
+# CHECK-NEXT: ShaderStage:     3
+# CHECK-NEXT: InputControlPointCount: 1024
+# CHECK-NEXT: OutputControlPointCount: 4096
+# CHECK-NEXT: TessellatorDomain: 2056
+# CHECK-NEXT: TessellatorOutputPrimitive: 8192
+# CHECK-NEXT: MinimumWaveLaneCount: 0
+# CHECK-NEXT: MaximumWaveLaneCount: 4294967295
+# CHECK-NEXT: UsesViewID:      0
+# CHECK-NEXT: SigPatchConstOrPrimVectors:  0
+# CHECK-NEXT: SigInputVectors: 0
+# CHECK-NEXT: SigOutputVectors: [ 0, 16, 32, 64 ]
+# CHECK-NEXT: NumThreadsX:     512
+# CHECK-NEXT: NumThreadsY:     1024
+# CHECK-NEXT: NumThreadsZ:     2048
+# CHECK-NEXT: EntryName: HSEntry
+# CHECK-NEXT: ResourceStride: 24
+# CHECK-NEXT: Resources:
+# CHECK-NEXT: - Type:            1
+# CHECK-NEXT: Space:           2
+# CHECK-NEXT: LowerBound:      3
+# CHECK-NEXT: UpperBound:      4
+# CHECK-NEXT: Kind:            5
+# CHECK-NEXT: Flags:           6
+# CHECK-NEXT: - Type:            128
+# CHECK-NEXT: Space:           32768
+# CHECK-NEXT: LowerBound:      8388608
+# CHECK-NEXT: UpperBound:      2147483648
+# CHECK-NEXT: Kind:            65535
+# CHECK-NEXT: Flags:           16776960
+# CHECK-NEXT: SigInputElements: []
+# CHECK-NEXT: SigOutputElements: []
+# CHECK-NEXT: SigPatchOrPrimElements: []
+# CHECK-NEXT: InputOutputMap:
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT: InputPatchMap: [  ]
+# CHECK-NEXT: Name
diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml
new file mode 100644
index 000000000000..c5ea1fcf0780
--- /dev/null
+++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml
@@ -0,0 +1,109 @@
+# RUN: yaml2obj %s | obj2yaml | FileCheck %s
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+  PartCount:       2
+Parts:
+  - Name:            PSV0
+    Size:            144
+    PSVInfo:
+      Version:         3
+      ShaderStage:     13
+      GroupSharedBytesUsed: 1024
+      GroupSharedBytesDependentOnViewID: 2056
+      PayloadSizeInBytes: 4092
+      MaxOutputVertices: 8196
+      MaxOutputPrimitives: 4092
+      MinimumWaveLaneCount: 0
+      MaximumWaveLaneCount: 4294967295
+      UsesViewID:      0
+      SigPrimVectors:  128
+      MeshOutputTopology: 16
+      SigInputVectors: 0
+      SigOutputVectors: [ 8, 16, 32, 64 ]
+      NumThreadsX:     512
+      NumThreadsY:     1024
+      NumThreadsZ:     2048
+      EntryName: MSEntry
+      ResourceStride:       24
+      Resources:
+        - Type:            1
+          Space:           2
+          LowerBound:      3
+          UpperBound:      4
+          Kind:            5
+          Flags:           6
+        - Type:            128
+          Space:           32768
+          LowerBound:      8388608
+          UpperBound:      2147483648
+          Kind:            65535
+          Flags:           16776960
+      SigInputElements: []
+      SigOutputElements: []
+      SigPatchOrPrimElements: []
+      InputOutputMap:
+        - [  ]
+        - [  ]
+        - [  ]
+        - [  ]
+  - Name:            DXIL
+    Size:            24
+    Program:
+      MajorVersion:    6
+      MinorVersion:    0
+      ShaderKind:      13
+      Size:            6
+      DXILMajorVersion: 0
+      DXILMinorVersion: 1
+      DXILSize:        0
+...
+
+# CHECK: Name:            PSV0
+# CHECK: PSVInfo:
+# CHECK-NEXT: Version:         3
+# CHECK-NEXT: ShaderStage:     13
+# CHECK-NEXT: GroupSharedBytesUsed: 1024
+# CHECK-NEXT: GroupSharedBytesDependentOnViewID: 2056
+# CHECK-NEXT: PayloadSizeInBytes: 4092
+# CHECK-NEXT: MaxOutputVertices: 8196
+# CHECK-NEXT: MaxOutputPrimitives: 4092
+# CHECK-NEXT: MinimumWaveLaneCount: 0
+# CHECK-NEXT: MaximumWaveLaneCount: 4294967295
+# CHECK-NEXT: UsesViewID:      0
+# CHECK-NEXT: SigPrimVectors:  128
+# CHECK-NEXT: MeshOutputTopology: 16
+# CHECK-NEXT: SigInputVectors: 0
+# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ]
+# CHECK-NEXT: NumThreadsX:     512
+# CHECK-NEXT: NumThreadsY:     1024
+# CHECK-NEXT: NumThreadsZ:     2048
+# CHECK-NEXT: EntryName: MSEntry
+# CHECK-NEXT: ResourceStride: 24
+# CHECK-NEXT: Resources:
+# CHECK-NEXT: - Type:            1
+# CHECK-NEXT: Space:           2
+# CHECK-NEXT: LowerBound:      3
+# CHECK-NEXT: UpperBound:      4
+# CHECK-NEXT: Kind:            5
+# CHECK-NEXT: Flags:           6
+# CHECK-NEXT: - Type:            128
+# CHECK-NEXT: Space:           32768
+# CHECK-NEXT: LowerBound:      8388608
+# CHECK-NEXT: UpperBound:      2147483648
+# CHECK-NEXT: Kind:            65535
+# CHECK-NEXT: Flags:           16776960
+# CHECK-NEXT: SigInputElements: []
+# CHECK-NEXT: SigOutputElements: []
+# CHECK-NEXT: SigPatchOrPrimElements: []
+# CHECK-NEXT: InputOutputMap:
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT: Name
diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml
new file mode 100644
index 000000000000..b28d5ec8074d
--- /dev/null
+++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml
@@ -0,0 +1,99 @@
+# RUN: yaml2obj %s | obj2yaml | FileCheck %s
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+  PartCount:       2
+Parts:
+  - Name:            PSV0
+    Size:            144
+    PSVInfo:
+      Version:         3
+      ShaderStage:     0
+      DepthOutput:     7
+      SampleFrequency: 96
+      MinimumWaveLaneCount: 0
+      MaximumWaveLaneCount: 4294967295
+      UsesViewID:      0
+      SigInputVectors: 0
+      SigOutputVectors: [ 8, 16, 32, 64 ]
+      NumThreadsX:     512
+      NumThreadsY:     1024
+      NumThreadsZ:     2048
+      EntryName: PSEntry
+      ResourceStride:       24
+      Resources:
+        - Type:            1
+          Space:           2
+          LowerBound:      3
+          UpperBound:      4
+          Kind:            5
+          Flags:           6
+        - Type:            128
+          Space:           32768
+          LowerBound:      8388608
+          UpperBound:      2147483648
+          Kind:            65535
+          Flags:           16776960
+      SigInputElements: []
+      SigOutputElements: []
+      SigPatchOrPrimElements: []
+      InputOutputMap:
+        - [  ]
+        - [  ]
+        - [  ]
+        - [  ]
+  - Name:            DXIL
+    Size:            24
+    Program:
+      MajorVersion:    6
+      MinorVersion:    0
+      ShaderKind:      0
+      Size:            6
+      DXILMajorVersion: 0
+      DXILMinorVersion: 1
+      DXILSize:        0
+...
+
+# CHECK: Name:            PSV0
+# CHECK: PSVInfo:
+# CHECK-NEXT: Version:         3
+# CHECK-NEXT: ShaderStage:     0
+# CHECK-NEXT: DepthOutput:     7
+# CHECK-NEXT: SampleFrequency: 96
+# CHECK-NEXT: MinimumWaveLaneCount: 0
+# CHECK-NEXT: MaximumWaveLaneCount: 4294967295
+# CHECK-NEXT: UsesViewID:      0
+# CHECK-NEXT: SigInputVectors: 0
+# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ]
+# CHECK-NEXT: NumThreadsX:     512
+# CHECK-NEXT: NumThreadsY:     1024
+# CHECK-NEXT: NumThreadsZ:     2048
+# CHECK-NEXT: EntryName: PSEntry
+# CHECK-NEXT: ResourceStride: 24
+# CHECK-NEXT: Resources:
+# CHECK-NEXT: - Type:            1
+# CHECK-NEXT: Space:           2
+# CHECK-NEXT: LowerBound:      3
+# CHECK-NEXT: UpperBound:      4
+# CHECK-NEXT: Kind:            5
+# CHECK-NEXT: Flags:           6
+# CHECK-NEXT: - Type:            128
+# CHECK-NEXT: Space:           32768
+# CHECK-NEXT: LowerBound:      8388608
+# CHECK-NEXT: UpperBound:      2147483648
+# CHECK-NEXT: Kind:            65535
+# CHECK-NEXT: Flags:           16776960
+# CHECK-NEXT: SigInputElements: []
+# CHECK-NEXT: SigOutputElements: []
+# CHECK-NEXT: SigPatchOrPrimElements: []
+# CHECK-NEXT: InputOutputMap:
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT: Name
diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml
new file mode 100644
index 000000000000..d1fb55839931
--- /dev/null
+++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml
@@ -0,0 +1,97 @@
+# RUN: yaml2obj %s | obj2yaml | FileCheck %s
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+  PartCount:       2
+Parts:
+  - Name:            PSV0
+    Size:            144
+    PSVInfo:
+      Version:         3
+      ShaderStage:     1
+      OutputPositionPresent: 1
+      MinimumWaveLaneCount: 0
+      MaximumWaveLaneCount: 4294967295
+      UsesViewID:      0
+      SigInputVectors: 0
+      SigOutputVectors: [ 8, 16, 32, 64 ]
+      NumThreadsX:     512
+      NumThreadsY:     1024
+      NumThreadsZ:     2048
+      EntryName: VSEntry
+      ResourceStride:       24
+      Resources:
+        - Type:            1
+          Space:           2
+          LowerBound:      3
+          UpperBound:      4
+          Kind:            5
+          Flags:           6
+        - Type:            128
+          Space:           32768
+          LowerBound:      8388608
+          UpperBound:      2147483648
+          Kind:            65535
+          Flags:           16776960
+      SigInputElements: []
+      SigOutputElements: []
+      SigPatchOrPrimElements: []
+      InputOutputMap:
+        - [  ]
+        - [  ]
+        - [  ]
+        - [  ]
+  - Name:            DXIL
+    Size:            24
+    Program:
+      MajorVersion:    6
+      MinorVersion:    0
+      ShaderKind:      1
+      Size:            6
+      DXILMajorVersion: 0
+      DXILMinorVersion: 1
+      DXILSize:        0
+...
+
+# CHECK: Name:            PSV0
+# CHECK: PSVInfo:
+# CHECK-NEXT: Version:         3
+# CHECK-NEXT: ShaderStage:     1
+# CHECK-NEXT: OutputPositionPresent: 1
+# CHECK-NEXT: MinimumWaveLaneCount: 0
+# CHECK-NEXT: MaximumWaveLaneCount: 4294967295
+# CHECK-NEXT: UsesViewID:      0
+# CHECK-NEXT: SigInputVectors: 0
+# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ]
+# CHECK-NEXT: NumThreadsX:     512
+# CHECK-NEXT: NumThreadsY:     1024
+# CHECK-NEXT: NumThreadsZ:     2048
+# CHECK-NEXT: EntryName: VSEntry
+# CHECK-NEXT: ResourceStride: 24
+# CHECK-NEXT: Resources:
+# CHECK-NEXT: - Type:            1
+# CHECK-NEXT: Space:           2
+# CHECK-NEXT: LowerBound:      3
+# CHECK-NEXT: UpperBound:      4
+# CHECK-NEXT: Kind:            5
+# CHECK-NEXT: Flags:           6
+# CHECK-NEXT: - Type:            128
+# CHECK-NEXT: Space:           32768
+# CHECK-NEXT: LowerBound:      8388608
+# CHECK-NEXT: UpperBound:      2147483648
+# CHECK-NEXT: Kind:            65535
+# CHECK-NEXT: Flags:           16776960
+# CHECK-NEXT: SigInputElements: []
+# CHECK-NEXT: SigOutputElements: []
+# CHECK-NEXT: SigPatchOrPrimElements: []
+# CHECK-NEXT: InputOutputMap:
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT:   - [  ]
+# CHECK-NEXT: Name
diff --git a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp
index 69d9b9a2f784..ec4f5c74498f 100644
--- a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp
+++ b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp
@@ -99,6 +99,9 @@ dumpDXContainer(MemoryBufferRef Source) {
       else if (const auto *P =
                    std::get_if<dxbc::PSV::v2::RuntimeInfo>(&PSVInfo->getInfo()))
         NewPart.Info = DXContainerYAML::PSVInfo(P);
+      else if (const auto *P =
+                   std::get_if<dxbc::PSV::v3::RuntimeInfo>(&PSVInfo->getInfo()))
+        NewPart.Info = DXContainerYAML::PSVInfo(P, PSVInfo->getStringTable());
       NewPart.Info->ResourceStride = PSVInfo->getResourceStride();
       for (auto Res : PSVInfo->getResources())
         NewPart.Info->Resources.push_back(Res);
-- 
GitLab


From 33cf27b0a3f307f4c57e868fedb72f4d65ba72d9 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Fri, 8 Mar 2024 11:40:21 -0800
Subject: [PATCH 149/404] [RISCV][GISEL][NFC] Add typeIsLegalBoolVec and
 typeIsLegalIntOrFPVec helpers

---
 .../Target/RISCV/GISel/RISCVLegalizerInfo.cpp | 52 ++++++++++++++-----
 1 file changed, 39 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 64ae4e94a8c9..3a865fcf921b 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -40,6 +40,30 @@ static LegalityPredicate typeIsScalarFPArith(unsigned TypeIdx,
   };
 }
 
+static LegalityPredicate
+typeIsLegalIntOrFPVec(unsigned TypeIdx,
+                      std::initializer_list<LLT> IntOrFPVecTys,
+                      const RISCVSubtarget &ST) {
+  LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
+    return ST.hasVInstructions() &&
+           (Query.Types[TypeIdx].getScalarSizeInBits() != 64 ||
+            ST.hasVInstructionsI64()) &&
+           (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
+            ST.getELen() == 64);
+  };
+
+  return all(typeInSet(TypeIdx, IntOrFPVecTys), P);
+}
+
+static LegalityPredicate
+typeIsLegalBoolVec(unsigned TypeIdx, std::initializer_list<LLT> BoolVecTys,
+                   const RISCVSubtarget &ST) {
+  LegalityPredicate HasV = [=, &ST](const LegalityQuery &Query) {
+    return ST.hasVInstructions();
+  };
+  return all(typeInSet(TypeIdx, BoolVecTys), HasV);
+}
+
 RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
     : STI(ST), XLen(STI.getXLen()), sXLen(LLT::scalar(XLen)) {
   const LLT sDoubleXLen = LLT::scalar(2 * XLen);
@@ -50,6 +74,14 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
   const LLT s32 = LLT::scalar(32);
   const LLT s64 = LLT::scalar(64);
 
+  const LLT nxv1s1 = LLT::scalable_vector(1, s1);
+  const LLT nxv2s1 = LLT::scalable_vector(2, s1);
+  const LLT nxv4s1 = LLT::scalable_vector(4, s1);
+  const LLT nxv8s1 = LLT::scalable_vector(8, s1);
+  const LLT nxv16s1 = LLT::scalable_vector(16, s1);
+  const LLT nxv32s1 = LLT::scalable_vector(32, s1);
+  const LLT nxv64s1 = LLT::scalable_vector(64, s1);
+
   const LLT nxv1s8 = LLT::scalable_vector(1, s8);
   const LLT nxv2s8 = LLT::scalable_vector(2, s8);
   const LLT nxv4s8 = LLT::scalable_vector(4, s8);
@@ -78,22 +110,16 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
 
   using namespace TargetOpcode;
 
-  auto AllVecTys = {nxv1s8,   nxv2s8,  nxv4s8,  nxv8s8,  nxv16s8, nxv32s8,
-                    nxv64s8,  nxv1s16, nxv2s16, nxv4s16, nxv8s16, nxv16s16,
-                    nxv32s16, nxv1s32, nxv2s32, nxv4s32, nxv8s32, nxv16s32,
-                    nxv1s64,  nxv2s64, nxv4s64, nxv8s64};
+  auto BoolVecTys = {nxv1s1, nxv2s1, nxv4s1, nxv8s1, nxv16s1, nxv32s1, nxv64s1};
+
+  auto IntOrFPVecTys = {nxv1s8,   nxv2s8,  nxv4s8,  nxv8s8,  nxv16s8, nxv32s8,
+                        nxv64s8,  nxv1s16, nxv2s16, nxv4s16, nxv8s16, nxv16s16,
+                        nxv32s16, nxv1s32, nxv2s32, nxv4s32, nxv8s32, nxv16s32,
+                        nxv1s64,  nxv2s64, nxv4s64, nxv8s64};
 
   getActionDefinitionsBuilder({G_ADD, G_SUB, G_AND, G_OR, G_XOR})
       .legalFor({s32, sXLen})
-      .legalIf(all(
-          typeInSet(0, AllVecTys),
-          LegalityPredicate([=, &ST](const LegalityQuery &Query) {
-            return ST.hasVInstructions() &&
-                   (Query.Types[0].getScalarSizeInBits() != 64 ||
-                    ST.hasVInstructionsI64()) &&
-                   (Query.Types[0].getElementCount().getKnownMinValue() != 1 ||
-                    ST.getELen() == 64);
-          })))
+      .legalIf(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST))
       .widenScalarToNextPow2(0)
       .clampScalar(0, s32, sXLen);
 
-- 
GitLab


From ea798a79000f5df53b469ecb491d7d21479d6f24 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Tue, 5 Mar 2024 10:48:18 -0800
Subject: [PATCH 150/404] [RISCV][GISEL] Legalize and regbankselect vector
 typed G_IMPLICIT_DEF

---
 .../Target/RISCV/GISel/RISCVLegalizerInfo.cpp |   3 +
 .../RISCV/GISel/RISCVRegisterBankInfo.cpp     |   8 +-
 .../legalizer/rvv/legalize-implicit-def.mir   | 410 +++++++++++++++++
 .../regbankselect/rvv/implicit-def.mir        | 425 ++++++++++++++++++
 4 files changed, 845 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-implicit-def.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/implicit-def.mir

diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 3a865fcf921b..4b47f0b8bc6a 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -217,8 +217,11 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
     ConstantActions.customFor({s64});
   ConstantActions.widenScalarToNextPow2(0).clampScalar(0, s32, sXLen);
 
+  // TODO: transform illegal vector types into legal vector type
   getActionDefinitionsBuilder(G_IMPLICIT_DEF)
       .legalFor({s32, sXLen, p0})
+      .legalIf(typeIsLegalBoolVec(0, BoolVecTys, ST))
+      .legalIf(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST))
       .widenScalarToNextPow2(0)
       .clampScalar(0, s32, sXLen);
 
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
index 58c971aee2f4..a922991b7338 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
@@ -343,6 +343,8 @@ RISCVRegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   }
   case TargetOpcode::G_IMPLICIT_DEF: {
     Register Dst = MI.getOperand(0).getReg();
+    LLT DstTy = MRI.getType(Dst);
+    uint64_t DstMinSize = DstTy.getSizeInBits().getKnownMinValue();
     auto Mapping = GPRValueMapping;
     // FIXME: May need to do a better job determining when to use FPRB.
     // For example, the look through COPY case:
@@ -350,7 +352,11 @@ RISCVRegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     // %1:_(s32) = COPY %0
     // $f10_d = COPY %1(s32)
     if (anyUseOnlyUseFP(Dst, MRI, TRI))
-      Mapping = getFPValueMapping(MRI.getType(Dst).getSizeInBits());
+      Mapping = getFPValueMapping(DstMinSize);
+
+    if (DstTy.isVector())
+      Mapping = getVRBValueMapping(DstMinSize);
+
     return getInstructionMapping(DefaultMappingID, /*Cost=*/1, Mapping,
                                  NumOperands);
   }
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-implicit-def.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-implicit-def.mir
new file mode 100644
index 000000000000..8ee40861ce02
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-implicit-def.mir
@@ -0,0 +1,410 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+v -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=riscv64 -mattr=+v -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            implicitdef_nxv1i1
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv1i1
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8 = COPY [[DEF]](<vscale x 1 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 1 x s1>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv2i1
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv2i1
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8 = COPY [[DEF]](<vscale x 2 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 2 x s1>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv4i1
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv4i1
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8 = COPY [[DEF]](<vscale x 4 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 4 x s1>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv8i1
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv8i1
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8 = COPY [[DEF]](<vscale x 8 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 8 x s1>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv16i1
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv16i1
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 16 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8 = COPY [[DEF]](<vscale x 16 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 16 x s1>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 16 x s1>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv32i1
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv32i1
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 32 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8 = COPY [[DEF]](<vscale x 32 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 32 x s1>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 32 x s1>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv64i1
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv64i1
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 64 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8 = COPY [[DEF]](<vscale x 64 x s1>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 64 x s1>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 64 x s1>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv1i8
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv1i8
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 1 x s8>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8 = COPY [[DEF]](<vscale x 1 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 1 x s8>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 1 x s8>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv2i8
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv2i8
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 2 x s8>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8 = COPY [[DEF]](<vscale x 2 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 2 x s8>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 2 x s8>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv4i8
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv4i8
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 4 x s8>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8 = COPY [[DEF]](<vscale x 4 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 4 x s8>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 4 x s8>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv8i8
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv8i8
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 8 x s8>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8 = COPY [[DEF]](<vscale x 8 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 8 x s8>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 8 x s8>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv16i8
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv16i8
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 16 x s8>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8m2 = COPY [[DEF]](<vscale x 16 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %0:_(<vscale x 16 x s8>) = G_IMPLICIT_DEF
+    $v8m2 = COPY %0(<vscale x 16 x s8>)
+    PseudoRET implicit $v8m2
+...
+---
+name:            implicitdef_nxv32i8
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv32i8
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 32 x s8>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8m4 = COPY [[DEF]](<vscale x 32 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %0:_(<vscale x 32 x s8>) = G_IMPLICIT_DEF
+    $v8m4 = COPY %0(<vscale x 32 x s8>)
+    PseudoRET implicit $v8m4
+...
+---
+name:            implicitdef_nxv64i8
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv64i8
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 64 x s8>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8m8 = COPY [[DEF]](<vscale x 64 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %0:_(<vscale x 64 x s8>) = G_IMPLICIT_DEF
+    $v8m8 = COPY %0(<vscale x 64 x s8>)
+    PseudoRET implicit $v8m8
+...
+---
+name:            implicitdef_nxv1i16
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv1i16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 1 x s16>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8 = COPY [[DEF]](<vscale x 1 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 1 x s16>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 1 x s16>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv2i16
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv2i16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 2 x s16>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8 = COPY [[DEF]](<vscale x 2 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 2 x s16>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 2 x s16>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv4i16
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv4i16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8 = COPY [[DEF]](<vscale x 4 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 4 x s16>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 4 x s16>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv8i16
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv8i16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 8 x s16>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8m2 = COPY [[DEF]](<vscale x 8 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %0:_(<vscale x 8 x s16>) = G_IMPLICIT_DEF
+    $v8m2 = COPY %0(<vscale x 8 x s16>)
+    PseudoRET implicit $v8m2
+...
+---
+name:            implicitdef_nxv16i16
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv16i16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 16 x s16>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8m4 = COPY [[DEF]](<vscale x 16 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %0:_(<vscale x 16 x s16>) = G_IMPLICIT_DEF
+    $v8m4 = COPY %0(<vscale x 16 x s16>)
+    PseudoRET implicit $v8m4
+...
+---
+name:            implicitdef_nxv32i16
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv32i16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 32 x s16>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8m8 = COPY [[DEF]](<vscale x 32 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %0:_(<vscale x 32 x s16>) = G_IMPLICIT_DEF
+    $v8m8 = COPY %0(<vscale x 32 x s16>)
+    PseudoRET implicit $v8m8
+...
+---
+name:            implicitdef_nxv1i32
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv1i32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 1 x s32>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8 = COPY [[DEF]](<vscale x 1 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 1 x s32>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 1 x s32>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv2i32
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv2i32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 2 x s32>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8 = COPY [[DEF]](<vscale x 2 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 2 x s32>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 2 x s32>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv4i32
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv4i32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 4 x s32>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8m2 = COPY [[DEF]](<vscale x 4 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %0:_(<vscale x 4 x s32>) = G_IMPLICIT_DEF
+    $v8m2 = COPY %0(<vscale x 4 x s32>)
+    PseudoRET implicit $v8m2
+...
+---
+name:            implicitdef_nxv8i32
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv8i32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 8 x s32>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8m4 = COPY [[DEF]](<vscale x 8 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %0:_(<vscale x 8 x s32>) = G_IMPLICIT_DEF
+    $v8m4 = COPY %0(<vscale x 8 x s32>)
+    PseudoRET implicit $v8m4
+...
+---
+name:            implicitdef_nxv16i32
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv16i32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 16 x s32>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8m8 = COPY [[DEF]](<vscale x 16 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %0:_(<vscale x 16 x s32>) = G_IMPLICIT_DEF
+    $v8m8 = COPY %0(<vscale x 16 x s32>)
+    PseudoRET implicit $v8m8
+...
+---
+name:            implicitdef_nxv1i64
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv1i64
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 1 x s64>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8 = COPY [[DEF]](<vscale x 1 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 1 x s64>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 1 x s64>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv2i64
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv2i64
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 2 x s64>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8m2 = COPY [[DEF]](<vscale x 2 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %0:_(<vscale x 2 x s64>) = G_IMPLICIT_DEF
+    $v8m2 = COPY %0(<vscale x 2 x s64>)
+    PseudoRET implicit $v8m2
+...
+---
+name:            implicitdef_nxv4i64
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv4i64
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 4 x s64>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8m4 = COPY [[DEF]](<vscale x 4 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %0:_(<vscale x 4 x s64>) = G_IMPLICIT_DEF
+    $v8m4 = COPY %0(<vscale x 4 x s64>)
+    PseudoRET implicit $v8m4
+...
+---
+name:            implicitdef_nxv8i64
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: implicitdef_nxv8i64
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 8 x s64>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $v8m8 = COPY [[DEF]](<vscale x 8 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %0:_(<vscale x 8 x s64>) = G_IMPLICIT_DEF
+    $v8m8 = COPY %0(<vscale x 8 x s64>)
+    PseudoRET implicit $v8m8
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/implicit-def.mir b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/implicit-def.mir
new file mode 100644
index 000000000000..ef1e355252e1
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/implicit-def.mir
@@ -0,0 +1,425 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+m,+v -run-pass=regbankselect \
+# RUN:   -disable-gisel-legality-check -simplify-mir -verify-machineinstrs %s \
+# RUN:   -o - | FileCheck -check-prefix=RV32I %s
+# RUN: llc -mtriple=riscv64 -mattr=+m,+v -run-pass=regbankselect \
+# RUN:   -disable-gisel-legality-check -simplify-mir -verify-machineinstrs %s \
+# RUN:   -o - | FileCheck -check-prefix=RV64I %s
+---
+name:            implicitdef_nxv1i8
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv1i8
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 1 x s8>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8 = COPY [[DEF]](<vscale x 1 x s8>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv1i8
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 1 x s8>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8 = COPY [[DEF]](<vscale x 1 x s8>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 1 x s8>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 1 x s8>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv2i8
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv2i8
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 2 x s8>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8 = COPY [[DEF]](<vscale x 2 x s8>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv2i8
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 2 x s8>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8 = COPY [[DEF]](<vscale x 2 x s8>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 2 x s8>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 2 x s8>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv4i8
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv4i8
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 4 x s8>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8 = COPY [[DEF]](<vscale x 4 x s8>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv4i8
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 4 x s8>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8 = COPY [[DEF]](<vscale x 4 x s8>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 4 x s8>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 4 x s8>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv8i8
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv8i8
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 8 x s8>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8 = COPY [[DEF]](<vscale x 8 x s8>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv8i8
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 8 x s8>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8 = COPY [[DEF]](<vscale x 8 x s8>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 8 x s8>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 8 x s8>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv16i8
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv16i8
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 16 x s8>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8m2 = COPY [[DEF]](<vscale x 16 x s8>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m2
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv16i8
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 16 x s8>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8m2 = COPY [[DEF]](<vscale x 16 x s8>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m2
+    %0:_(<vscale x 16 x s8>) = G_IMPLICIT_DEF
+    $v8m2 = COPY %0(<vscale x 16 x s8>)
+    PseudoRET implicit $v8m2
+...
+---
+name:            implicitdef_nxv32i8
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv32i8
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 32 x s8>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8m4 = COPY [[DEF]](<vscale x 32 x s8>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m4
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv32i8
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 32 x s8>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8m4 = COPY [[DEF]](<vscale x 32 x s8>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m4
+    %0:_(<vscale x 32 x s8>) = G_IMPLICIT_DEF
+    $v8m4 = COPY %0(<vscale x 32 x s8>)
+    PseudoRET implicit $v8m4
+...
+---
+name:            implicitdef_nxv64i8
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv64i8
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 64 x s8>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8m8 = COPY [[DEF]](<vscale x 64 x s8>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m8
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv64i8
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 64 x s8>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8m8 = COPY [[DEF]](<vscale x 64 x s8>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m8
+    %0:_(<vscale x 64 x s8>) = G_IMPLICIT_DEF
+    $v8m8 = COPY %0(<vscale x 64 x s8>)
+    PseudoRET implicit $v8m8
+...
+---
+name:            implicitdef_nxv1i16
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv1i16
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 1 x s16>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8 = COPY [[DEF]](<vscale x 1 x s16>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv1i16
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 1 x s16>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8 = COPY [[DEF]](<vscale x 1 x s16>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 1 x s16>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 1 x s16>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv2i16
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv2i16
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 2 x s16>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8 = COPY [[DEF]](<vscale x 2 x s16>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv2i16
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 2 x s16>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8 = COPY [[DEF]](<vscale x 2 x s16>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 2 x s16>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 2 x s16>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv4i16
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv4i16
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 4 x s16>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8 = COPY [[DEF]](<vscale x 4 x s16>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv4i16
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 4 x s16>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8 = COPY [[DEF]](<vscale x 4 x s16>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 4 x s16>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 4 x s16>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv8i16
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv8i16
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 8 x s16>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8m2 = COPY [[DEF]](<vscale x 8 x s16>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m2
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv8i16
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 8 x s16>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8m2 = COPY [[DEF]](<vscale x 8 x s16>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m2
+    %0:_(<vscale x 8 x s16>) = G_IMPLICIT_DEF
+    $v8m2 = COPY %0(<vscale x 8 x s16>)
+    PseudoRET implicit $v8m2
+...
+---
+name:            implicitdef_nxv16i16
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv16i16
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 16 x s16>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8m4 = COPY [[DEF]](<vscale x 16 x s16>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m4
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv16i16
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 16 x s16>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8m4 = COPY [[DEF]](<vscale x 16 x s16>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m4
+    %0:_(<vscale x 16 x s16>) = G_IMPLICIT_DEF
+    $v8m4 = COPY %0(<vscale x 16 x s16>)
+    PseudoRET implicit $v8m4
+...
+---
+name:            implicitdef_nxv32i16
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv32i16
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 32 x s16>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8m8 = COPY [[DEF]](<vscale x 32 x s16>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m8
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv32i16
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 32 x s16>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8m8 = COPY [[DEF]](<vscale x 32 x s16>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m8
+    %0:_(<vscale x 32 x s16>) = G_IMPLICIT_DEF
+    $v8m8 = COPY %0(<vscale x 32 x s16>)
+    PseudoRET implicit $v8m8
+...
+---
+name:            implicitdef_nxv1i32
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv1i32
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 1 x s32>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8 = COPY [[DEF]](<vscale x 1 x s32>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv1i32
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 1 x s32>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8 = COPY [[DEF]](<vscale x 1 x s32>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 1 x s32>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 1 x s32>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv2i32
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv2i32
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 2 x s32>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8 = COPY [[DEF]](<vscale x 2 x s32>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv2i32
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 2 x s32>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8 = COPY [[DEF]](<vscale x 2 x s32>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 2 x s32>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 2 x s32>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv4i32
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv4i32
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 4 x s32>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8m2 = COPY [[DEF]](<vscale x 4 x s32>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m2
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv4i32
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 4 x s32>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8m2 = COPY [[DEF]](<vscale x 4 x s32>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m2
+    %0:_(<vscale x 4 x s32>) = G_IMPLICIT_DEF
+    $v8m2 = COPY %0(<vscale x 4 x s32>)
+    PseudoRET implicit $v8m2
+...
+---
+name:            implicitdef_nxv8i32
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv8i32
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 8 x s32>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8m4 = COPY [[DEF]](<vscale x 8 x s32>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m4
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv8i32
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 8 x s32>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8m4 = COPY [[DEF]](<vscale x 8 x s32>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m4
+    %0:_(<vscale x 8 x s32>) = G_IMPLICIT_DEF
+    $v8m4 = COPY %0(<vscale x 8 x s32>)
+    PseudoRET implicit $v8m4
+...
+---
+name:            implicitdef_nxv16i32
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv16i32
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 16 x s32>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8m8 = COPY [[DEF]](<vscale x 16 x s32>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m8
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv16i32
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 16 x s32>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8m8 = COPY [[DEF]](<vscale x 16 x s32>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m8
+    %0:_(<vscale x 16 x s32>) = G_IMPLICIT_DEF
+    $v8m8 = COPY %0(<vscale x 16 x s32>)
+    PseudoRET implicit $v8m8
+...
+---
+name:            implicitdef_nxv1i64
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv1i64
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 1 x s64>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8 = COPY [[DEF]](<vscale x 1 x s64>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv1i64
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 1 x s64>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8 = COPY [[DEF]](<vscale x 1 x s64>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %0:_(<vscale x 1 x s64>) = G_IMPLICIT_DEF
+    $v8 = COPY %0(<vscale x 1 x s64>)
+    PseudoRET implicit $v8
+...
+---
+name:            implicitdef_nxv2i64
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv2i64
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 2 x s64>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8m2 = COPY [[DEF]](<vscale x 2 x s64>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m2
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv2i64
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 2 x s64>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8m2 = COPY [[DEF]](<vscale x 2 x s64>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m2
+    %0:_(<vscale x 2 x s64>) = G_IMPLICIT_DEF
+    $v8m2 = COPY %0(<vscale x 2 x s64>)
+    PseudoRET implicit $v8m2
+...
+---
+name:            implicitdef_nxv4i64
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv4i64
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 4 x s64>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8m4 = COPY [[DEF]](<vscale x 4 x s64>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m4
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv4i64
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 4 x s64>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8m4 = COPY [[DEF]](<vscale x 4 x s64>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m4
+    %0:_(<vscale x 4 x s64>) = G_IMPLICIT_DEF
+    $v8m4 = COPY %0(<vscale x 4 x s64>)
+    PseudoRET implicit $v8m4
+...
+---
+name:            implicitdef_nxv8i64
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: implicitdef_nxv8i64
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 8 x s64>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: $v8m8 = COPY [[DEF]](<vscale x 8 x s64>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m8
+    ;
+    ; RV64I-LABEL: name: implicitdef_nxv8i64
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 8 x s64>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: $v8m8 = COPY [[DEF]](<vscale x 8 x s64>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m8
+    %0:_(<vscale x 8 x s64>) = G_IMPLICIT_DEF
+    $v8m8 = COPY %0(<vscale x 8 x s64>)
+    PseudoRET implicit $v8m8
+...
-- 
GitLab


From f3f594c6b569a1b126f390c28ec0ca3704f39e93 Mon Sep 17 00:00:00 2001
From: Daniil Kovalev <dkovalev@accesssoftek.com>
Date: Mon, 25 Mar 2024 20:25:42 +0300
Subject: [PATCH 151/404] [NFC][lld] Pass `sym` as const reference to
 `GotSection`'s methods (#86481)

---
 lld/ELF/SyntheticSections.cpp | 6 +++---
 lld/ELF/SyntheticSections.h   | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index f924756ddddf..650bd6cd3900 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -628,19 +628,19 @@ GotSection::GotSection()
 }
 
 void GotSection::addConstant(const Relocation &r) { relocations.push_back(r); }
-void GotSection::addEntry(Symbol &sym) {
+void GotSection::addEntry(const Symbol &sym) {
   assert(sym.auxIdx == symAux.size() - 1);
   symAux.back().gotIdx = numEntries++;
 }
 
-bool GotSection::addTlsDescEntry(Symbol &sym) {
+bool GotSection::addTlsDescEntry(const Symbol &sym) {
   assert(sym.auxIdx == symAux.size() - 1);
   symAux.back().tlsDescIdx = numEntries;
   numEntries += 2;
   return true;
 }
 
-bool GotSection::addDynTlsEntry(Symbol &sym) {
+bool GotSection::addDynTlsEntry(const Symbol &sym) {
   assert(sym.auxIdx == symAux.size() - 1);
   symAux.back().tlsGdIdx = numEntries;
   // Global Dynamic TLS entries take two GOT slots.
diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h
index fa21b80a5a5e..02a669b01d15 100644
--- a/lld/ELF/SyntheticSections.h
+++ b/lld/ELF/SyntheticSections.h
@@ -106,9 +106,9 @@ public:
   void writeTo(uint8_t *buf) override;
 
   void addConstant(const Relocation &r);
-  void addEntry(Symbol &sym);
-  bool addTlsDescEntry(Symbol &sym);
-  bool addDynTlsEntry(Symbol &sym);
+  void addEntry(const Symbol &sym);
+  bool addTlsDescEntry(const Symbol &sym);
+  bool addDynTlsEntry(const Symbol &sym);
   bool addTlsIndex();
   uint32_t getTlsDescOffset(const Symbol &sym) const;
   uint64_t getTlsDescAddr(const Symbol &sym) const;
-- 
GitLab


From 2745f72a81aa73517cf43e8e4ddc4c91a1739a6a Mon Sep 17 00:00:00 2001
From: Alan Zhao <ayzhao@google.com>
Date: Mon, 25 Mar 2024 10:32:59 -0700
Subject: [PATCH 152/404] [libc][NFC] fix typo (#86540)

---
 libc/src/__support/File/file.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libc/src/__support/File/file.h b/libc/src/__support/File/file.h
index 2ea3843749ff..eafd3ab7d910 100644
--- a/libc/src/__support/File/file.h
+++ b/libc/src/__support/File/file.h
@@ -76,7 +76,7 @@ public:
 private:
   enum class FileOp : uint8_t { NONE, READ, WRITE, SEEK };
 
-  // Platfrom specific functions which create new file objects should initialize
+  // Platform specific functions which create new file objects should initialize
   // these fields suitably via the constructor. Typically, they should be simple
   // syscall wrappers for the corresponding functionality.
   WriteFunc *platform_write;
@@ -299,7 +299,7 @@ private:
   }
 };
 
-// The implementaiton of this function is provided by the platfrom_file
+// The implementaiton of this function is provided by the platform_file
 // library.
 ErrorOr<File *> openfile(const char *path, const char *mode);
 
-- 
GitLab


From d179176f3e3505044ad14be19ccb3459a70206a9 Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Mon, 25 Mar 2024 18:33:30 +0100
Subject: [PATCH 153/404] [libc++][format] Adds ABI tags to inline constexpr
 variables. (#86293)

This uses the macro on record types and inline constexpr variables. The
tagged declarations are very likely to change in future versions of
libc++:
- __fields are internal types used to control the formatter's parse
functions which fields to expect. Newer formatters may add new fields.
For example the filesystem::path formatter accepted in the recent Tokyo
meeting added a new 'g' flag, which differs from the 'g' type.
- The Unicode tables. The number of entries in these table likely differ
between Unicode versions. The tables contain only a part of all Unicode
properties. Typically they are stored in a 32-bit entry where some bits
contain the properties and other bits the size of the range. Changes in
the Unicode or C++ algorithms may require more properties to be
available in C++. This may affect the number of bits available in the
range. If needed, other declarations get the macro. This is mainly a
first time to review this approach.

This was originally https://reviews.llvm.org/D143494 where a new macro
_LIBCPP_HIDE_FROM_ABI_TYPE was defined. Testing revealed the existing
macro _LIBCPP_HIDE_FROM_ABI could be used. The "parts" of the macro that
do not affect records are not harmful. Based on this information the
existing macro was used and additional documentation was written.
---
 libcxx/include/__config                                   | 6 ++++++
 libcxx/include/__format/escaped_output_table.h            | 2 +-
 libcxx/include/__format/extended_grapheme_cluster_table.h | 2 +-
 libcxx/include/__format/parser_std_format_spec.h          | 3 +--
 libcxx/include/__format/width_estimation_table.h          | 2 +-
 libcxx/utils/generate_escaped_output_table.py             | 2 +-
 libcxx/utils/generate_extended_grapheme_cluster_table.py  | 2 +-
 libcxx/utils/generate_width_estimation_table.py           | 2 +-
 8 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/libcxx/include/__config b/libcxx/include/__config
index af44af203e7c..8550b1da4a27 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -805,6 +805,12 @@ typedef __char32_t char32_t;
 // the implementation of a virtual function in an ABI-incompatible way in the first place,
 // since that would be an ABI break anyway. Hence, the lack of ABI tag should not be noticeable.
 //
+// The macro can be applied to record and enum types. When the tagged type is nested in
+// a record this "parent" record needs to have the macro too. Another use case for applying
+// this macro to records and unions is to apply an ABI tag to inline constexpr variables.
+// This can be useful for inline variables that are implementation details which are expected
+// to change in the future.
+//
 // TODO: We provide a escape hatch with _LIBCPP_NO_ABI_TAG for folks who want to avoid increasing
 //       the length of symbols with an ABI tag. In practice, we should remove the escape hatch and
 //       use compression mangling instead, see https://github.com/itanium-cxx-abi/cxx-abi/issues/70.
diff --git a/libcxx/include/__format/escaped_output_table.h b/libcxx/include/__format/escaped_output_table.h
index 495a2fbc7b03..e9f4a6e4f63f 100644
--- a/libcxx/include/__format/escaped_output_table.h
+++ b/libcxx/include/__format/escaped_output_table.h
@@ -110,7 +110,7 @@ namespace __escaped_output_table {
 /// - bits [0, 10] The size of the range, allowing 2048 elements.
 /// - bits [11, 31] The lower bound code point of the range. The upper bound of
 ///   the range is lower bound + size.
-inline constexpr uint32_t __entries[893] = {
+_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[893] = {
     0x00000020,
     0x0003f821,
     0x00056800,
diff --git a/libcxx/include/__format/extended_grapheme_cluster_table.h b/libcxx/include/__format/extended_grapheme_cluster_table.h
index 9616dfecd604..48581d8a5dde 100644
--- a/libcxx/include/__format/extended_grapheme_cluster_table.h
+++ b/libcxx/include/__format/extended_grapheme_cluster_table.h
@@ -125,7 +125,7 @@ enum class __property : uint8_t {
 /// following benchmark.
 /// libcxx/benchmarks/std_format_spec_string_unicode.bench.cpp
 // clang-format off
-inline constexpr uint32_t __entries[1496] = {
+_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1496] = {
     0x00000091,
     0x00005005,
     0x00005811,
diff --git a/libcxx/include/__format/parser_std_format_spec.h b/libcxx/include/__format/parser_std_format_spec.h
index a4b47abff40d..9818f37b518e 100644
--- a/libcxx/include/__format/parser_std_format_spec.h
+++ b/libcxx/include/__format/parser_std_format_spec.h
@@ -129,8 +129,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr uint32_t __substitute_arg_id(basic_format_arg<_C
 ///
 /// They default to false so when a new field is added it needs to be opted in
 /// explicitly.
-// TODO FMT Use an ABI tag for this struct.
-struct __fields {
+struct _LIBCPP_HIDE_FROM_ABI __fields {
   uint16_t __sign_                 : 1 {false};
   uint16_t __alternate_form_       : 1 {false};
   uint16_t __zero_padding_         : 1 {false};
diff --git a/libcxx/include/__format/width_estimation_table.h b/libcxx/include/__format/width_estimation_table.h
index cfb488975d57..6309483367f1 100644
--- a/libcxx/include/__format/width_estimation_table.h
+++ b/libcxx/include/__format/width_estimation_table.h
@@ -119,7 +119,7 @@ namespace __width_estimation_table {
 /// - bits [0, 13] The size of the range, allowing 16384 elements.
 /// - bits [14, 31] The lower bound code point of the range. The upper bound of
 ///   the range is lower bound + size.
-inline constexpr uint32_t __entries[108] = {
+_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[108] = {
     0x0440005f /* 00001100 - 0000115f [   96] */, //
     0x08c68001 /* 0000231a - 0000231b [    2] */, //
     0x08ca4001 /* 00002329 - 0000232a [    2] */, //
diff --git a/libcxx/utils/generate_escaped_output_table.py b/libcxx/utils/generate_escaped_output_table.py
index 5816435c96ca..c6bde8f2411c 100755
--- a/libcxx/utils/generate_escaped_output_table.py
+++ b/libcxx/utils/generate_escaped_output_table.py
@@ -124,7 +124,7 @@ DATA_ARRAY_TEMPLATE = """
 /// - bits [0, 10] The size of the range, allowing 2048 elements.
 /// - bits [11, 31] The lower bound code point of the range. The upper bound of
 ///   the range is lower bound + size.
-inline constexpr uint32_t __entries[{size}] = {{
+_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[{size}] = {{
 {entries}}};
 
 /// At the end of the valid Unicode code points space a lot of code points are
diff --git a/libcxx/utils/generate_extended_grapheme_cluster_table.py b/libcxx/utils/generate_extended_grapheme_cluster_table.py
index b0074b58cd20..6a598399ce47 100755
--- a/libcxx/utils/generate_extended_grapheme_cluster_table.py
+++ b/libcxx/utils/generate_extended_grapheme_cluster_table.py
@@ -113,7 +113,7 @@ DATA_ARRAY_TEMPLATE = """
 /// following benchmark.
 /// libcxx/benchmarks/std_format_spec_string_unicode.bench.cpp
 // clang-format off
-inline constexpr uint32_t __entries[{size}] = {{
+_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[{size}] = {{
 {entries}}};
 // clang-format on
 
diff --git a/libcxx/utils/generate_width_estimation_table.py b/libcxx/utils/generate_width_estimation_table.py
index 76b70f121eab..918dae25fe49 100644
--- a/libcxx/utils/generate_width_estimation_table.py
+++ b/libcxx/utils/generate_width_estimation_table.py
@@ -143,7 +143,7 @@ DATA_ARRAY_TEMPLATE = """
 /// - bits [0, 13] The size of the range, allowing 16384 elements.
 /// - bits [14, 31] The lower bound code point of the range. The upper bound of
 ///   the range is lower bound + size.
-inline constexpr uint32_t __entries[{size}] = {{
+_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[{size}] = {{
 {entries}}};
 
 /// The upper bound entry of EastAsianWidth.txt.
-- 
GitLab


From 8c51ac9ddb026ca247b25b08b1d08e32b1ed58ec Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Fri, 15 Mar 2024 09:18:16 -0700
Subject: [PATCH 154/404] [RISCV][GISEL] Legalize G_SELECT for scalable vectors

---
 .../Target/RISCV/GISel/RISCVLegalizerInfo.cpp |   7 +-
 .../legalizer/rvv/legalize-select.mir         | 400 ++++++++++++++++++
 2 files changed, 405 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-select.mir

diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 4b47f0b8bc6a..8fc401da2f6b 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -231,8 +231,11 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
       .clampScalar(1, sXLen, sXLen)
       .clampScalar(0, sXLen, sXLen);
 
-  auto &SelectActions = getActionDefinitionsBuilder(G_SELECT).legalFor(
-      {{s32, sXLen}, {p0, sXLen}});
+  auto &SelectActions =
+      getActionDefinitionsBuilder(G_SELECT)
+          .legalFor({{s32, sXLen}, {p0, sXLen}})
+          .legalIf(all(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST),
+                       typeIsLegalBoolVec(1, BoolVecTys, ST)));
   if (XLen == 64 || ST.hasStdExtD())
     SelectActions.legalFor({{s64, sXLen}});
   SelectActions.widenScalarToNextPow2(0)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-select.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-select.mir
new file mode 100644
index 000000000000..6e1d4aa4d7d6
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-select.mir
@@ -0,0 +1,400 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+v -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=riscv64 -mattr=+v -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            select_nxv1i8
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv1i8
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 1 x s8>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 1 x s8>) = G_SELECT [[DEF]](<vscale x 1 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8 = COPY [[SELECT]](<vscale x 1 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 1 x s8>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 1 x s8>) = G_SELECT %1(<vscale x 1 x s1>), %2(<vscale x 1 x s8>), %2(<vscale x 1 x s8>)
+    $v8 = COPY %0(<vscale x 1 x s8>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv2i8
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv2i8
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 2 x s8>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 2 x s8>) = G_SELECT [[DEF]](<vscale x 2 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8 = COPY [[SELECT]](<vscale x 2 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 2 x s8>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 2 x s8>) = G_SELECT %1(<vscale x 2 x s1>), %2(<vscale x 2 x s8>), %2(<vscale x 2 x s8>)
+    $v8 = COPY %0(<vscale x 2 x s8>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv4i8
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv4i8
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 4 x s8>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 4 x s8>) = G_SELECT [[DEF]](<vscale x 4 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8 = COPY [[SELECT]](<vscale x 4 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 4 x s8>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 4 x s8>) = G_SELECT %1(<vscale x 4 x s1>), %2(<vscale x 4 x s8>), %2(<vscale x 4 x s8>)
+    $v8 = COPY %0(<vscale x 4 x s8>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv8i8
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv8i8
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 8 x s8>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 8 x s8>) = G_SELECT [[DEF]](<vscale x 8 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8 = COPY [[SELECT]](<vscale x 8 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 8 x s8>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 8 x s8>) = G_SELECT %1(<vscale x 8 x s1>), %2(<vscale x 8 x s8>), %2(<vscale x 8 x s8>)
+    $v8 = COPY %0(<vscale x 8 x s8>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv16i8
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv16i8
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 16 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 16 x s8>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 16 x s8>) = G_SELECT [[DEF]](<vscale x 16 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8m2 = COPY [[SELECT]](<vscale x 16 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %1:_(<vscale x 16 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 16 x s8>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 16 x s8>) = G_SELECT %1(<vscale x 16 x s1>), %2(<vscale x 16 x s8>), %2(<vscale x 16 x s8>)
+    $v8m2 = COPY %0(<vscale x 16 x s8>)
+    PseudoRET implicit $v8m2
+...
+---
+name:            select_nxv32i8
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv32i8
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 32 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 32 x s8>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 32 x s8>) = G_SELECT [[DEF]](<vscale x 32 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8m4 = COPY [[SELECT]](<vscale x 32 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %1:_(<vscale x 32 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 32 x s8>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 32 x s8>) = G_SELECT %1(<vscale x 32 x s1>), %2(<vscale x 32 x s8>), %2(<vscale x 32 x s8>)
+    $v8m4 = COPY %0(<vscale x 32 x s8>)
+    PseudoRET implicit $v8m4
+...
+---
+name:            select_nxv64i8
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv64i8
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 64 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 64 x s8>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 64 x s8>) = G_SELECT [[DEF]](<vscale x 64 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8m8 = COPY [[SELECT]](<vscale x 64 x s8>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %1:_(<vscale x 64 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 64 x s8>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 64 x s8>) = G_SELECT %1(<vscale x 64 x s1>), %2(<vscale x 64 x s8>), %2(<vscale x 64 x s8>)
+    $v8m8 = COPY %0(<vscale x 64 x s8>)
+    PseudoRET implicit $v8m8
+...
+---
+name:            select_nxv1i16
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv1i16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 1 x s16>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 1 x s16>) = G_SELECT [[DEF]](<vscale x 1 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8 = COPY [[SELECT]](<vscale x 1 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 1 x s16>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 1 x s16>) = G_SELECT %1(<vscale x 1 x s1>), %2(<vscale x 1 x s16>), %2(<vscale x 1 x s16>)
+    $v8 = COPY %0(<vscale x 1 x s16>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv2i16
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv2i16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 2 x s16>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 2 x s16>) = G_SELECT [[DEF]](<vscale x 2 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8 = COPY [[SELECT]](<vscale x 2 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 2 x s16>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 2 x s16>) = G_SELECT %1(<vscale x 2 x s1>), %2(<vscale x 2 x s16>), %2(<vscale x 2 x s16>)
+    $v8 = COPY %0(<vscale x 2 x s16>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv4i16
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv4i16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 4 x s16>) = G_SELECT [[DEF]](<vscale x 4 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8 = COPY [[SELECT]](<vscale x 4 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 4 x s16>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 4 x s16>) = G_SELECT %1(<vscale x 4 x s1>), %2(<vscale x 4 x s16>), %2(<vscale x 4 x s16>)
+    $v8 = COPY %0(<vscale x 4 x s16>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv8i16
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv8i16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 8 x s16>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 8 x s16>) = G_SELECT [[DEF]](<vscale x 8 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8m2 = COPY [[SELECT]](<vscale x 8 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %1:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 8 x s16>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 8 x s16>) = G_SELECT %1(<vscale x 8 x s1>), %2(<vscale x 8 x s16>), %2(<vscale x 8 x s16>)
+    $v8m2 = COPY %0(<vscale x 8 x s16>)
+    PseudoRET implicit $v8m2
+...
+---
+name:            select_nxv16i16
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv16i16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 16 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 16 x s16>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 16 x s16>) = G_SELECT [[DEF]](<vscale x 16 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8m4 = COPY [[SELECT]](<vscale x 16 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %1:_(<vscale x 16 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 16 x s16>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 16 x s16>) = G_SELECT %1(<vscale x 16 x s1>), %2(<vscale x 16 x s16>), %2(<vscale x 16 x s16>)
+    $v8m4 = COPY %0(<vscale x 16 x s16>)
+    PseudoRET implicit $v8m4
+...
+---
+name:            select_nxv32i16
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv32i16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 32 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 32 x s16>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 32 x s16>) = G_SELECT [[DEF]](<vscale x 32 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8m8 = COPY [[SELECT]](<vscale x 32 x s16>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %1:_(<vscale x 32 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 32 x s16>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 32 x s16>) = G_SELECT %1(<vscale x 32 x s1>), %2(<vscale x 32 x s16>), %2(<vscale x 32 x s16>)
+    $v8m8 = COPY %0(<vscale x 32 x s16>)
+    PseudoRET implicit $v8m8
+...
+---
+name:            select_nxv1i32
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv1i32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 1 x s32>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 1 x s32>) = G_SELECT [[DEF]](<vscale x 1 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8 = COPY [[SELECT]](<vscale x 1 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 1 x s32>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 1 x s32>) = G_SELECT %1(<vscale x 1 x s1>), %2(<vscale x 1 x s32>), %2(<vscale x 1 x s32>)
+    $v8 = COPY %0(<vscale x 1 x s32>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv2i32
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv2i32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 2 x s32>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 2 x s32>) = G_SELECT [[DEF]](<vscale x 2 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8 = COPY [[SELECT]](<vscale x 2 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 2 x s32>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 2 x s32>) = G_SELECT %1(<vscale x 2 x s1>), %2(<vscale x 2 x s32>), %2(<vscale x 2 x s32>)
+    $v8 = COPY %0(<vscale x 2 x s32>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv4i32
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv4i32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 4 x s32>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 4 x s32>) = G_SELECT [[DEF]](<vscale x 4 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8m2 = COPY [[SELECT]](<vscale x 4 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %1:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 4 x s32>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 4 x s32>) = G_SELECT %1(<vscale x 4 x s1>), %2(<vscale x 4 x s32>), %2(<vscale x 4 x s32>)
+    $v8m2 = COPY %0(<vscale x 4 x s32>)
+    PseudoRET implicit $v8m2
+...
+---
+name:            select_nxv8i32
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv8i32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 8 x s32>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 8 x s32>) = G_SELECT [[DEF]](<vscale x 8 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8m4 = COPY [[SELECT]](<vscale x 8 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %1:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 8 x s32>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 8 x s32>) = G_SELECT %1(<vscale x 8 x s1>), %2(<vscale x 8 x s32>), %2(<vscale x 8 x s32>)
+    $v8m4 = COPY %0(<vscale x 8 x s32>)
+    PseudoRET implicit $v8m4
+...
+---
+name:            select_nxv16i32
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv16i32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 16 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 16 x s32>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 16 x s32>) = G_SELECT [[DEF]](<vscale x 16 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8m8 = COPY [[SELECT]](<vscale x 16 x s32>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %1:_(<vscale x 16 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 16 x s32>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 16 x s32>) = G_SELECT %1(<vscale x 16 x s1>), %2(<vscale x 16 x s32>), %2(<vscale x 16 x s32>)
+    $v8m8 = COPY %0(<vscale x 16 x s32>)
+    PseudoRET implicit $v8m8
+...
+---
+name:            select_nxv1i64
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv1i64
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 1 x s64>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 1 x s64>) = G_SELECT [[DEF]](<vscale x 1 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8 = COPY [[SELECT]](<vscale x 1 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 1 x s64>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 1 x s64>) = G_SELECT %1(<vscale x 1 x s1>), %2(<vscale x 1 x s64>), %2(<vscale x 1 x s64>)
+    $v8 = COPY %0(<vscale x 1 x s64>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv2i64
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv2i64
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 2 x s64>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 2 x s64>) = G_SELECT [[DEF]](<vscale x 2 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8m2 = COPY [[SELECT]](<vscale x 2 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %1:_(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 2 x s64>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 2 x s64>) = G_SELECT %1(<vscale x 2 x s1>), %2(<vscale x 2 x s64>), %2(<vscale x 2 x s64>)
+    $v8m2 = COPY %0(<vscale x 2 x s64>)
+    PseudoRET implicit $v8m2
+...
+---
+name:            select_nxv4i64
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv4i64
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 4 x s64>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 4 x s64>) = G_SELECT [[DEF]](<vscale x 4 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8m4 = COPY [[SELECT]](<vscale x 4 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %1:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 4 x s64>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 4 x s64>) = G_SELECT %1(<vscale x 4 x s1>), %2(<vscale x 4 x s64>), %2(<vscale x 4 x s64>)
+    $v8m4 = COPY %0(<vscale x 4 x s64>)
+    PseudoRET implicit $v8m4
+...
+---
+name:            select_nxv8i64
+legalized:       false
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: select_nxv8i64
+    ; CHECK: [[DEF:%[0-9]+]]:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<vscale x 8 x s64>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<vscale x 8 x s64>) = G_SELECT [[DEF]](<vscale x 8 x s1>), [[DEF1]], [[DEF1]]
+    ; CHECK-NEXT: $v8m8 = COPY [[SELECT]](<vscale x 8 x s64>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %1:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 8 x s64>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 8 x s64>) = G_SELECT %1(<vscale x 8 x s1>), %2(<vscale x 8 x s64>), %2(<vscale x 8 x s64>)
+    $v8m8 = COPY %0(<vscale x 8 x s64>)
+    PseudoRET implicit $v8m8
+...
-- 
GitLab


From 973e9dbd57bcdfcc3e0deef37d1c37fc4fbbb856 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Fri, 15 Mar 2024 13:09:57 -0700
Subject: [PATCH 155/404] [RISCV][GISEL] Regbank select for scalable G_SELECT

---
 .../RISCV/GISel/RISCVRegisterBankInfo.cpp     |  12 +
 .../GlobalISel/regbankselect/rvv/select.mir   | 558 ++++++++++++++++++
 2 files changed, 570 insertions(+)
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/select.mir

diff --git a/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
index a922991b7338..888bcc46ea1e 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
@@ -13,6 +13,7 @@
 #include "RISCVRegisterBankInfo.h"
 #include "MCTargetDesc/RISCVMCTargetDesc.h"
 #include "RISCVSubtarget.h"
+#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterBank.h"
 #include "llvm/CodeGen/RegisterBankInfo.h"
@@ -407,6 +408,17 @@ RISCVRegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case TargetOpcode::G_SELECT: {
     LLT Ty = MRI.getType(MI.getOperand(0).getReg());
 
+    if (Ty.isVector()) {
+      auto &Sel = cast<GSelect>(MI);
+      LLT TestTy = MRI.getType(Sel.getCondReg());
+      assert(TestTy.isVector() && "Unexpected condition argument type");
+      OpdsMapping[0] = OpdsMapping[2] = OpdsMapping[3] =
+          getVRBValueMapping(Ty.getSizeInBits().getKnownMinValue());
+      OpdsMapping[1] =
+          getVRBValueMapping(TestTy.getSizeInBits().getKnownMinValue());
+      break;
+    }
+
     // Try to minimize the number of copies. If we have more floating point
     // constrained values than not, then we'll put everything on FPR. Otherwise,
     // everything has to be on GPR.
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/select.mir b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/select.mir
new file mode 100644
index 000000000000..4dc077ae6bfe
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/select.mir
@@ -0,0 +1,558 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+m,+v -run-pass=regbankselect \
+# RUN:   -disable-gisel-legality-check -simplify-mir -verify-machineinstrs %s \
+# RUN:   -o - | FileCheck -check-prefix=RV32I %s
+# RUN: llc -mtriple=riscv64 -mattr=+m,+v -run-pass=regbankselect \
+# RUN:   -disable-gisel-legality-check -simplify-mir -verify-machineinstrs %s \
+# RUN:   -o - | FileCheck -check-prefix=RV64I %s
+
+---
+name:            select_nxv1i8
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv1i8
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 1 x s8>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 1 x s8>) = G_SELECT [[DEF]](<vscale x 1 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8 = COPY [[SELECT]](<vscale x 1 x s8>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: select_nxv1i8
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 1 x s8>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 1 x s8>) = G_SELECT [[DEF]](<vscale x 1 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8 = COPY [[SELECT]](<vscale x 1 x s8>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 1 x s8>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 1 x s8>) = G_SELECT %1(<vscale x 1 x s1>), %2(<vscale x 1 x s8>), %2(<vscale x 1 x s8>)
+    $v8 = COPY %0(<vscale x 1 x s8>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv2i8
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv2i8
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 2 x s8>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 2 x s8>) = G_SELECT [[DEF]](<vscale x 2 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8 = COPY [[SELECT]](<vscale x 2 x s8>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: select_nxv2i8
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 2 x s8>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 2 x s8>) = G_SELECT [[DEF]](<vscale x 2 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8 = COPY [[SELECT]](<vscale x 2 x s8>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 2 x s8>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 2 x s8>) = G_SELECT %1(<vscale x 2 x s1>), %2(<vscale x 2 x s8>), %2(<vscale x 2 x s8>)
+    $v8 = COPY %0(<vscale x 2 x s8>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv4i8
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv4i8
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 4 x s8>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 4 x s8>) = G_SELECT [[DEF]](<vscale x 4 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8 = COPY [[SELECT]](<vscale x 4 x s8>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: select_nxv4i8
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 4 x s8>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 4 x s8>) = G_SELECT [[DEF]](<vscale x 4 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8 = COPY [[SELECT]](<vscale x 4 x s8>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 4 x s8>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 4 x s8>) = G_SELECT %1(<vscale x 4 x s1>), %2(<vscale x 4 x s8>), %2(<vscale x 4 x s8>)
+    $v8 = COPY %0(<vscale x 4 x s8>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv8i8
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv8i8
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 8 x s8>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 8 x s8>) = G_SELECT [[DEF]](<vscale x 8 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8 = COPY [[SELECT]](<vscale x 8 x s8>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: select_nxv8i8
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 8 x s8>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 8 x s8>) = G_SELECT [[DEF]](<vscale x 8 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8 = COPY [[SELECT]](<vscale x 8 x s8>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 8 x s8>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 8 x s8>) = G_SELECT %1(<vscale x 8 x s1>), %2(<vscale x 8 x s8>), %2(<vscale x 8 x s8>)
+    $v8 = COPY %0(<vscale x 8 x s8>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv16i8
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv16i8
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 16 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 16 x s8>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 16 x s8>) = G_SELECT [[DEF]](<vscale x 16 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8m2 = COPY [[SELECT]](<vscale x 16 x s8>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m2
+    ;
+    ; RV64I-LABEL: name: select_nxv16i8
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 16 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 16 x s8>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 16 x s8>) = G_SELECT [[DEF]](<vscale x 16 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8m2 = COPY [[SELECT]](<vscale x 16 x s8>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m2
+    %1:_(<vscale x 16 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 16 x s8>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 16 x s8>) = G_SELECT %1(<vscale x 16 x s1>), %2(<vscale x 16 x s8>), %2(<vscale x 16 x s8>)
+    $v8m2 = COPY %0(<vscale x 16 x s8>)
+    PseudoRET implicit $v8m2
+...
+---
+name:            select_nxv32i8
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv32i8
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 32 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 32 x s8>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 32 x s8>) = G_SELECT [[DEF]](<vscale x 32 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8m4 = COPY [[SELECT]](<vscale x 32 x s8>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m4
+    ;
+    ; RV64I-LABEL: name: select_nxv32i8
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 32 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 32 x s8>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 32 x s8>) = G_SELECT [[DEF]](<vscale x 32 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8m4 = COPY [[SELECT]](<vscale x 32 x s8>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m4
+    %1:_(<vscale x 32 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 32 x s8>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 32 x s8>) = G_SELECT %1(<vscale x 32 x s1>), %2(<vscale x 32 x s8>), %2(<vscale x 32 x s8>)
+    $v8m4 = COPY %0(<vscale x 32 x s8>)
+    PseudoRET implicit $v8m4
+...
+---
+name:            select_nxv64i8
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv64i8
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 64 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 64 x s8>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 64 x s8>) = G_SELECT [[DEF]](<vscale x 64 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8m8 = COPY [[SELECT]](<vscale x 64 x s8>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m8
+    ;
+    ; RV64I-LABEL: name: select_nxv64i8
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 64 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 64 x s8>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 64 x s8>) = G_SELECT [[DEF]](<vscale x 64 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8m8 = COPY [[SELECT]](<vscale x 64 x s8>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m8
+    %1:_(<vscale x 64 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 64 x s8>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 64 x s8>) = G_SELECT %1(<vscale x 64 x s1>), %2(<vscale x 64 x s8>), %2(<vscale x 64 x s8>)
+    $v8m8 = COPY %0(<vscale x 64 x s8>)
+    PseudoRET implicit $v8m8
+...
+---
+name:            select_nxv1i16
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv1i16
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 1 x s16>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 1 x s16>) = G_SELECT [[DEF]](<vscale x 1 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8 = COPY [[SELECT]](<vscale x 1 x s16>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: select_nxv1i16
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 1 x s16>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 1 x s16>) = G_SELECT [[DEF]](<vscale x 1 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8 = COPY [[SELECT]](<vscale x 1 x s16>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 1 x s16>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 1 x s16>) = G_SELECT %1(<vscale x 1 x s1>), %2(<vscale x 1 x s16>), %2(<vscale x 1 x s16>)
+    $v8 = COPY %0(<vscale x 1 x s16>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv2i16
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv2i16
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 2 x s16>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 2 x s16>) = G_SELECT [[DEF]](<vscale x 2 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8 = COPY [[SELECT]](<vscale x 2 x s16>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: select_nxv2i16
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 2 x s16>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 2 x s16>) = G_SELECT [[DEF]](<vscale x 2 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8 = COPY [[SELECT]](<vscale x 2 x s16>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 2 x s16>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 2 x s16>) = G_SELECT %1(<vscale x 2 x s1>), %2(<vscale x 2 x s16>), %2(<vscale x 2 x s16>)
+    $v8 = COPY %0(<vscale x 2 x s16>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv4i16
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv4i16
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 4 x s16>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 4 x s16>) = G_SELECT [[DEF]](<vscale x 4 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8 = COPY [[SELECT]](<vscale x 4 x s16>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: select_nxv4i16
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 4 x s16>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 4 x s16>) = G_SELECT [[DEF]](<vscale x 4 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8 = COPY [[SELECT]](<vscale x 4 x s16>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 4 x s16>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 4 x s16>) = G_SELECT %1(<vscale x 4 x s1>), %2(<vscale x 4 x s16>), %2(<vscale x 4 x s16>)
+    $v8 = COPY %0(<vscale x 4 x s16>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv8i16
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv8i16
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 8 x s16>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 8 x s16>) = G_SELECT [[DEF]](<vscale x 8 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8m2 = COPY [[SELECT]](<vscale x 8 x s16>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m2
+    ;
+    ; RV64I-LABEL: name: select_nxv8i16
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 8 x s16>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 8 x s16>) = G_SELECT [[DEF]](<vscale x 8 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8m2 = COPY [[SELECT]](<vscale x 8 x s16>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m2
+    %1:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 8 x s16>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 8 x s16>) = G_SELECT %1(<vscale x 8 x s1>), %2(<vscale x 8 x s16>), %2(<vscale x 8 x s16>)
+    $v8m2 = COPY %0(<vscale x 8 x s16>)
+    PseudoRET implicit $v8m2
+...
+---
+name:            select_nxv16i16
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv16i16
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 16 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 16 x s16>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 16 x s16>) = G_SELECT [[DEF]](<vscale x 16 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8m4 = COPY [[SELECT]](<vscale x 16 x s16>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m4
+    ;
+    ; RV64I-LABEL: name: select_nxv16i16
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 16 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 16 x s16>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 16 x s16>) = G_SELECT [[DEF]](<vscale x 16 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8m4 = COPY [[SELECT]](<vscale x 16 x s16>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m4
+    %1:_(<vscale x 16 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 16 x s16>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 16 x s16>) = G_SELECT %1(<vscale x 16 x s1>), %2(<vscale x 16 x s16>), %2(<vscale x 16 x s16>)
+    $v8m4 = COPY %0(<vscale x 16 x s16>)
+    PseudoRET implicit $v8m4
+...
+---
+name:            select_nxv32i16
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv32i16
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 32 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 32 x s16>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 32 x s16>) = G_SELECT [[DEF]](<vscale x 32 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8m8 = COPY [[SELECT]](<vscale x 32 x s16>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m8
+    ;
+    ; RV64I-LABEL: name: select_nxv32i16
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 32 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 32 x s16>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 32 x s16>) = G_SELECT [[DEF]](<vscale x 32 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8m8 = COPY [[SELECT]](<vscale x 32 x s16>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m8
+    %1:_(<vscale x 32 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 32 x s16>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 32 x s16>) = G_SELECT %1(<vscale x 32 x s1>), %2(<vscale x 32 x s16>), %2(<vscale x 32 x s16>)
+    $v8m8 = COPY %0(<vscale x 32 x s16>)
+    PseudoRET implicit $v8m8
+...
+---
+name:            select_nxv1i32
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv1i32
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 1 x s32>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 1 x s32>) = G_SELECT [[DEF]](<vscale x 1 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8 = COPY [[SELECT]](<vscale x 1 x s32>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: select_nxv1i32
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 1 x s32>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 1 x s32>) = G_SELECT [[DEF]](<vscale x 1 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8 = COPY [[SELECT]](<vscale x 1 x s32>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 1 x s32>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 1 x s32>) = G_SELECT %1(<vscale x 1 x s1>), %2(<vscale x 1 x s32>), %2(<vscale x 1 x s32>)
+    $v8 = COPY %0(<vscale x 1 x s32>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv2i32
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv2i32
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 2 x s32>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 2 x s32>) = G_SELECT [[DEF]](<vscale x 2 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8 = COPY [[SELECT]](<vscale x 2 x s32>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: select_nxv2i32
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 2 x s32>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 2 x s32>) = G_SELECT [[DEF]](<vscale x 2 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8 = COPY [[SELECT]](<vscale x 2 x s32>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 2 x s32>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 2 x s32>) = G_SELECT %1(<vscale x 2 x s1>), %2(<vscale x 2 x s32>), %2(<vscale x 2 x s32>)
+    $v8 = COPY %0(<vscale x 2 x s32>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv4i32
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv4i32
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 4 x s32>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 4 x s32>) = G_SELECT [[DEF]](<vscale x 4 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8m2 = COPY [[SELECT]](<vscale x 4 x s32>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m2
+    ;
+    ; RV64I-LABEL: name: select_nxv4i32
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 4 x s32>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 4 x s32>) = G_SELECT [[DEF]](<vscale x 4 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8m2 = COPY [[SELECT]](<vscale x 4 x s32>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m2
+    %1:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 4 x s32>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 4 x s32>) = G_SELECT %1(<vscale x 4 x s1>), %2(<vscale x 4 x s32>), %2(<vscale x 4 x s32>)
+    $v8m2 = COPY %0(<vscale x 4 x s32>)
+    PseudoRET implicit $v8m2
+...
+---
+name:            select_nxv8i32
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv8i32
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 8 x s32>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 8 x s32>) = G_SELECT [[DEF]](<vscale x 8 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8m4 = COPY [[SELECT]](<vscale x 8 x s32>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m4
+    ;
+    ; RV64I-LABEL: name: select_nxv8i32
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 8 x s32>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 8 x s32>) = G_SELECT [[DEF]](<vscale x 8 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8m4 = COPY [[SELECT]](<vscale x 8 x s32>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m4
+    %1:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 8 x s32>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 8 x s32>) = G_SELECT %1(<vscale x 8 x s1>), %2(<vscale x 8 x s32>), %2(<vscale x 8 x s32>)
+    $v8m4 = COPY %0(<vscale x 8 x s32>)
+    PseudoRET implicit $v8m4
+...
+---
+name:            select_nxv16i32
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv16i32
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 16 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 16 x s32>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 16 x s32>) = G_SELECT [[DEF]](<vscale x 16 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8m8 = COPY [[SELECT]](<vscale x 16 x s32>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m8
+    ;
+    ; RV64I-LABEL: name: select_nxv16i32
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 16 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 16 x s32>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 16 x s32>) = G_SELECT [[DEF]](<vscale x 16 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8m8 = COPY [[SELECT]](<vscale x 16 x s32>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m8
+    %1:_(<vscale x 16 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 16 x s32>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 16 x s32>) = G_SELECT %1(<vscale x 16 x s1>), %2(<vscale x 16 x s32>), %2(<vscale x 16 x s32>)
+    $v8m8 = COPY %0(<vscale x 16 x s32>)
+    PseudoRET implicit $v8m8
+...
+---
+name:            select_nxv1i64
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv1i64
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 1 x s64>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 1 x s64>) = G_SELECT [[DEF]](<vscale x 1 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8 = COPY [[SELECT]](<vscale x 1 x s64>)
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: select_nxv1i64
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 1 x s64>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 1 x s64>) = G_SELECT [[DEF]](<vscale x 1 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8 = COPY [[SELECT]](<vscale x 1 x s64>)
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %1:_(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 1 x s64>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 1 x s64>) = G_SELECT %1(<vscale x 1 x s1>), %2(<vscale x 1 x s64>), %2(<vscale x 1 x s64>)
+    $v8 = COPY %0(<vscale x 1 x s64>)
+    PseudoRET implicit $v8
+...
+---
+name:            select_nxv2i64
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv2i64
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 2 x s64>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 2 x s64>) = G_SELECT [[DEF]](<vscale x 2 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8m2 = COPY [[SELECT]](<vscale x 2 x s64>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m2
+    ;
+    ; RV64I-LABEL: name: select_nxv2i64
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 2 x s64>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 2 x s64>) = G_SELECT [[DEF]](<vscale x 2 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8m2 = COPY [[SELECT]](<vscale x 2 x s64>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m2
+    %1:_(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 2 x s64>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 2 x s64>) = G_SELECT %1(<vscale x 2 x s1>), %2(<vscale x 2 x s64>), %2(<vscale x 2 x s64>)
+    $v8m2 = COPY %0(<vscale x 2 x s64>)
+    PseudoRET implicit $v8m2
+...
+---
+name:            select_nxv4i64
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv4i64
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 4 x s64>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 4 x s64>) = G_SELECT [[DEF]](<vscale x 4 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8m4 = COPY [[SELECT]](<vscale x 4 x s64>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m4
+    ;
+    ; RV64I-LABEL: name: select_nxv4i64
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 4 x s64>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 4 x s64>) = G_SELECT [[DEF]](<vscale x 4 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8m4 = COPY [[SELECT]](<vscale x 4 x s64>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m4
+    %1:_(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 4 x s64>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 4 x s64>) = G_SELECT %1(<vscale x 4 x s1>), %2(<vscale x 4 x s64>), %2(<vscale x 4 x s64>)
+    $v8m4 = COPY %0(<vscale x 4 x s64>)
+    PseudoRET implicit $v8m4
+...
+---
+name:            select_nxv8i64
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv8i64
+    ; RV32I: [[DEF:%[0-9]+]]:vrb(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 8 x s64>) = G_IMPLICIT_DEF
+    ; RV32I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 8 x s64>) = G_SELECT [[DEF]](<vscale x 8 x s1>), [[DEF1]], [[DEF1]]
+    ; RV32I-NEXT: $v8m8 = COPY [[SELECT]](<vscale x 8 x s64>)
+    ; RV32I-NEXT: PseudoRET implicit $v8m8
+    ;
+    ; RV64I-LABEL: name: select_nxv8i64
+    ; RV64I: [[DEF:%[0-9]+]]:vrb(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrb(<vscale x 8 x s64>) = G_IMPLICIT_DEF
+    ; RV64I-NEXT: [[SELECT:%[0-9]+]]:vrb(<vscale x 8 x s64>) = G_SELECT [[DEF]](<vscale x 8 x s1>), [[DEF1]], [[DEF1]]
+    ; RV64I-NEXT: $v8m8 = COPY [[SELECT]](<vscale x 8 x s64>)
+    ; RV64I-NEXT: PseudoRET implicit $v8m8
+    %1:_(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    %2:_(<vscale x 8 x s64>) = G_IMPLICIT_DEF
+    %0:_(<vscale x 8 x s64>) = G_SELECT %1(<vscale x 8 x s1>), %2(<vscale x 8 x s64>), %2(<vscale x 8 x s64>)
+    $v8m8 = COPY %0(<vscale x 8 x s64>)
+    PseudoRET implicit $v8m8
+...
-- 
GitLab


From 05840c8714d798c8f2873c205aeac146b4993ae5 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Fri, 15 Mar 2024 13:09:16 -0700
Subject: [PATCH 156/404] [RISCV][GISEL] Instruction select for scalable
 G_SELECT

SelectionDAG has SELECT and VSELECT

SELECT restricts the condition operand to an i1 and the true and false operands
can be vectors. The result of a SELECT has the same type as the true and
false operands.

VSELECT has a vector condition operand and the true and false operands
must be vectors. The result of a VSELECT has a vector result.

GlobalISel has G_SELECT which has condition operand that is an i1 if the
true and false operands are scalar and a vector type with i1 elements if
the true and false operands are vector.

A G_SELECT acts like a ISD::SELECT when the operands are all scalar, and
an ISD::VSELECT when the operands are are scalar. A G_SELECT cannot act
like a ISD::SELECT with an i1 condition and vector operands because the
type system.

In this patch, we would like to take advantage of the patterns written
for SELECT and VSELECT, so we mark G_SELECT equivalent to both SELECT
and VSELECT to reuse the patterns. Since we cannot write a `G_SELECT (s1),
(vector-ty), (vector-ty)`, we don't have to worry about accidently
matching the SDAG patterns of that nature.

We will probably need a way to represent an i1 condition with vector
true and false operands in the future. That can be the topic of another
patch.
---
 .../Target/GlobalISel/SelectionDAGCompat.td   |   1 +
 .../instruction-select/rvv/select.mir         | 345 ++++++++++++++++++
 2 files changed, 346 insertions(+)
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/select.mir

diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
index 4bb9929e2cf8..3208c63fb42d 100644
--- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
+++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
@@ -90,6 +90,7 @@ def : GINodeEquiv<G_UDIVFIX, udivfix>;
 def : GINodeEquiv<G_SDIVFIXSAT, sdivfixsat>;
 def : GINodeEquiv<G_UDIVFIXSAT, udivfixsat>;
 def : GINodeEquiv<G_SELECT, select>;
+def : GINodeEquiv<G_SELECT, vselect>;
 def : GINodeEquiv<G_FNEG, fneg>;
 def : GINodeEquiv<G_FPEXT, fpextend>;
 def : GINodeEquiv<G_FPTRUNC, fpround>;
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/select.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/select.mir
new file mode 100644
index 000000000000..42bf32122870
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/select.mir
@@ -0,0 +1,345 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+v -run-pass=instruction-select -simplify-mir -verify-machineinstrs %s -o - | FileCheck -check-prefix=RV32I %s
+# RUN: llc -mtriple=riscv64 -mattr=+v -run-pass=instruction-select -simplify-mir -verify-machineinstrs %s -o - | FileCheck -check-prefix=RV64I %s
+
+---
+name:            select_nxv1i8
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv1i8
+    ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF
+    ; RV32I-NEXT: $v0 = COPY [[DEF]]
+    ; RV32I-NEXT: [[PseudoVMERGE_VVM_MF4_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_MF4 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 3 /* e8 */
+    ; RV32I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_MF4_]]
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: select_nxv1i8
+    ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF
+    ; RV64I-NEXT: $v0 = COPY [[DEF]]
+    ; RV64I-NEXT: [[PseudoVMERGE_VVM_MF4_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_MF4 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 3 /* e8 */
+    ; RV64I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_MF4_]]
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %0:vrb(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    %1:vrb(<vscale x 2 x s8>) = G_IMPLICIT_DEF
+    %2:vrb(<vscale x 2 x s8>) = G_SELECT %0(<vscale x 2 x s1>), %1, %1
+    $v8 = COPY %2(<vscale x 2 x s8>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            select_nxv4i8
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv4i8
+    ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF
+    ; RV32I-NEXT: $v0 = COPY [[DEF]]
+    ; RV32I-NEXT: [[PseudoVMERGE_VVM_M1_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_M1 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 3 /* e8 */
+    ; RV32I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_M1_]]
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: select_nxv4i8
+    ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF
+    ; RV64I-NEXT: $v0 = COPY [[DEF]]
+    ; RV64I-NEXT: [[PseudoVMERGE_VVM_M1_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_M1 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 3 /* e8 */
+    ; RV64I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_M1_]]
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %0:vrb(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    %1:vrb(<vscale x 8 x s8>) = G_IMPLICIT_DEF
+    %2:vrb(<vscale x 8 x s8>) = G_SELECT %0(<vscale x 8 x s1>), %1, %1
+    $v8 = COPY %2(<vscale x 8 x s8>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            select_nxv16i8
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv16i8
+    ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrm4 = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrm4nov0 = IMPLICIT_DEF
+    ; RV32I-NEXT: $v0 = COPY [[DEF]]
+    ; RV32I-NEXT: [[PseudoVMERGE_VVM_M4_:%[0-9]+]]:vrm4nov0 = PseudoVMERGE_VVM_M4 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 3 /* e8 */
+    ; RV32I-NEXT: $v8m4 = COPY [[PseudoVMERGE_VVM_M4_]]
+    ; RV32I-NEXT: PseudoRET implicit $v8m4
+    ;
+    ; RV64I-LABEL: name: select_nxv16i8
+    ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrm4 = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrm4nov0 = IMPLICIT_DEF
+    ; RV64I-NEXT: $v0 = COPY [[DEF]]
+    ; RV64I-NEXT: [[PseudoVMERGE_VVM_M4_:%[0-9]+]]:vrm4nov0 = PseudoVMERGE_VVM_M4 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 3 /* e8 */
+    ; RV64I-NEXT: $v8m4 = COPY [[PseudoVMERGE_VVM_M4_]]
+    ; RV64I-NEXT: PseudoRET implicit $v8m4
+    %0:vrb(<vscale x 32 x s1>) = G_IMPLICIT_DEF
+    %1:vrb(<vscale x 32 x s8>) = G_IMPLICIT_DEF
+    %2:vrb(<vscale x 32 x s8>) = G_SELECT %0(<vscale x 32 x s1>), %1, %1
+    $v8m4 = COPY %2(<vscale x 32 x s8>)
+    PseudoRET implicit $v8m4
+
+...
+---
+name:            select_nxv64i8
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv64i8
+    ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF
+    ; RV32I-NEXT: $v0 = COPY [[DEF]]
+    ; RV32I-NEXT: [[PseudoVMERGE_VVM_MF4_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_MF4 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 4 /* e16 */
+    ; RV32I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_MF4_]]
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: select_nxv64i8
+    ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF
+    ; RV64I-NEXT: $v0 = COPY [[DEF]]
+    ; RV64I-NEXT: [[PseudoVMERGE_VVM_MF4_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_MF4 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 4 /* e16 */
+    ; RV64I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_MF4_]]
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %0:vrb(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    %1:vrb(<vscale x 1 x s16>) = G_IMPLICIT_DEF
+    %2:vrb(<vscale x 1 x s16>) = G_SELECT %0(<vscale x 1 x s1>), %1, %1
+    $v8 = COPY %2(<vscale x 1 x s16>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            select_nxv2i16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv2i16
+    ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF
+    ; RV32I-NEXT: $v0 = COPY [[DEF]]
+    ; RV32I-NEXT: [[PseudoVMERGE_VVM_M1_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_M1 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 4 /* e16 */
+    ; RV32I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_M1_]]
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: select_nxv2i16
+    ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF
+    ; RV64I-NEXT: $v0 = COPY [[DEF]]
+    ; RV64I-NEXT: [[PseudoVMERGE_VVM_M1_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_M1 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 4 /* e16 */
+    ; RV64I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_M1_]]
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %0:vrb(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    %1:vrb(<vscale x 4 x s16>) = G_IMPLICIT_DEF
+    %2:vrb(<vscale x 4 x s16>) = G_SELECT %0(<vscale x 4 x s1>), %1, %1
+    $v8 = COPY %2(<vscale x 4 x s16>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            select_nxv8i16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv8i16
+    ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrm4 = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrm4nov0 = IMPLICIT_DEF
+    ; RV32I-NEXT: $v0 = COPY [[DEF]]
+    ; RV32I-NEXT: [[PseudoVMERGE_VVM_M4_:%[0-9]+]]:vrm4nov0 = PseudoVMERGE_VVM_M4 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 4 /* e16 */
+    ; RV32I-NEXT: $v8m4 = COPY [[PseudoVMERGE_VVM_M4_]]
+    ; RV32I-NEXT: PseudoRET implicit $v8m4
+    ;
+    ; RV64I-LABEL: name: select_nxv8i16
+    ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrm4 = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrm4nov0 = IMPLICIT_DEF
+    ; RV64I-NEXT: $v0 = COPY [[DEF]]
+    ; RV64I-NEXT: [[PseudoVMERGE_VVM_M4_:%[0-9]+]]:vrm4nov0 = PseudoVMERGE_VVM_M4 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 4 /* e16 */
+    ; RV64I-NEXT: $v8m4 = COPY [[PseudoVMERGE_VVM_M4_]]
+    ; RV64I-NEXT: PseudoRET implicit $v8m4
+    %0:vrb(<vscale x 16 x s1>) = G_IMPLICIT_DEF
+    %1:vrb(<vscale x 16 x s16>) = G_IMPLICIT_DEF
+    %2:vrb(<vscale x 16 x s16>) = G_SELECT %0(<vscale x 16 x s1>), %1, %1
+    $v8m4 = COPY %2(<vscale x 16 x s16>)
+    PseudoRET implicit $v8m4
+
+...
+---
+name:            select_nxv32i16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv32i16
+    ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF
+    ; RV32I-NEXT: $v0 = COPY [[DEF]]
+    ; RV32I-NEXT: [[PseudoVMERGE_VVM_MF2_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_MF2 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 5 /* e32 */
+    ; RV32I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_MF2_]]
+    ; RV32I-NEXT: PseudoRET implicit $v8
+    ;
+    ; RV64I-LABEL: name: select_nxv32i16
+    ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF
+    ; RV64I-NEXT: $v0 = COPY [[DEF]]
+    ; RV64I-NEXT: [[PseudoVMERGE_VVM_MF2_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_MF2 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 5 /* e32 */
+    ; RV64I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_MF2_]]
+    ; RV64I-NEXT: PseudoRET implicit $v8
+    %0:vrb(<vscale x 1 x s1>) = G_IMPLICIT_DEF
+    %1:vrb(<vscale x 1 x s32>) = G_IMPLICIT_DEF
+    %2:vrb(<vscale x 1 x s32>) = G_SELECT %0(<vscale x 1 x s1>), %1, %1
+    $v8 = COPY %2(<vscale x 1 x s32>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            select_nxv2i32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv2i32
+    ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrm2 = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrm2nov0 = IMPLICIT_DEF
+    ; RV32I-NEXT: $v0 = COPY [[DEF]]
+    ; RV32I-NEXT: [[PseudoVMERGE_VVM_M2_:%[0-9]+]]:vrm2nov0 = PseudoVMERGE_VVM_M2 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 5 /* e32 */
+    ; RV32I-NEXT: $v8m2 = COPY [[PseudoVMERGE_VVM_M2_]]
+    ; RV32I-NEXT: PseudoRET implicit $v8m2
+    ;
+    ; RV64I-LABEL: name: select_nxv2i32
+    ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrm2 = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrm2nov0 = IMPLICIT_DEF
+    ; RV64I-NEXT: $v0 = COPY [[DEF]]
+    ; RV64I-NEXT: [[PseudoVMERGE_VVM_M2_:%[0-9]+]]:vrm2nov0 = PseudoVMERGE_VVM_M2 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 5 /* e32 */
+    ; RV64I-NEXT: $v8m2 = COPY [[PseudoVMERGE_VVM_M2_]]
+    ; RV64I-NEXT: PseudoRET implicit $v8m2
+    %0:vrb(<vscale x 4 x s1>) = G_IMPLICIT_DEF
+    %1:vrb(<vscale x 4 x s32>) = G_IMPLICIT_DEF
+    %2:vrb(<vscale x 4 x s32>) = G_SELECT %0(<vscale x 4 x s1>), %1, %1
+    $v8m2 = COPY %2(<vscale x 4 x s32>)
+    PseudoRET implicit $v8m2
+
+...
+---
+name:            select_nxv8i32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv8i32
+    ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrm8 = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrm8nov0 = IMPLICIT_DEF
+    ; RV32I-NEXT: $v0 = COPY [[DEF]]
+    ; RV32I-NEXT: [[PseudoVMERGE_VVM_M8_:%[0-9]+]]:vrm8nov0 = PseudoVMERGE_VVM_M8 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 5 /* e32 */
+    ; RV32I-NEXT: $v8m8 = COPY [[PseudoVMERGE_VVM_M8_]]
+    ; RV32I-NEXT: PseudoRET implicit $v8m8
+    ;
+    ; RV64I-LABEL: name: select_nxv8i32
+    ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrm8 = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrm8nov0 = IMPLICIT_DEF
+    ; RV64I-NEXT: $v0 = COPY [[DEF]]
+    ; RV64I-NEXT: [[PseudoVMERGE_VVM_M8_:%[0-9]+]]:vrm8nov0 = PseudoVMERGE_VVM_M8 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 5 /* e32 */
+    ; RV64I-NEXT: $v8m8 = COPY [[PseudoVMERGE_VVM_M8_]]
+    ; RV64I-NEXT: PseudoRET implicit $v8m8
+    %0:vrb(<vscale x 16 x s1>) = G_IMPLICIT_DEF
+    %1:vrb(<vscale x 16 x s32>) = G_IMPLICIT_DEF
+    %2:vrb(<vscale x 16 x s32>) = G_SELECT %0(<vscale x 16 x s1>), %1, %1
+    $v8m8 = COPY %2(<vscale x 16 x s32>)
+    PseudoRET implicit $v8m8
+
+...
+---
+name:            select_nxv1i64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv1i64
+    ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrm2 = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrm2nov0 = IMPLICIT_DEF
+    ; RV32I-NEXT: $v0 = COPY [[DEF]]
+    ; RV32I-NEXT: [[PseudoVMERGE_VVM_M2_:%[0-9]+]]:vrm2nov0 = PseudoVMERGE_VVM_M2 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 6 /* e64 */
+    ; RV32I-NEXT: $v8m2 = COPY [[PseudoVMERGE_VVM_M2_]]
+    ; RV32I-NEXT: PseudoRET implicit $v8m2
+    ;
+    ; RV64I-LABEL: name: select_nxv1i64
+    ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrm2 = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrm2nov0 = IMPLICIT_DEF
+    ; RV64I-NEXT: $v0 = COPY [[DEF]]
+    ; RV64I-NEXT: [[PseudoVMERGE_VVM_M2_:%[0-9]+]]:vrm2nov0 = PseudoVMERGE_VVM_M2 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 6 /* e64 */
+    ; RV64I-NEXT: $v8m2 = COPY [[PseudoVMERGE_VVM_M2_]]
+    ; RV64I-NEXT: PseudoRET implicit $v8m2
+    %0:vrb(<vscale x 2 x s1>) = G_IMPLICIT_DEF
+    %1:vrb(<vscale x 2 x s64>) = G_IMPLICIT_DEF
+    %2:vrb(<vscale x 2 x s64>) = G_SELECT %0(<vscale x 2 x s1>), %1, %1
+    $v8m2 = COPY %2(<vscale x 2 x s64>)
+    PseudoRET implicit $v8m2
+
+...
+---
+name:            select_nxv4i64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; RV32I-LABEL: name: select_nxv4i64
+    ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrm8 = IMPLICIT_DEF
+    ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrm8nov0 = IMPLICIT_DEF
+    ; RV32I-NEXT: $v0 = COPY [[DEF]]
+    ; RV32I-NEXT: [[PseudoVMERGE_VVM_M8_:%[0-9]+]]:vrm8nov0 = PseudoVMERGE_VVM_M8 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 6 /* e64 */
+    ; RV32I-NEXT: $v8m8 = COPY [[PseudoVMERGE_VVM_M8_]]
+    ; RV32I-NEXT: PseudoRET implicit $v8m8
+    ;
+    ; RV64I-LABEL: name: select_nxv4i64
+    ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrm8 = IMPLICIT_DEF
+    ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrm8nov0 = IMPLICIT_DEF
+    ; RV64I-NEXT: $v0 = COPY [[DEF]]
+    ; RV64I-NEXT: [[PseudoVMERGE_VVM_M8_:%[0-9]+]]:vrm8nov0 = PseudoVMERGE_VVM_M8 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 6 /* e64 */
+    ; RV64I-NEXT: $v8m8 = COPY [[PseudoVMERGE_VVM_M8_]]
+    ; RV64I-NEXT: PseudoRET implicit $v8m8
+    %0:vrb(<vscale x 8 x s1>) = G_IMPLICIT_DEF
+    %1:vrb(<vscale x 8 x s64>) = G_IMPLICIT_DEF
+    %2:vrb(<vscale x 8 x s64>) = G_SELECT %0(<vscale x 8 x s1>), %1, %1
+    $v8m8 = COPY %2(<vscale x 8 x s64>)
+    PseudoRET implicit $v8m8
+
+...
-- 
GitLab


From 05dc5d927bc5f0eeadd277a5e31b04399415f9d0 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <yifanzhu@rochester.edu>
Date: Mon, 25 Mar 2024 13:37:50 -0400
Subject: [PATCH 157/404] [libc] add statvfs/fstatvfs (#86169)

fixes #86449
---
 libc/config/linux/api.td                      |  4 +
 libc/config/linux/x86_64/entrypoints.txt      |  4 +
 libc/include/CMakeLists.txt                   |  9 ++
 libc/include/llvm-libc-types/CMakeLists.txt   | 10 ++
 libc/include/llvm-libc-types/fsblkcnt_t.h     | 14 +++
 libc/include/llvm-libc-types/fsfilcnt_t.h     | 14 +++
 libc/include/llvm-libc-types/struct_statvfs.h | 29 ++++++
 libc/include/sys/statvfs.h.def                | 16 ++++
 libc/spec/posix.td                            | 30 ++++++
 libc/src/sys/CMakeLists.txt                   |  1 +
 libc/src/sys/statvfs/CMakeLists.txt           | 17 ++++
 libc/src/sys/statvfs/fstatvfs.h               | 20 ++++
 libc/src/sys/statvfs/linux/CMakeLists.txt     | 36 +++++++
 libc/src/sys/statvfs/linux/fstatvfs.cpp       | 26 +++++
 libc/src/sys/statvfs/linux/statfs_utils.h     | 95 +++++++++++++++++++
 libc/src/sys/statvfs/linux/statvfs.cpp        | 28 ++++++
 libc/src/sys/statvfs/statvfs.h                | 20 ++++
 libc/test/src/sys/CMakeLists.txt              |  1 +
 libc/test/src/sys/statvfs/CMakeLists.txt      |  3 +
 .../test/src/sys/statvfs/linux/CMakeLists.txt | 29 ++++++
 .../src/sys/statvfs/linux/fstatvfs_test.cpp   | 42 ++++++++
 .../src/sys/statvfs/linux/statvfs_test.cpp    | 47 +++++++++
 22 files changed, 495 insertions(+)
 create mode 100644 libc/include/llvm-libc-types/fsblkcnt_t.h
 create mode 100644 libc/include/llvm-libc-types/fsfilcnt_t.h
 create mode 100644 libc/include/llvm-libc-types/struct_statvfs.h
 create mode 100644 libc/include/sys/statvfs.h.def
 create mode 100644 libc/src/sys/statvfs/CMakeLists.txt
 create mode 100644 libc/src/sys/statvfs/fstatvfs.h
 create mode 100644 libc/src/sys/statvfs/linux/CMakeLists.txt
 create mode 100644 libc/src/sys/statvfs/linux/fstatvfs.cpp
 create mode 100644 libc/src/sys/statvfs/linux/statfs_utils.h
 create mode 100644 libc/src/sys/statvfs/linux/statvfs.cpp
 create mode 100644 libc/src/sys/statvfs/statvfs.h
 create mode 100644 libc/test/src/sys/statvfs/CMakeLists.txt
 create mode 100644 libc/test/src/sys/statvfs/linux/CMakeLists.txt
 create mode 100644 libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp
 create mode 100644 libc/test/src/sys/statvfs/linux/statvfs_test.cpp

diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td
index e9e82c5d4789..eb5ed8089850 100644
--- a/libc/config/linux/api.td
+++ b/libc/config/linux/api.td
@@ -262,3 +262,7 @@ def SetJmpAPI : PublicAPI<"setjmp.h"> {
 def SearchAPI : PublicAPI<"search.h"> {
   let Types = ["ACTION", "ENTRY", "struct hsearch_data"];
 }
+
+def SysStatvfsAPI : PublicAPI<"sys/statvfs.h"> {
+  let Types = ["fsblkcnt_t", "fsfilcnt_t", "struct statvfs"];
+}
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index de741e218a37..2f5a3bffadc0 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -257,6 +257,10 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.sys.stat.mkdirat
     libc.src.sys.stat.stat
 
+    # sys/statvfs.h
+    libc.src.sys.statvfs.statvfs
+    libc.src.sys.statvfs.fstatvfs
+
     # sys/utsname.h entrypoints
     libc.src.sys.utsname.uname
 
diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt
index b2cb10459c53..4203f0bc901b 100644
--- a/libc/include/CMakeLists.txt
+++ b/libc/include/CMakeLists.txt
@@ -498,6 +498,15 @@ add_gen_header(
     .llvm-libc-types.struct_sockaddr_un
 )
 
+add_gen_header(
+  sys_statvfs
+  DEF_FILE sys/statvfs.h.def
+  GEN_HDR sys/statvfs.h
+  DEPENDS
+    .llvm_libc_common_h
+    .llvm-libc-types.struct_statvfs
+)
+
 add_gen_header(
   sys_syscall
   DEF_FILE sys/syscall.h.def
diff --git a/libc/include/llvm-libc-types/CMakeLists.txt b/libc/include/llvm-libc-types/CMakeLists.txt
index 7fef976d7b32..93a79e1477b3 100644
--- a/libc/include/llvm-libc-types/CMakeLists.txt
+++ b/libc/include/llvm-libc-types/CMakeLists.txt
@@ -106,3 +106,13 @@ add_header(
   DEPENDS
     libc.include.llvm-libc-macros.float_macros
 )
+add_header(fsblkcnt_t HDR fsblkcnt_t.h)
+add_header(fsfilcnt_t HDR fsfilcnt_t.h)
+add_header(
+  struct_statvfs 
+HDR 
+  struct_statvfs.h
+DEPENDS
+  .fsblkcnt_t
+  .fsfilcnt_t
+)
diff --git a/libc/include/llvm-libc-types/fsblkcnt_t.h b/libc/include/llvm-libc-types/fsblkcnt_t.h
new file mode 100644
index 000000000000..88a53d38cb1b
--- /dev/null
+++ b/libc/include/llvm-libc-types/fsblkcnt_t.h
@@ -0,0 +1,14 @@
+//===-- Definition of fsblkcnt_t type -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TYPES_FSBLKCNT_T_H
+#define LLVM_LIBC_TYPES_FSBLKCNT_T_H
+
+typedef __SIZE_TYPE__ fsblkcnt_t;
+
+#endif // LLVM_LIBC_TYPES_FSBLKCNT_T_H
diff --git a/libc/include/llvm-libc-types/fsfilcnt_t.h b/libc/include/llvm-libc-types/fsfilcnt_t.h
new file mode 100644
index 000000000000..c5693591907a
--- /dev/null
+++ b/libc/include/llvm-libc-types/fsfilcnt_t.h
@@ -0,0 +1,14 @@
+//===-- Definition of fsfilcnt_t type -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TYPES_FSFILCNT_T_H
+#define LLVM_LIBC_TYPES_FSFILCNT_T_H
+
+typedef __SIZE_TYPE__ fsfilcnt_t;
+
+#endif // LLVM_LIBC_TYPES_FSFILCNT_T_H
diff --git a/libc/include/llvm-libc-types/struct_statvfs.h b/libc/include/llvm-libc-types/struct_statvfs.h
new file mode 100644
index 000000000000..f467cfd936bd
--- /dev/null
+++ b/libc/include/llvm-libc-types/struct_statvfs.h
@@ -0,0 +1,29 @@
+//===-- Definition of type struct statvfs ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TYPES_STRUCT_STATVFS_H
+#define LLVM_LIBC_TYPES_STRUCT_STATVFS_H
+
+#include <llvm-libc-types/fsblkcnt_t.h>
+#include <llvm-libc-types/fsfilcnt_t.h>
+
+struct statvfs {
+  unsigned long f_bsize;   /* Filesystem block size */
+  unsigned long f_frsize;  /* Fragment size */
+  fsblkcnt_t f_blocks;     /* Size of fs in f_frsize units */
+  fsblkcnt_t f_bfree;      /* Number of free blocks */
+  fsblkcnt_t f_bavail;     /* Number of free blocks for unprivileged users */
+  fsfilcnt_t f_files;      /* Number of inodes */
+  fsfilcnt_t f_ffree;      /* Number of free inodes */
+  fsfilcnt_t f_favail;     /* Number of free inodes for unprivileged users */
+  unsigned long f_fsid;    /* Filesystem ID */
+  unsigned long f_flag;    /* Mount flags */
+  unsigned long f_namemax; /* Maximum filename length */
+};
+
+#endif // LLVM_LIBC_TYPES_STRUCT_STATVFS_H
diff --git a/libc/include/sys/statvfs.h.def b/libc/include/sys/statvfs.h.def
new file mode 100644
index 000000000000..f23c9a3d5b1f
--- /dev/null
+++ b/libc/include/sys/statvfs.h.def
@@ -0,0 +1,16 @@
+//===-- POSIX header statvfs.h --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SYS_STATVFS_H
+#define LLVM_LIBC_SYS_STATVFS_H
+
+#include <__llvm-libc-common.h>
+
+%%public_api()
+
+#endif // LLVM_LIBC_SYS_STATVFS_H
diff --git a/libc/spec/posix.td b/libc/spec/posix.td
index 3b793ea90ffd..8444a449ebe5 100644
--- a/libc/spec/posix.td
+++ b/libc/spec/posix.td
@@ -86,6 +86,10 @@ def ConstStructSockAddrPtr : ConstType<StructSockAddrPtr>;
 
 def StructSockAddrUn : NamedType<"struct sockaddr_un">;
 
+def StructStatvfs : NamedType<"struct statvfs">;
+def StructStatvfsPtr : PtrType<StructStatvfs>;
+def RestrictedStructStatvfsPtr : RestrictedPtrType<StructStatvfs>;
+
 def POSIX : StandardSpec<"POSIX"> {
   PtrType CharPtr = PtrType<CharType>;
   RestrictedPtrType RestrictedCharPtr = RestrictedPtrType<CharType>;
@@ -888,6 +892,31 @@ def POSIX : StandardSpec<"POSIX"> {
     ]
   >;
 
+  HeaderSpec SysStatvfs = HeaderSpec<
+      "sys/statvfs.h",
+      [], // Macros
+      [StructStatvfs], // Types
+      [], // Enumerations
+      [
+        FunctionSpec<
+          "statvfs",
+          RetValSpec<IntType>,
+          [
+            ArgSpec<ConstRestrictedCharPtr>,
+            ArgSpec<RestrictedStructStatvfsPtr>
+          ]
+        >,
+        FunctionSpec<
+          "fstatvfs",
+          RetValSpec<IntType>,
+          [
+            ArgSpec<IntType>,
+            ArgSpec<StructStatvfsPtr>
+          ]
+        >,
+      ]  // Functions
+    >;
+
   NamedType StructUtsName = NamedType<"struct utsname">;
   PtrType StructUtsNamePtr = PtrType<StructUtsName>;
   HeaderSpec SysUtsName = HeaderSpec<
@@ -1505,6 +1534,7 @@ def POSIX : StandardSpec<"POSIX"> {
     SysSelect,
     SysSocket,
     SysStat,
+    SysStatvfs,
     SysTypes,
     SysUtsName,
     SysWait,
diff --git a/libc/src/sys/CMakeLists.txt b/libc/src/sys/CMakeLists.txt
index 57ea7b4beaca..adc666b94202 100644
--- a/libc/src/sys/CMakeLists.txt
+++ b/libc/src/sys/CMakeLists.txt
@@ -7,6 +7,7 @@ add_subdirectory(select)
 add_subdirectory(socket)
 add_subdirectory(sendfile)
 add_subdirectory(stat)
+add_subdirectory(statvfs)
 add_subdirectory(utsname)
 add_subdirectory(wait)
 add_subdirectory(prctl)
diff --git a/libc/src/sys/statvfs/CMakeLists.txt b/libc/src/sys/statvfs/CMakeLists.txt
new file mode 100644
index 000000000000..6a10187a135a
--- /dev/null
+++ b/libc/src/sys/statvfs/CMakeLists.txt
@@ -0,0 +1,17 @@
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+endif()
+
+add_entrypoint_object(
+  statvfs
+  ALIAS
+  DEPENDS
+    .${LIBC_TARGET_OS}.statvfs
+)
+
+add_entrypoint_object(
+  fstatvfs
+  ALIAS
+  DEPENDS
+    .${LIBC_TARGET_OS}.fstatvfs
+)
diff --git a/libc/src/sys/statvfs/fstatvfs.h b/libc/src/sys/statvfs/fstatvfs.h
new file mode 100644
index 000000000000..6ca76a459ae5
--- /dev/null
+++ b/libc/src/sys/statvfs/fstatvfs.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for fstatvfs ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_SYS_STATVFS_FSTATVFS_H
+#define LLVM_LIBC_SRC_SYS_STATVFS_FSTATVFS_H
+
+#include "llvm-libc-types/struct_statvfs.h"
+
+namespace LIBC_NAMESPACE {
+
+int fstatvfs(int fd, struct statvfs *buf);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_SYS_STATVFS_FSTATVFS_H
diff --git a/libc/src/sys/statvfs/linux/CMakeLists.txt b/libc/src/sys/statvfs/linux/CMakeLists.txt
new file mode 100644
index 000000000000..f818863bb470
--- /dev/null
+++ b/libc/src/sys/statvfs/linux/CMakeLists.txt
@@ -0,0 +1,36 @@
+add_header_library(
+  statfs_utils
+  HDRS
+    statfs_utils.h
+  DEPENDS
+    libc.src.errno.errno
+    libc.src.__support.OSUtil.osutil
+    libc.src.__support.common
+    libc.src.__support.CPP.optional
+    libc.include.sys_syscall
+)
+
+add_entrypoint_object(
+  statvfs
+  SRCS
+    statvfs.cpp
+  HDRS
+    ../statvfs.h
+  DEPENDS
+    libc.src.__support.libc_assert
+    libc.include.llvm-libc-types.struct_statvfs
+    .statfs_utils
+)
+
+add_entrypoint_object(
+  fstatvfs
+  SRCS
+    fstatvfs.cpp
+  HDRS
+    ../fstatvfs.h
+  DEPENDS
+    libc.src.__support.libc_assert
+    libc.include.llvm-libc-types.struct_statvfs
+    .statfs_utils
+)
+
diff --git a/libc/src/sys/statvfs/linux/fstatvfs.cpp b/libc/src/sys/statvfs/linux/fstatvfs.cpp
new file mode 100644
index 000000000000..488989abbad7
--- /dev/null
+++ b/libc/src/sys/statvfs/linux/fstatvfs.cpp
@@ -0,0 +1,26 @@
+//===-- Linux implementation of fstatvfs ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/sys/statvfs/fstatvfs.h"
+#include "src/__support/common.h"
+#include "src/__support/libc_assert.h"
+#include "src/sys/statvfs/linux/statfs_utils.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(int, fstatvfs, (int fd, struct statvfs *buf)) {
+  using namespace statfs_utils;
+  cpp::optional<LinuxStatFs> result = linux_fstatfs(fd);
+  if (result) {
+    LIBC_ASSERT(buf != nullptr);
+    *buf = statfs_to_statvfs(*result);
+  }
+  return result ? 0 : -1;
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/sys/statvfs/linux/statfs_utils.h b/libc/src/sys/statvfs/linux/statfs_utils.h
new file mode 100644
index 000000000000..606786a57183
--- /dev/null
+++ b/libc/src/sys/statvfs/linux/statfs_utils.h
@@ -0,0 +1,95 @@
+//===-- Convert Statfs to Statvfs -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_SYS_STATVFS_LINUX_STATFS_TO_STATVFS_H
+#define LLVM_LIBC_SRC_SYS_STATVFS_LINUX_STATFS_TO_STATVFS_H
+
+#include "llvm-libc-types/struct_statvfs.h"
+#include "src/__support/CPP/optional.h"
+#include "src/__support/OSUtil/syscall.h"
+#include "src/__support/macros/attributes.h"
+#include "src/errno/libc_errno.h"
+#include <asm/statfs.h>
+#include <sys/syscall.h>
+namespace LIBC_NAMESPACE {
+
+namespace statfs_utils {
+#ifdef SYS_statfs64
+using LinuxStatFs = statfs64;
+#else
+using LinuxStatFs = statfs;
+#endif
+
+// Linux kernel set an additional flag to f_flags. Libc should mask it out.
+LIBC_INLINE_VAR constexpr decltype(LinuxStatFs::f_flags) ST_VALID = 0x0020;
+
+LIBC_INLINE cpp::optional<LinuxStatFs> linux_statfs(const char *path) {
+  // The kernel syscall routine checks the validity of the path before filling
+  // the statfs structure. So, it is possible that the result is not initialized
+  // after the syscall. Since the struct is trvial, the compiler will generate
+  // pattern filling for the struct.
+  LinuxStatFs result;
+  // On 32-bit platforms, original statfs cannot handle large file systems.
+  // In such cases, SYS_statfs64 is defined and should be used.
+#ifdef SYS_statfs64
+  int ret = syscall_impl<int>(SYS_statfs64, path, sizeof(result), &result);
+#else
+  int ret = syscall_impl<int>(SYS_statfs, path, &result);
+#endif
+  if (ret < 0) {
+    libc_errno = -ret;
+    return cpp::nullopt;
+  }
+  result.f_flags &= ~ST_VALID;
+  return result;
+}
+
+LIBC_INLINE cpp::optional<LinuxStatFs> linux_fstatfs(int fd) {
+  // The kernel syscall routine checks the validity of the path before filling
+  // the statfs structure. So, it is possible that the result is not initialized
+  // after the syscall. Since the struct is trvial, the compiler will generate
+  // pattern filling for the struct.
+  LinuxStatFs result;
+  // On 32-bit platforms, original fstatfs cannot handle large file systems.
+  // In such cases, SYS_fstatfs64 is defined and should be used.
+#ifdef SYS_fstatfs64
+  int ret = syscall_impl<int>(SYS_fstatfs64, fd, sizeof(result), &result);
+#else
+  int ret = syscall_impl<int>(SYS_fstatfs, fd, &result);
+#endif
+  if (ret < 0) {
+    libc_errno = -ret;
+    return cpp::nullopt;
+  }
+  result.f_flags &= ~ST_VALID;
+  return result;
+}
+
+// must use 'struct' tag to refer to type 'statvfs' in this scope. There will be
+// a function in the same namespace with the same name. For consistency, we use
+// struct prefix for all statvfs/statfs related types.
+LIBC_INLINE struct statvfs statfs_to_statvfs(const LinuxStatFs &in) {
+  struct statvfs out;
+  out.f_bsize = in.f_bsize;
+  out.f_frsize = in.f_frsize;
+  out.f_blocks = in.f_blocks;
+  out.f_bfree = in.f_bfree;
+  out.f_bavail = in.f_bavail;
+  out.f_files = in.f_files;
+  out.f_ffree = in.f_ffree;
+  out.f_favail = in.f_ffree;
+  out.f_fsid = in.f_fsid.val[0] |
+               static_cast<decltype(out.f_fsid)>(in.f_fsid.val[1]) << 32;
+  out.f_flag = in.f_flags;
+  out.f_namemax = in.f_namelen;
+  return out;
+}
+} // namespace statfs_utils
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_SYS_STATVFS_LINUX_STATFS_TO_STATVFS_H
diff --git a/libc/src/sys/statvfs/linux/statvfs.cpp b/libc/src/sys/statvfs/linux/statvfs.cpp
new file mode 100644
index 000000000000..a438ef1f0117
--- /dev/null
+++ b/libc/src/sys/statvfs/linux/statvfs.cpp
@@ -0,0 +1,28 @@
+//===-- Linux implementation of statvfs -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/sys/statvfs/statvfs.h"
+#include "src/__support/common.h"
+#include "src/__support/libc_assert.h"
+#include "src/sys/statvfs/linux/statfs_utils.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(int, statvfs,
+                   (const char *__restrict path,
+                    struct statvfs *__restrict buf)) {
+  using namespace statfs_utils;
+  cpp::optional<LinuxStatFs> result = linux_statfs(path);
+  if (result) {
+    LIBC_ASSERT(buf != nullptr);
+    *buf = statfs_to_statvfs(*result);
+  }
+  return result ? 0 : -1;
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/sys/statvfs/statvfs.h b/libc/src/sys/statvfs/statvfs.h
new file mode 100644
index 000000000000..792c7ddd0164
--- /dev/null
+++ b/libc/src/sys/statvfs/statvfs.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for statvfs -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_SYS_STATVFS_STATVFS_H
+#define LLVM_LIBC_SRC_SYS_STATVFS_STATVFS_H
+
+#include "llvm-libc-types/struct_statvfs.h"
+
+namespace LIBC_NAMESPACE {
+
+int statvfs(const char *__restrict path, struct statvfs *__restrict buf);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_SYS_STATVFS_STATVFS_H
diff --git a/libc/test/src/sys/CMakeLists.txt b/libc/test/src/sys/CMakeLists.txt
index 7f228e709046..dc0aa8bf7b75 100644
--- a/libc/test/src/sys/CMakeLists.txt
+++ b/libc/test/src/sys/CMakeLists.txt
@@ -5,6 +5,7 @@ add_subdirectory(select)
 add_subdirectory(sendfile)
 add_subdirectory(socket)
 add_subdirectory(stat)
+add_subdirectory(statvfs)
 add_subdirectory(utsname)
 add_subdirectory(wait)
 add_subdirectory(prctl)
diff --git a/libc/test/src/sys/statvfs/CMakeLists.txt b/libc/test/src/sys/statvfs/CMakeLists.txt
new file mode 100644
index 000000000000..b4bbe81c92ff
--- /dev/null
+++ b/libc/test/src/sys/statvfs/CMakeLists.txt
@@ -0,0 +1,3 @@
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
+  add_subdirectory(${LIBC_TARGET_OS})
+endif()
diff --git a/libc/test/src/sys/statvfs/linux/CMakeLists.txt b/libc/test/src/sys/statvfs/linux/CMakeLists.txt
new file mode 100644
index 000000000000..1f8688868e04
--- /dev/null
+++ b/libc/test/src/sys/statvfs/linux/CMakeLists.txt
@@ -0,0 +1,29 @@
+add_custom_target(libc_sys_statvfs_unittests)
+
+add_libc_unittest(
+  statvfs_test
+  SUITE
+    libc_sys_statvfs_unittests
+  SRCS
+    statvfs_test.cpp
+  DEPENDS
+    libc.src.errno.errno
+    libc.src.sys.statvfs.linux.statfs_utils
+    libc.src.sys.statvfs.statvfs
+    libc.test.UnitTest.ErrnoSetterMatcher
+)
+
+add_libc_unittest(
+  fstatvfs_test
+  SUITE
+    libc_sys_statvfs_unittests
+  SRCS
+    fstatvfs_test.cpp
+  DEPENDS
+    libc.src.errno.errno
+    libc.src.sys.statvfs.linux.statfs_utils
+    libc.src.sys.statvfs.fstatvfs
+    libc.src.fcntl.open
+    libc.src.unistd.close
+    libc.test.UnitTest.ErrnoSetterMatcher
+)
diff --git a/libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp b/libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp
new file mode 100644
index 000000000000..bd5195c7969b
--- /dev/null
+++ b/libc/test/src/sys/statvfs/linux/fstatvfs_test.cpp
@@ -0,0 +1,42 @@
+#include "llvm-libc-macros/linux/fcntl-macros.h"
+#include "src/fcntl/open.h"
+#include "src/sys/statvfs/fstatvfs.h"
+#include "src/sys/statvfs/linux/statfs_utils.h"
+#include "src/unistd/close.h"
+#include "test/UnitTest/ErrnoSetterMatcher.h"
+#include "test/UnitTest/LibcTest.h"
+#include <linux/magic.h>
+using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
+
+namespace LIBC_NAMESPACE {
+static int fstatfs(int fd, struct statfs *buf) {
+  using namespace statfs_utils;
+  if (cpp::optional<LinuxStatFs> result = linux_fstatfs(fd)) {
+    *buf = *result;
+    return 0;
+  }
+  return -1;
+}
+} // namespace LIBC_NAMESPACE
+
+struct PathFD {
+  int fd;
+  explicit PathFD(const char *path)
+      : fd(LIBC_NAMESPACE::open(path, O_CLOEXEC | O_PATH)) {}
+  ~PathFD() { LIBC_NAMESPACE::close(fd); }
+  operator int() const { return fd; }
+};
+
+TEST(LlvmLibcSysStatvfsTest, FstatfsBasic) {
+  struct statfs buf;
+  ASSERT_THAT(LIBC_NAMESPACE::fstatfs(PathFD("/"), &buf), Succeeds());
+  ASSERT_THAT(LIBC_NAMESPACE::fstatfs(PathFD("/proc"), &buf), Succeeds());
+  ASSERT_EQ(buf.f_type, static_cast<decltype(buf.f_type)>(PROC_SUPER_MAGIC));
+  ASSERT_THAT(LIBC_NAMESPACE::fstatfs(PathFD("/sys"), &buf), Succeeds());
+  ASSERT_EQ(buf.f_type, static_cast<decltype(buf.f_type)>(SYSFS_MAGIC));
+}
+
+TEST(LlvmLibcSysStatvfsTest, FstatvfsInvalidFD) {
+  struct statvfs buf;
+  ASSERT_THAT(LIBC_NAMESPACE::fstatvfs(-1, &buf), Fails(EBADF));
+}
diff --git a/libc/test/src/sys/statvfs/linux/statvfs_test.cpp b/libc/test/src/sys/statvfs/linux/statvfs_test.cpp
new file mode 100644
index 000000000000..695d2c0d5e98
--- /dev/null
+++ b/libc/test/src/sys/statvfs/linux/statvfs_test.cpp
@@ -0,0 +1,47 @@
+#include "src/sys/statvfs/linux/statfs_utils.h"
+#include "src/sys/statvfs/statvfs.h"
+#include "test/UnitTest/ErrnoSetterMatcher.h"
+#include "test/UnitTest/LibcTest.h"
+#include <linux/magic.h>
+using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
+
+namespace LIBC_NAMESPACE {
+static int statfs(const char *path, struct statfs *buf) {
+  using namespace statfs_utils;
+  if (cpp::optional<LinuxStatFs> result = linux_statfs(path)) {
+    *buf = *result;
+    return 0;
+  }
+  return -1;
+}
+} // namespace LIBC_NAMESPACE
+
+TEST(LlvmLibcSysStatfsTest, StatfsBasic) {
+  struct statfs buf;
+  ASSERT_THAT(LIBC_NAMESPACE::statfs("/", &buf), Succeeds());
+  ASSERT_THAT(LIBC_NAMESPACE::statfs("/proc", &buf), Succeeds());
+  ASSERT_EQ(buf.f_type, static_cast<decltype(buf.f_type)>(PROC_SUPER_MAGIC));
+  ASSERT_THAT(LIBC_NAMESPACE::statfs("/sys", &buf), Succeeds());
+  ASSERT_EQ(buf.f_type, static_cast<decltype(buf.f_type)>(SYSFS_MAGIC));
+}
+
+TEST(LlvmLibcSysStatfsTest, StatvfsInvalidPath) {
+  struct statvfs buf;
+  ASSERT_THAT(LIBC_NAMESPACE::statvfs("", &buf), Fails(ENOENT));
+  ASSERT_THAT(LIBC_NAMESPACE::statvfs("/nonexistent", &buf), Fails(ENOENT));
+  ASSERT_THAT(LIBC_NAMESPACE::statvfs("/dev/null/whatever", &buf),
+              Fails(ENOTDIR));
+  ASSERT_THAT(LIBC_NAMESPACE::statvfs(nullptr, &buf), Fails(EFAULT));
+}
+
+TEST(LlvmLibcSysStatfsTest, StatvfsNameTooLong) {
+  struct statvfs buf;
+  ASSERT_THAT(LIBC_NAMESPACE::statvfs("/", &buf), Succeeds());
+  char *name = static_cast<char *>(__builtin_alloca(buf.f_namemax + 3));
+  name[0] = '/';
+  name[buf.f_namemax + 2] = '\0';
+  for (unsigned i = 1; i < buf.f_namemax + 2; ++i) {
+    name[i] = 'a';
+  }
+  ASSERT_THAT(LIBC_NAMESPACE::statvfs(name, &buf), Fails(ENAMETOOLONG));
+}
-- 
GitLab


From 47681506ded30fada68f180b5e80f740bc76abcd Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Thu, 7 Mar 2024 13:35:22 -0800
Subject: [PATCH 158/404] [RISCV][GISEL] Legalize G_VSCALE

G_VSCALE should be lowered using VLENB.
---
 .../CodeGen/GlobalISel/MachineIRBuilder.h     |  11 +
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    |  51 +++-
 .../CodeGen/GlobalISel/MachineIRBuilder.cpp   |   7 +
 .../Target/RISCV/GISel/RISCVLegalizerInfo.cpp |  48 +++
 .../Target/RISCV/GISel/RISCVLegalizerInfo.h   |   1 +
 llvm/lib/Target/RISCV/RISCVInstrGISel.td      |   8 +
 .../legalizer/rvv/legalize-vscale-rv32.mir    | 283 ++++++++++++++++++
 .../legalizer/rvv/legalize-vscale-rv64.mir    | 120 ++++++++
 8 files changed, 528 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv32.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv64.mir

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 3ba036ae713f..16a7fc446fbe 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -1165,6 +1165,17 @@ public:
   /// \return a MachineInstrBuilder for the newly created instruction.
   MachineInstrBuilder buildVScale(const DstOp &Res, const ConstantInt &MinElts);
 
+  /// Build and insert \p Res = G_VSCALE \p MinElts
+  ///
+  /// G_VSCALE puts the value of the runtime vscale multiplied by \p MinElts
+  /// into \p Res.
+  ///
+  /// \pre setBasicBlock or setMI must have been called.
+  /// \pre \p Res must be a generic virtual register with scalar type.
+  ///
+  /// \return a MachineInstrBuilder for the newly created instruction.
+  MachineInstrBuilder buildVScale(const DstOp &Res, const APInt &MinElts);
+
   /// Build and insert a G_INTRINSIC instruction.
   ///
   /// There are four different opcodes based on combinations of whether the
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 1b25da8833e4..28b73599e411 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -1699,6 +1699,36 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
   case TargetOpcode::G_FLDEXP:
   case TargetOpcode::G_STRICT_FLDEXP:
     return narrowScalarFLDEXP(MI, TypeIdx, NarrowTy);
+  case TargetOpcode::G_VSCALE: {
+    LLT Ty = MRI.getType(MI.getOperand(0).getReg());
+    const APInt &Val = MI.getOperand(1).getCImm()->getValue();
+    unsigned TotalSize = Ty.getSizeInBits();
+    unsigned NarrowSize = NarrowTy.getSizeInBits();
+    int NumParts = TotalSize / NarrowSize;
+
+    SmallVector<Register, 4> PartRegs;
+    for (int I = 0; I != NumParts; ++I) {
+      unsigned Offset = I * NarrowSize;
+      auto K =
+          MIRBuilder.buildVScale(NarrowTy, Val.lshr(Offset).trunc(NarrowSize));
+      PartRegs.push_back(K.getReg(0));
+    }
+    LLT LeftoverTy;
+    unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
+    SmallVector<Register, 1> LeftoverRegs;
+    if (LeftoverBits != 0) {
+      LeftoverTy = LLT::scalar(LeftoverBits);
+      auto K = MIRBuilder.buildVScale(
+          LeftoverTy, Val.lshr(NumParts * NarrowSize).trunc(LeftoverBits));
+      LeftoverRegs.push_back(K.getReg(0));
+    }
+
+    insertParts(MI.getOperand(0).getReg(), Ty, NarrowTy, PartRegs, LeftoverTy,
+                LeftoverRegs);
+
+    MI.eraseFromParent();
+    return Legalized;
+  }
   }
 }
 
@@ -2966,7 +2996,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
   case TargetOpcode::G_VECREDUCE_FMIN:
   case TargetOpcode::G_VECREDUCE_FMAX:
   case TargetOpcode::G_VECREDUCE_FMINIMUM:
-  case TargetOpcode::G_VECREDUCE_FMAXIMUM:
+  case TargetOpcode::G_VECREDUCE_FMAXIMUM: {
     if (TypeIdx != 0)
       return UnableToLegalize;
     Observer.changingInstr(MI);
@@ -2980,6 +3010,25 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     Observer.changedInstr(MI);
     return Legalized;
   }
+  case TargetOpcode::G_VSCALE: {
+    MachineOperand &SrcMO = MI.getOperand(1);
+    LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
+    unsigned ExtOpc = LI.getExtOpcodeForWideningConstant(
+        MRI.getType(MI.getOperand(0).getReg()));
+    assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT ||
+            ExtOpc == TargetOpcode::G_ANYEXT) &&
+           "Illegal Extend");
+    const APInt &SrcVal = SrcMO.getCImm()->getValue();
+    const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT)
+                           ? SrcVal.sext(WideTy.getSizeInBits())
+                           : SrcVal.zext(WideTy.getSizeInBits());
+    Observer.changingInstr(MI);
+    SrcMO.setCImm(ConstantInt::get(Ctx, Val));
+    widenScalarDst(MI, WideTy);
+    Observer.changedInstr(MI);
+    return Legalized;
+  }
+  }
 }
 
 static void getUnmergePieces(SmallVectorImpl<Register> &Pieces,
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index f7aaa0f02efc..f2665e25d195 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -811,6 +811,13 @@ MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
   return VScale;
 }
 
+MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
+                                                  const APInt &MinElts) {
+  ConstantInt *CI =
+      ConstantInt::get(getMF().getFunction().getContext(), MinElts);
+  return buildVScale(Res, *CI);
+}
+
 static unsigned getIntrinsicOpcode(bool HasSideEffects, bool IsConvergent) {
   if (HasSideEffects && IsConvergent)
     return TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS;
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 8fc401da2f6b..44938f0bb35d 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -406,6 +406,10 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
       .clampScalar(0, s32, sXLen)
       .lowerForCartesianProduct({s32, sXLen, p0}, {p0});
 
+  getActionDefinitionsBuilder(G_VSCALE)
+      .clampScalar(0, sXLen, sXLen)
+      .customFor({sXLen});
+
   getLegacyLegalizerInfo().computeTables();
 }
 
@@ -527,6 +531,48 @@ bool RISCVLegalizerInfo::shouldBeInConstantPool(APInt APImm,
   return !(!SeqLo.empty() && (SeqLo.size() + 2) <= STI.getMaxBuildIntsCost());
 }
 
+bool RISCVLegalizerInfo::legalizeVScale(MachineInstr &MI,
+                                        MachineIRBuilder &MIB) const {
+  const LLT XLenTy(STI.getXLenVT());
+  Register Dst = MI.getOperand(0).getReg();
+
+  // We define our scalable vector types for lmul=1 to use a 64 bit known
+  // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
+  // vscale as VLENB / 8.
+  static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
+  if (STI.getRealMinVLen() < RISCV::RVVBitsPerBlock)
+    // Support for VLEN==32 is incomplete.
+    return false;
+
+  // We assume VLENB is a multiple of 8. We manually choose the best shift
+  // here because SimplifyDemandedBits isn't always able to simplify it.
+  uint64_t Val = MI.getOperand(1).getCImm()->getZExtValue();
+  if (isPowerOf2_64(Val)) {
+    uint64_t Log2 = Log2_64(Val);
+    if (Log2 < 3) {
+      auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {XLenTy}, {});
+      MIB.buildLShr(Dst, VLENB, MIB.buildConstant(XLenTy, 3 - Log2));
+    } else if (Log2 > 3) {
+      auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {XLenTy}, {});
+      MIB.buildShl(Dst, VLENB, MIB.buildConstant(XLenTy, Log2 - 3));
+    } else {
+      MIB.buildInstr(RISCV::G_READ_VLENB, {Dst}, {});
+    }
+  } else if ((Val % 8) == 0) {
+    // If the multiplier is a multiple of 8, scale it down to avoid needing
+    // to shift the VLENB value.
+    auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {XLenTy}, {});
+    MIB.buildMul(Dst, VLENB, MIB.buildConstant(XLenTy, Val / 8));
+  } else {
+    auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {XLenTy}, {});
+    auto VScale = MIB.buildLShr(XLenTy, VLENB, MIB.buildConstant(XLenTy, 3));
+    MIB.buildMul(Dst, VScale, MIB.buildConstant(XLenTy, Val));
+  }
+
+  MI.eraseFromParent();
+  return true;
+}
+
 bool RISCVLegalizerInfo::legalizeCustom(
     LegalizerHelper &Helper, MachineInstr &MI,
     LostDebugLocObserver &LocObserver) const {
@@ -584,6 +630,8 @@ bool RISCVLegalizerInfo::legalizeCustom(
   }
   case TargetOpcode::G_VASTART:
     return legalizeVAStart(MI, MIRBuilder);
+  case TargetOpcode::G_VSCALE:
+    return legalizeVScale(MI, MIRBuilder);
   }
 
   llvm_unreachable("expected switch to return");
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
index 323426034827..e2a98c8d2c73 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
@@ -42,6 +42,7 @@ private:
                            GISelChangeObserver &Observer) const;
 
   bool legalizeVAStart(MachineInstr &MI, MachineIRBuilder &MIRBuilder) const;
+  bool legalizeVScale(MachineInstr &MI, MachineIRBuilder &MIB) const;
 };
 } // end namespace llvm
 #endif
diff --git a/llvm/lib/Target/RISCV/RISCVInstrGISel.td b/llvm/lib/Target/RISCV/RISCVInstrGISel.td
index ede8c9809833..54e22d625781 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrGISel.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrGISel.td
@@ -24,3 +24,11 @@ def G_FCLASS : RISCVGenericInstruction {
   let hasSideEffects = false;
 }
 def : GINodeEquiv<G_FCLASS, riscv_fclass>;
+
+// Pseudo equivalent to a RISCVISD::READ_VLENB.
+def G_READ_VLENB : RISCVGenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins);
+  let hasSideEffects = false;
+}
+def : GINodeEquiv<G_READ_VLENB, riscv_read_vlenb>;
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv32.mir
new file mode 100644
index 000000000000..396349542236
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv32.mir
@@ -0,0 +1,283 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+v -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            test_1_s32
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_1_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = G_VSCALE i32 1
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_2_s32
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_2_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = G_VSCALE i32 2
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_3_s32
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_3_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = G_VSCALE i32 3
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_4_s32
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_4_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = G_VSCALE i32 4
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_8_s32
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_8_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = G_VSCALE i32 8
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_16_s32
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_16_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[SHL]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = G_VSCALE i32 16
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_40_s32
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_40_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = G_VSCALE i32 40
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+
+---
+name:            test_1_s64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_1_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 1
+    %1:_(s32) = G_TRUNC %0
+    $x10 = COPY %1
+    PseudoRET implicit $x10
+...
+---
+name:            test_2_s64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_2_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 2
+    %1:_(s32) = G_TRUNC %0
+    $x10 = COPY %1
+    PseudoRET implicit $x10
+...
+---
+name:            test_3_s64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_3_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C2]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 3
+    %1:_(s32) = G_TRUNC %0
+    $x10 = COPY %1
+    PseudoRET implicit $x10
+...
+---
+name:            test_4_s64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_4_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 4
+    %1:_(s32) = G_TRUNC %0
+    $x10 = COPY %1
+    PseudoRET implicit $x10
+...
+---
+name:            test_8_s64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_8_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 8
+    %1:_(s32) = G_TRUNC %0
+    $x10 = COPY %1
+    PseudoRET implicit $x10
+...
+---
+name:            test_16_s64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_16_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[SHL]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 16
+    %1:_(s32) = G_TRUNC %0
+    $x10 = COPY %1
+    PseudoRET implicit $x10
+...
+---
+name:            test_40_s64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_40_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 40
+    %1:_(s32) = G_TRUNC %0
+    $x10 = COPY %1
+    PseudoRET implicit $x10
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv64.mir
new file mode 100644
index 000000000000..3e140a5ef72a
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv64.mir
@@ -0,0 +1,120 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -mattr=+v -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            test_1
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_1
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 1
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_2
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_2
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 2
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_3
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_3
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[C1]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 3
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_4
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_4
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 4
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_8
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_8
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 8
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_16
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_16
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[READ_VLENB]], [[C]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[SHL]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 16
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_40
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_40
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 5
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[C]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 40
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+
+
-- 
GitLab


From a2476c99b745381380eab245fc9499a4ecf0b39e Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Thu, 21 Mar 2024 09:22:04 -0700
Subject: [PATCH 159/404] [RISCV][GISEL] Add regbankselect tests for G_VSCALE

---
 .../regbankselect/rvv/vscale-rv32.mir         | 409 ++++++++++++++++++
 .../regbankselect/rvv/vscale-rv64.mir         | 158 +++++++
 2 files changed, 567 insertions(+)
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv32.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv64.mir

diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv32.mir
new file mode 100644
index 000000000000..2bc33f94978c
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv32.mir
@@ -0,0 +1,409 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+m,+v -run-pass=regbankselect \
+# RUN:   -disable-gisel-legality-check -simplify-mir -verify-machineinstrs %s \
+# RUN:   -o - | FileCheck %s
+
+---
+name:            test_1_s32
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_1_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = G_READ_VLENB
+    %2:_(s32) = G_CONSTANT i32 3
+    %0:_(s32) = G_LSHR %1, %2(s32)
+    $x10 = COPY %0(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_2_s32
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_2_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 2
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = G_READ_VLENB
+    %2:_(s32) = G_CONSTANT i32 2
+    %0:_(s32) = G_LSHR %1, %2(s32)
+    $x10 = COPY %0(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_3_s32
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_3_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = G_READ_VLENB
+    %2:_(s32) = G_CONSTANT i32 3
+    %3:_(s32) = G_LSHR %1, %2(s32)
+    %4:_(s32) = G_CONSTANT i32 3
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %3(s32)
+    $x11 = COPY %4(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    %0:_(s32) = COPY $x10
+    $x10 = COPY %0(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_4_s32
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_4_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = G_READ_VLENB
+    %2:_(s32) = G_CONSTANT i32 1
+    %0:_(s32) = G_LSHR %1, %2(s32)
+    $x10 = COPY %0(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_8_s32
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_8_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = G_READ_VLENB
+    $x10 = COPY %0(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_16_s32
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_16_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:gprb(s32) = G_SHL [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[SHL]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = G_READ_VLENB
+    %2:_(s32) = G_CONSTANT i32 1
+    %0:_(s32) = G_SHL %1, %2(s32)
+    $x10 = COPY %0(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_40_s32
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_40_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 5
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(s32) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = G_READ_VLENB
+    %2:_(s32) = G_CONSTANT i32 5
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %1(s32)
+    $x11 = COPY %2(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    %0:_(s32) = COPY $x10
+    $x10 = COPY %0(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_1_s64
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_1_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %6:_(s32) = G_READ_VLENB
+    %7:_(s32) = G_CONSTANT i32 3
+    %2:_(s32) = G_LSHR %6, %7(s32)
+    %4:_(s32) = G_READ_VLENB
+    %5:_(s32) = G_CONSTANT i32 0
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %4(s32)
+    $x11 = COPY %5(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_2_s64
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_2_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 2
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %6:_(s32) = G_READ_VLENB
+    %7:_(s32) = G_CONSTANT i32 2
+    %2:_(s32) = G_LSHR %6, %7(s32)
+    %4:_(s32) = G_READ_VLENB
+    %5:_(s32) = G_CONSTANT i32 0
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %4(s32)
+    $x11 = COPY %5(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_3_s64
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_3_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(s32) = COPY $x10
+    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C2]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %6:_(s32) = G_READ_VLENB
+    %7:_(s32) = G_CONSTANT i32 3
+    %8:_(s32) = G_LSHR %6, %7(s32)
+    %9:_(s32) = G_CONSTANT i32 3
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %8(s32)
+    $x11 = COPY %9(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    %2:_(s32) = COPY $x10
+    %4:_(s32) = G_READ_VLENB
+    %5:_(s32) = G_CONSTANT i32 0
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %4(s32)
+    $x11 = COPY %5(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_4_s64
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_4_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %6:_(s32) = G_READ_VLENB
+    %7:_(s32) = G_CONSTANT i32 1
+    %2:_(s32) = G_LSHR %6, %7(s32)
+    %4:_(s32) = G_READ_VLENB
+    %5:_(s32) = G_CONSTANT i32 0
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %4(s32)
+    $x11 = COPY %5(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_8_s64
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_8_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %2:_(s32) = G_READ_VLENB
+    %4:_(s32) = G_READ_VLENB
+    %5:_(s32) = G_CONSTANT i32 0
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %4(s32)
+    $x11 = COPY %5(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_16_s64
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_16_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:gprb(s32) = G_SHL [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[SHL]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %6:_(s32) = G_READ_VLENB
+    %7:_(s32) = G_CONSTANT i32 1
+    %2:_(s32) = G_SHL %6, %7(s32)
+    %4:_(s32) = G_READ_VLENB
+    %5:_(s32) = G_CONSTANT i32 0
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %4(s32)
+    $x11 = COPY %5(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_40_s64
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_40_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 5
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(s32) = COPY $x10
+    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %6:_(s32) = G_READ_VLENB
+    %7:_(s32) = G_CONSTANT i32 5
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %6(s32)
+    $x11 = COPY %7(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    %2:_(s32) = COPY $x10
+    %4:_(s32) = G_READ_VLENB
+    %5:_(s32) = G_CONSTANT i32 0
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %4(s32)
+    $x11 = COPY %5(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv64.mir
new file mode 100644
index 000000000000..45a5a1d84646
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv64.mir
@@ -0,0 +1,158 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -mattr=+m,+v -run-pass=regbankselect \
+# RUN:   -disable-gisel-legality-check -simplify-mir -verify-machineinstrs %s \
+# RUN:   -o - | FileCheck %s
+
+---
+name:            test_1
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_1
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s64) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = G_READ_VLENB
+    %2:_(s64) = G_CONSTANT i64 3
+    %0:_(s64) = G_LSHR %1, %2(s64)
+    $x10 = COPY %0(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_2
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_2
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s64) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 2
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = G_READ_VLENB
+    %2:_(s64) = G_CONSTANT i64 2
+    %0:_(s64) = G_LSHR %1, %2(s64)
+    $x10 = COPY %0(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_3
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_3
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s64) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 3
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[C1]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = G_READ_VLENB
+    %2:_(s64) = G_CONSTANT i64 3
+    %3:_(s64) = G_LSHR %1, %2(s64)
+    %4:_(s64) = G_CONSTANT i64 3
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %3(s64)
+    $x11 = COPY %4(s64)
+    PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    %0:_(s64) = COPY $x10
+    $x10 = COPY %0(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_4
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_4
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s64) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = G_READ_VLENB
+    %2:_(s64) = G_CONSTANT i64 1
+    %0:_(s64) = G_LSHR %1, %2(s64)
+    $x10 = COPY %0(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_8
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_8
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s64) = G_READ_VLENB
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_READ_VLENB
+    $x10 = COPY %0(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_16
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_16
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s64) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:gprb(s64) = G_SHL [[READ_VLENB]], [[C]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[SHL]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = G_READ_VLENB
+    %2:_(s64) = G_CONSTANT i64 1
+    %0:_(s64) = G_SHL %1, %2(s64)
+    $x10 = COPY %0(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_40
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_40
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s64) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 5
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[C]](s64)
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(s64) = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = G_READ_VLENB
+    %2:_(s64) = G_CONSTANT i64 5
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %1(s64)
+    $x11 = COPY %2(s64)
+    PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    %0:_(s64) = COPY $x10
+    $x10 = COPY %0(s64)
+    PseudoRET implicit $x10
+
+...
-- 
GitLab


From c00a5ab8c4be14f63735ec61c5c9245c233cbcfc Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Thu, 21 Mar 2024 09:27:33 -0700
Subject: [PATCH 160/404] [RISCV][GISEL] Add instruction select tests for
 G_VSCALE

---
 .../instruction-select/rvv/vscale32.mir       | 412 ++++++++++++++++++
 .../instruction-select/rvv/vscale64.mir       | 140 ++++++
 2 files changed, 552 insertions(+)
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale32.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale64.mir

diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale32.mir
new file mode 100644
index 000000000000..084d5531c459
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale32.mir
@@ -0,0 +1,412 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+v -run-pass=instruction-select \
+# RUN:   -simplify-mir -verify-machineinstrs %s -o - | FileCheck  %s
+
+---
+name:            test_1_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_1_s32
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
+    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s32) = G_READ_VLENB
+    %1:gprb(s32) = G_CONSTANT i32 3
+    %2:gprb(s32) = G_LSHR %0, %1(s32)
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_2_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_2_s32
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 2
+    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s32) = G_READ_VLENB
+    %1:gprb(s32) = G_CONSTANT i32 2
+    %2:gprb(s32) = G_LSHR %0, %1(s32)
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_3_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_3_s32
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 3
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
+    ; CHECK-NEXT: $x11 = COPY [[ADDI]]
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s32) = G_READ_VLENB
+    %1:gprb(s32) = G_CONSTANT i32 3
+    %2:gprb(s32) = G_LSHR %0, %1(s32)
+    %3:gprb(s32) = G_CONSTANT i32 3
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %2(s32)
+    $x11 = COPY %3(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    %4:gprb(s32) = COPY $x10
+    $x10 = COPY %4(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_4_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_4_s32
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 1
+    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s32) = G_READ_VLENB
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_LSHR %0, %1(s32)
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_8_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_8_s32
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s32) = G_READ_VLENB
+    $x10 = COPY %0(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_16_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_16_s32
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SLLI:%[0-9]+]]:gpr = SLLI [[PseudoReadVLENB]], 1
+    ; CHECK-NEXT: $x10 = COPY [[SLLI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s32) = G_READ_VLENB
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_SHL %0, %1(s32)
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_40_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_40_s32
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 5
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB]]
+    ; CHECK-NEXT: $x11 = COPY [[ADDI]]
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s32) = G_READ_VLENB
+    %1:gprb(s32) = G_CONSTANT i32 5
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %0(s32)
+    $x11 = COPY %1(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    %2:gprb(s32) = COPY $x10
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_1_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_1_s64
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
+    ; CHECK-NEXT: [[PseudoReadVLENB1:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB1]]
+    ; CHECK-NEXT: $x11 = COPY [[COPY]]
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s32) = G_READ_VLENB
+    %1:gprb(s32) = G_CONSTANT i32 3
+    %2:gprb(s32) = G_LSHR %0, %1(s32)
+    %3:gprb(s32) = G_READ_VLENB
+    %4:gprb(s32) = G_CONSTANT i32 0
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %3(s32)
+    $x11 = COPY %4(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_2_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_2_s64
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 2
+    ; CHECK-NEXT: [[PseudoReadVLENB1:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB1]]
+    ; CHECK-NEXT: $x11 = COPY [[COPY]]
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s32) = G_READ_VLENB
+    %1:gprb(s32) = G_CONSTANT i32 2
+    %2:gprb(s32) = G_LSHR %0, %1(s32)
+    %3:gprb(s32) = G_READ_VLENB
+    %4:gprb(s32) = G_CONSTANT i32 0
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %3(s32)
+    $x11 = COPY %4(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_3_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_3_s64
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 3
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
+    ; CHECK-NEXT: $x11 = COPY [[ADDI]]
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[PseudoReadVLENB1:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB1]]
+    ; CHECK-NEXT: $x11 = COPY [[COPY1]]
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s32) = G_READ_VLENB
+    %1:gprb(s32) = G_CONSTANT i32 3
+    %2:gprb(s32) = G_LSHR %0, %1(s32)
+    %3:gprb(s32) = G_CONSTANT i32 3
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %2(s32)
+    $x11 = COPY %3(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    %4:gprb(s32) = COPY $x10
+    %5:gprb(s32) = G_READ_VLENB
+    %6:gprb(s32) = G_CONSTANT i32 0
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %5(s32)
+    $x11 = COPY %6(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %4(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_4_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_4_s64
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 1
+    ; CHECK-NEXT: [[PseudoReadVLENB1:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB1]]
+    ; CHECK-NEXT: $x11 = COPY [[COPY]]
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s32) = G_READ_VLENB
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_LSHR %0, %1(s32)
+    %3:gprb(s32) = G_READ_VLENB
+    %4:gprb(s32) = G_CONSTANT i32 0
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %3(s32)
+    $x11 = COPY %4(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_8_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_8_s64
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[PseudoReadVLENB1:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB1]]
+    ; CHECK-NEXT: $x11 = COPY [[COPY]]
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s32) = G_READ_VLENB
+    %1:gprb(s32) = G_READ_VLENB
+    %2:gprb(s32) = G_CONSTANT i32 0
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %1(s32)
+    $x11 = COPY %2(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %0(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_16_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_16_s64
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SLLI:%[0-9]+]]:gpr = SLLI [[PseudoReadVLENB]], 1
+    ; CHECK-NEXT: [[PseudoReadVLENB1:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB1]]
+    ; CHECK-NEXT: $x11 = COPY [[COPY]]
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[SLLI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s32) = G_READ_VLENB
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_SHL %0, %1(s32)
+    %3:gprb(s32) = G_READ_VLENB
+    %4:gprb(s32) = G_CONSTANT i32 0
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %3(s32)
+    $x11 = COPY %4(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_40_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_40_s64
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 5
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB]]
+    ; CHECK-NEXT: $x11 = COPY [[ADDI]]
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[PseudoReadVLENB1:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x0
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB1]]
+    ; CHECK-NEXT: $x11 = COPY [[COPY1]]
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[COPY]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s32) = G_READ_VLENB
+    %1:gprb(s32) = G_CONSTANT i32 5
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %0(s32)
+    $x11 = COPY %1(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    %2:gprb(s32) = COPY $x10
+    %3:gprb(s32) = G_READ_VLENB
+    %4:gprb(s32) = G_CONSTANT i32 0
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %3(s32)
+    $x11 = COPY %4(s32)
+    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale64.mir
new file mode 100644
index 000000000000..3e480e5b8d73
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale64.mir
@@ -0,0 +1,140 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -mattr=+v -run-pass=instruction-select \
+# RUN:   -simplify-mir -verify-machineinstrs %s -o - | FileCheck  %s
+
+---
+name:            test_1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_1
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 2
+    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = G_READ_VLENB
+    %1:gprb(s64) = G_CONSTANT i64 2
+    %2:gprb(s64) = G_LSHR %0, %1(s64)
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_3
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_3
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 3
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
+    ; CHECK-NEXT: $x11 = COPY [[ADDI]]
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = G_READ_VLENB
+    %1:gprb(s64) = G_CONSTANT i64 3
+    %2:gprb(s64) = G_LSHR %0, %1(s64)
+    %3:gprb(s64) = G_CONSTANT i64 3
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %2(s64)
+    $x11 = COPY %3(s64)
+    PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    %4:gprb(s64) = COPY $x10
+    $x10 = COPY %4(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_4
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 1
+    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = G_READ_VLENB
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s64) = G_LSHR %0, %1(s64)
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_8
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_8
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = G_READ_VLENB
+    $x10 = COPY %0(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_16
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SLLI:%[0-9]+]]:gpr = SLLI [[PseudoReadVLENB]], 1
+    ; CHECK-NEXT: $x10 = COPY [[SLLI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = G_READ_VLENB
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s64) = G_SHL %0, %1(s64)
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_40
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_40
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 5
+    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB]]
+    ; CHECK-NEXT: $x11 = COPY [[ADDI]]
+    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = G_READ_VLENB
+    %1:gprb(s64) = G_CONSTANT i64 5
+    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+    $x10 = COPY %0(s64)
+    $x11 = COPY %1(s64)
+    PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
+    %2:gprb(s64) = COPY $x10
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+
+...
-- 
GitLab


From 6187f271d107c11574c8d8db9d64a83095a9072c Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Mon, 25 Mar 2024 13:45:49 -0400
Subject: [PATCH 161/404] [gn] port fa3d789df15b more (tblgen restructure)

I had missed that both tblgens still both need a bunch of identical
cpp files. Let's just reintroduce the source_set() for them, even
though the CMake build removed the object library for them in favor
of duplicating the file list for some reason.
---
 llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn
index 1c91a775e014..53a9d8d01519 100644
--- a/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn
@@ -1,4 +1,4 @@
-executable("llvm-min-tblgen") {
+source_set("llvm-min-tblgen-sources") {
   sources = [
     "Attributes.cpp",
     "DirectiveEmitter.cpp",
@@ -10,12 +10,19 @@ executable("llvm-min-tblgen") {
   deps = [
     "Basic",
     "//llvm/lib/Support",
-    "//llvm/lib/TableGen",
+  ]
+}
+
+executable("llvm-min-tblgen") {
+  deps = [
+    ":llvm-min-tblgen-sources",
+    "Basic",
   ]
 }
 
 executable("llvm-tblgen") {
   deps = [
+    ":llvm-min-tblgen-sources",
     "Basic",
     "Common",
     "//llvm/include/llvm/Config:llvm-config",
@@ -54,7 +61,6 @@ executable("llvm-tblgen") {
     "RegisterBankEmitter.cpp",
     "RegisterInfoEmitter.cpp",
     "SearchableTableEmitter.cpp",
-    "TableGen.cpp",
     "SubtargetEmitter.cpp",
     "WebAssemblyDisassemblerEmitter.cpp",
     "X86CompressEVEXTablesEmitter.cpp",
-- 
GitLab


From bd4086f0464cc6e3e85b57ae58bc6892e777297f Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Mon, 25 Mar 2024 13:45:30 -0400
Subject: [PATCH 162/404] [C99] Claim conformance to "additional predefined
 macro names"

This feature is listed in the C99 front matter as one of the changes to
C99, but there was no associated document with that title (or anything
close to it).

I eventually tracked this down to a new requirement in C99 that
implementations defining their own pedefined macros should not name
them with a __STDC_ prefix. Clang does not do this, but there's no
tests that would demonstrate we properly "implement" this feature. So
claiming full conformance without additional test coverage.
---
 clang/www/c_status.html | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/clang/www/c_status.html b/clang/www/c_status.html
index ddfdc2f21c87..435897277c49 100644
--- a/clang/www/c_status.html
+++ b/clang/www/c_status.html
@@ -315,7 +315,13 @@ conformance.</p>
     <tr>
       <td>additional predefined macro names</td>
       <td>Unknown</td>
-      <td class="unknown" align="center">Unknown</td>
+      <td class="full" align="center">Yes</td>
+      <!-- It is unknown which paper brought in this change, which was listed in
+           the C99 front matter. After hunting around for what these changes are,
+           I found a mention in the C99 rationale document that implementers who
+           wish to add their own predefined macros must not start them with
+           __STDC_, which was a new restriction in C99. As best I can tell, that
+           is what this particular feature is about. -->
     </tr>
     <tr>
       <td>_Pragma preprocessing operator</td>
-- 
GitLab


From 3305f5a980363d7e224094bc42bcbebcf162618e Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Mon, 25 Mar 2024 10:46:38 -0700
Subject: [PATCH 163/404] [RISCV][GISEL] Check for legal element count in
 typeIsLegalBoolVec

https://github.com/llvm/llvm-project/pull/84553 added
typeIsLegalBoolVec but it was missing a check for minimum element count.
This patch adds that.
---
 llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 44938f0bb35d..9a388f4cd271 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -58,10 +58,12 @@ typeIsLegalIntOrFPVec(unsigned TypeIdx,
 static LegalityPredicate
 typeIsLegalBoolVec(unsigned TypeIdx, std::initializer_list<LLT> BoolVecTys,
                    const RISCVSubtarget &ST) {
-  LegalityPredicate HasV = [=, &ST](const LegalityQuery &Query) {
-    return ST.hasVInstructions();
+  LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
+    return ST.hasVInstructions() &&
+           (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
+            ST.getELen() == 64);
   };
-  return all(typeInSet(TypeIdx, BoolVecTys), HasV);
+  return all(typeInSet(TypeIdx, BoolVecTys), P);
 }
 
 RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
-- 
GitLab


From 32466e55b19586880199347744fc1b76d6b04cfb Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Mon, 25 Mar 2024 13:48:35 -0400
Subject: [PATCH 164/404] Add a triple to this test

Amends d99cfa053998483e7283e91ff204fbe117345685

This should address a failure found by post commit CI:
https://lab.llvm.org/buildbot/#/builders/245/builds/22321
---
 clang/test/C/C99/n696.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/test/C/C99/n696.c b/clang/test/C/C99/n696.c
index c84ef2c7159e..4499c6e42226 100644
--- a/clang/test/C/C99/n696.c
+++ b/clang/test/C/C99/n696.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -verify %s
+// RUN: %clang_cc1 -triple x86_64 -verify %s
 
 /* WG14 N696: yes
  * Standard pragmas - improved wording
-- 
GitLab


From 3cf169ca160eaf5464503fbd93d73ee1d8597936 Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Mon, 25 Mar 2024 18:53:55 +0100
Subject: [PATCH 165/404] [libc++][CI] Updates the documentation. (#85264)

The documentation mentions manually pushing Docker images to the CI. The
preferred way is to use the proper GitHub action. This updates the
documentation.

---------

Co-authored-by: Will Hawkins <whh8b@obs.cr>
---
 libcxx/utils/ci/Dockerfile | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/libcxx/utils/ci/Dockerfile b/libcxx/utils/ci/Dockerfile
index 178cba415933..db88da20b977 100644
--- a/libcxx/utils/ci/Dockerfile
+++ b/libcxx/utils/ci/Dockerfile
@@ -7,15 +7,13 @@
 #===----------------------------------------------------------------------===##
 #
 # This file defines the buildkite and github actions builder images.
-# You can build & push both images using:
+# You can build both images using:
 #
 #   docker compose build
-#   docker compose push
 #
-# Or you can select a single image to build & push using:
+# Or you can select a single image to build
 #
 #  docker compose build buildkite-builder
-#  docker compose push buildkite-builder
 #
 # The final images can be found at
 #
@@ -23,7 +21,8 @@
 #  ghcr.io/libcxx/actions-builder
 #  ghcr.io/libcxx/android-buildkite-builder
 #
-# Members of the github.com/libcxx/ organizations have permissions required to push new images.
+# Members of the github.com/libcxx/ organizations can push new images to the CI.
+# This is done by GitHub actions in the https://github.com/libcxx/builders repo.
 #
 # ===----------------------------------------------------------------------===##
 #                     Running the buildkite image
-- 
GitLab


From ce37a7131ff33bbd04bd1893de78c1bd3a1f0897 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 25 Mar 2024 11:10:58 -0700
Subject: [PATCH 166/404] [RISCV] Add integer RISCVISD::SELECT_CC to
 canCreateUndefOrPoison and isGuaranteedNotToBeUndefOrPoison. (#84693)

Integer RISCVISD::SELECT_CC doesn't create poison. If none of the,
operands are poison, the result is not poison.

This allows ISD::FREEZE to be hoisted above RISCVISD::SELECT_CC.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |  17 +
 llvm/lib/Target/RISCV/RISCVISelLowering.h     |   6 +
 llvm/test/CodeGen/RISCV/double-convert.ll     | 118 ++-
 .../CodeGen/RISCV/double-round-conv-sat.ll    | 708 ++++++++----------
 llvm/test/CodeGen/RISCV/float-convert.ll      |  48 +-
 .../CodeGen/RISCV/float-round-conv-sat.ll     | 168 ++---
 llvm/test/CodeGen/RISCV/half-convert.ll       |  57 +-
 .../test/CodeGen/RISCV/half-round-conv-sat.ll | 336 ++++-----
 8 files changed, 702 insertions(+), 756 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 4ace50aa477a..c3f8a924a1da 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -17093,6 +17093,23 @@ unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
   return 1;
 }
 
+bool RISCVTargetLowering::canCreateUndefOrPoisonForTargetNode(
+    SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
+    bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const {
+
+  // TODO: Add more target nodes.
+  switch (Op.getOpcode()) {
+  case RISCVISD::SELECT_CC:
+    // Integer select_cc cannot create poison.
+    // TODO: What are the FP poison semantics?
+    // TODO: This instruction blocks poison from the unselected operand, can
+    // we do anything with that?
+    return !Op.getValueType().isInteger();
+  }
+  return TargetLowering::canCreateUndefOrPoisonForTargetNode(
+      Op, DemandedElts, DAG, PoisonOnly, ConsiderFlags, Depth);
+}
+
 const Constant *
 RISCVTargetLowering::getTargetConstantFromLoad(LoadSDNode *Ld) const {
   assert(Ld && "Unexpected null LoadSDNode");
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index f90cb4df6047..c11b1464757c 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -585,6 +585,12 @@ public:
                                            const SelectionDAG &DAG,
                                            unsigned Depth) const override;
 
+  bool canCreateUndefOrPoisonForTargetNode(SDValue Op,
+                                           const APInt &DemandedElts,
+                                           const SelectionDAG &DAG,
+                                           bool PoisonOnly, bool ConsiderFlags,
+                                           unsigned Depth) const override;
+
   const Constant *getTargetConstantFromLoad(LoadSDNode *LD) const override;
 
   // This method returns the name of a target specific DAG node.
diff --git a/llvm/test/CodeGen/RISCV/double-convert.ll b/llvm/test/CodeGen/RISCV/double-convert.ll
index c1429642962e..da882cafd997 100644
--- a/llvm/test/CodeGen/RISCV/double-convert.ll
+++ b/llvm/test/CodeGen/RISCV/double-convert.ll
@@ -683,47 +683,41 @@ define i64 @fcvt_l_d(double %a) nounwind {
 define i64 @fcvt_l_d_sat(double %a) nounwind {
 ; RV32IFD-LABEL: fcvt_l_d_sat:
 ; RV32IFD:       # %bb.0: # %start
-; RV32IFD-NEXT:    addi sp, sp, -32
-; RV32IFD-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    fsd fs0, 0(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    lui a0, %hi(.LCPI12_0)
 ; RV32IFD-NEXT:    fld fa5, %lo(.LCPI12_0)(a0)
-; RV32IFD-NEXT:    lui a0, %hi(.LCPI12_1)
-; RV32IFD-NEXT:    fld fa4, %lo(.LCPI12_1)(a0)
 ; RV32IFD-NEXT:    fmv.d fs0, fa0
-; RV32IFD-NEXT:    flt.d s0, fa5, fa0
-; RV32IFD-NEXT:    neg s1, s0
-; RV32IFD-NEXT:    fle.d s2, fa4, fa0
-; RV32IFD-NEXT:    neg s3, s2
+; RV32IFD-NEXT:    fle.d s0, fa5, fa0
 ; RV32IFD-NEXT:    call __fixdfdi
-; RV32IFD-NEXT:    and a0, s3, a0
-; RV32IFD-NEXT:    or a0, s1, a0
-; RV32IFD-NEXT:    feq.d a2, fs0, fs0
-; RV32IFD-NEXT:    neg a2, a2
-; RV32IFD-NEXT:    lui a4, 524288
-; RV32IFD-NEXT:    li a5, 1
 ; RV32IFD-NEXT:    lui a3, 524288
-; RV32IFD-NEXT:    bne s2, a5, .LBB12_2
+; RV32IFD-NEXT:    li a4, 1
+; RV32IFD-NEXT:    lui a2, 524288
+; RV32IFD-NEXT:    bne s0, a4, .LBB12_2
 ; RV32IFD-NEXT:  # %bb.1: # %start
-; RV32IFD-NEXT:    mv a3, a1
+; RV32IFD-NEXT:    mv a2, a1
 ; RV32IFD-NEXT:  .LBB12_2: # %start
-; RV32IFD-NEXT:    and a0, a2, a0
-; RV32IFD-NEXT:    beqz s0, .LBB12_4
+; RV32IFD-NEXT:    lui a1, %hi(.LCPI12_1)
+; RV32IFD-NEXT:    fld fa5, %lo(.LCPI12_1)(a1)
+; RV32IFD-NEXT:    flt.d a4, fa5, fs0
+; RV32IFD-NEXT:    beqz a4, .LBB12_4
 ; RV32IFD-NEXT:  # %bb.3:
-; RV32IFD-NEXT:    addi a3, a4, -1
+; RV32IFD-NEXT:    addi a2, a3, -1
 ; RV32IFD-NEXT:  .LBB12_4: # %start
-; RV32IFD-NEXT:    and a1, a2, a3
-; RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    feq.d a1, fs0, fs0
+; RV32IFD-NEXT:    neg a3, a1
+; RV32IFD-NEXT:    and a1, a3, a2
+; RV32IFD-NEXT:    neg a2, a4
+; RV32IFD-NEXT:    neg a4, s0
+; RV32IFD-NEXT:    and a0, a4, a0
+; RV32IFD-NEXT:    or a0, a2, a0
+; RV32IFD-NEXT:    and a0, a3, a0
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
-; RV32IFD-NEXT:    addi sp, sp, 32
+; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: fcvt_l_d_sat:
@@ -737,48 +731,44 @@ define i64 @fcvt_l_d_sat(double %a) nounwind {
 ;
 ; RV32IZFINXZDINX-LABEL: fcvt_l_d_sat:
 ; RV32IZFINXZDINX:       # %bb.0: # %start
-; RV32IZFINXZDINX-NEXT:    addi sp, sp, -32
-; RV32IZFINXZDINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI12_0)
-; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI12_0+4)(a2)
-; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI12_0)(a2)
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32IZFINXZDINX-NEXT:    mv s1, a1
 ; RV32IZFINXZDINX-NEXT:    mv s0, a0
-; RV32IZFINXZDINX-NEXT:    fle.d s2, a2, s0
-; RV32IZFINXZDINX-NEXT:    neg s3, s2
 ; RV32IZFINXZDINX-NEXT:    call __fixdfdi
-; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI12_1)
-; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI12_1+4)(a2)
-; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI12_1)(a2)
-; RV32IZFINXZDINX-NEXT:    and a0, s3, a0
-; RV32IZFINXZDINX-NEXT:    flt.d a3, a2, s0
-; RV32IZFINXZDINX-NEXT:    neg a2, a3
-; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    feq.d a2, s0, s0
-; RV32IZFINXZDINX-NEXT:    neg a2, a2
+; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI12_0)
+; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI12_0+4)(a2)
+; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI12_0)(a2)
+; RV32IZFINXZDINX-NEXT:    fle.d a2, a2, s0
 ; RV32IZFINXZDINX-NEXT:    lui a5, 524288
-; RV32IZFINXZDINX-NEXT:    li a6, 1
-; RV32IZFINXZDINX-NEXT:    lui a4, 524288
-; RV32IZFINXZDINX-NEXT:    bne s2, a6, .LBB12_2
+; RV32IZFINXZDINX-NEXT:    li a4, 1
+; RV32IZFINXZDINX-NEXT:    lui a3, 524288
+; RV32IZFINXZDINX-NEXT:    bne a2, a4, .LBB12_2
 ; RV32IZFINXZDINX-NEXT:  # %bb.1: # %start
-; RV32IZFINXZDINX-NEXT:    mv a4, a1
+; RV32IZFINXZDINX-NEXT:    mv a3, a1
 ; RV32IZFINXZDINX-NEXT:  .LBB12_2: # %start
-; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    beqz a3, .LBB12_4
+; RV32IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI12_1)
+; RV32IZFINXZDINX-NEXT:    lw a6, %lo(.LCPI12_1)(a1)
+; RV32IZFINXZDINX-NEXT:    lw a7, %lo(.LCPI12_1+4)(a1)
+; RV32IZFINXZDINX-NEXT:    flt.d a4, a6, s0
+; RV32IZFINXZDINX-NEXT:    beqz a4, .LBB12_4
 ; RV32IZFINXZDINX-NEXT:  # %bb.3:
-; RV32IZFINXZDINX-NEXT:    addi a4, a5, -1
+; RV32IZFINXZDINX-NEXT:    addi a3, a5, -1
 ; RV32IZFINXZDINX-NEXT:  .LBB12_4: # %start
-; RV32IZFINXZDINX-NEXT:    and a1, a2, a4
-; RV32IZFINXZDINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    addi sp, sp, 32
+; RV32IZFINXZDINX-NEXT:    feq.d a1, s0, s0
+; RV32IZFINXZDINX-NEXT:    neg a5, a1
+; RV32IZFINXZDINX-NEXT:    and a1, a5, a3
+; RV32IZFINXZDINX-NEXT:    neg a2, a2
+; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    neg a2, a4
+; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    and a0, a5, a0
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
 ; RV32IZFINXZDINX-NEXT:    ret
 ;
 ; RV64IZFINXZDINX-LABEL: fcvt_l_d_sat:
diff --git a/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll b/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll
index 29a9e5070950..f1c56b320b76 100644
--- a/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll
+++ b/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll
@@ -43,48 +43,42 @@ define signext i32 @test_floor_si32(double %x) {
 define i64 @test_floor_si64(double %x) nounwind {
 ; RV32IFD-LABEL: test_floor_si64:
 ; RV32IFD:       # %bb.0:
-; RV32IFD-NEXT:    addi sp, sp, -32
-; RV32IFD-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    fsd fs0, 0(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    call floor
 ; RV32IFD-NEXT:    lui a0, %hi(.LCPI1_0)
 ; RV32IFD-NEXT:    fld fa5, %lo(.LCPI1_0)(a0)
-; RV32IFD-NEXT:    lui a0, %hi(.LCPI1_1)
-; RV32IFD-NEXT:    fld fa4, %lo(.LCPI1_1)(a0)
 ; RV32IFD-NEXT:    fmv.d fs0, fa0
-; RV32IFD-NEXT:    flt.d s0, fa5, fa0
-; RV32IFD-NEXT:    neg s1, s0
-; RV32IFD-NEXT:    fle.d s2, fa4, fa0
-; RV32IFD-NEXT:    neg s3, s2
+; RV32IFD-NEXT:    fle.d s0, fa5, fa0
 ; RV32IFD-NEXT:    call __fixdfdi
-; RV32IFD-NEXT:    and a0, s3, a0
-; RV32IFD-NEXT:    or a0, s1, a0
-; RV32IFD-NEXT:    feq.d a2, fs0, fs0
-; RV32IFD-NEXT:    neg a2, a2
-; RV32IFD-NEXT:    lui a4, 524288
-; RV32IFD-NEXT:    li a5, 1
 ; RV32IFD-NEXT:    lui a3, 524288
-; RV32IFD-NEXT:    bne s2, a5, .LBB1_2
+; RV32IFD-NEXT:    li a4, 1
+; RV32IFD-NEXT:    lui a2, 524288
+; RV32IFD-NEXT:    bne s0, a4, .LBB1_2
 ; RV32IFD-NEXT:  # %bb.1:
-; RV32IFD-NEXT:    mv a3, a1
+; RV32IFD-NEXT:    mv a2, a1
 ; RV32IFD-NEXT:  .LBB1_2:
-; RV32IFD-NEXT:    and a0, a2, a0
-; RV32IFD-NEXT:    beqz s0, .LBB1_4
+; RV32IFD-NEXT:    lui a1, %hi(.LCPI1_1)
+; RV32IFD-NEXT:    fld fa5, %lo(.LCPI1_1)(a1)
+; RV32IFD-NEXT:    flt.d a4, fa5, fs0
+; RV32IFD-NEXT:    beqz a4, .LBB1_4
 ; RV32IFD-NEXT:  # %bb.3:
-; RV32IFD-NEXT:    addi a3, a4, -1
+; RV32IFD-NEXT:    addi a2, a3, -1
 ; RV32IFD-NEXT:  .LBB1_4:
-; RV32IFD-NEXT:    and a1, a2, a3
-; RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    feq.d a1, fs0, fs0
+; RV32IFD-NEXT:    neg a3, a1
+; RV32IFD-NEXT:    and a1, a3, a2
+; RV32IFD-NEXT:    neg a2, a4
+; RV32IFD-NEXT:    neg a4, s0
+; RV32IFD-NEXT:    and a0, a4, a0
+; RV32IFD-NEXT:    or a0, a2, a0
+; RV32IFD-NEXT:    and a0, a3, a0
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
-; RV32IFD-NEXT:    addi sp, sp, 32
+; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: test_floor_si64:
@@ -98,49 +92,45 @@ define i64 @test_floor_si64(double %x) nounwind {
 ;
 ; RV32IZFINXZDINX-LABEL: test_floor_si64:
 ; RV32IZFINXZDINX:       # %bb.0:
-; RV32IZFINXZDINX-NEXT:    addi sp, sp, -32
-; RV32IZFINXZDINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32IZFINXZDINX-NEXT:    call floor
-; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI1_0)
-; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI1_0+4)(a2)
-; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI1_0)(a2)
 ; RV32IZFINXZDINX-NEXT:    mv s0, a0
 ; RV32IZFINXZDINX-NEXT:    mv s1, a1
-; RV32IZFINXZDINX-NEXT:    fle.d s2, a2, s0
-; RV32IZFINXZDINX-NEXT:    neg s3, s2
 ; RV32IZFINXZDINX-NEXT:    call __fixdfdi
-; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI1_1)
-; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI1_1+4)(a2)
-; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI1_1)(a2)
-; RV32IZFINXZDINX-NEXT:    and a0, s3, a0
-; RV32IZFINXZDINX-NEXT:    flt.d a3, a2, s0
-; RV32IZFINXZDINX-NEXT:    neg a2, a3
-; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    feq.d a2, s0, s0
-; RV32IZFINXZDINX-NEXT:    neg a2, a2
+; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI1_0)
+; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI1_0+4)(a2)
+; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI1_0)(a2)
+; RV32IZFINXZDINX-NEXT:    fle.d a2, a2, s0
 ; RV32IZFINXZDINX-NEXT:    lui a5, 524288
-; RV32IZFINXZDINX-NEXT:    li a6, 1
-; RV32IZFINXZDINX-NEXT:    lui a4, 524288
-; RV32IZFINXZDINX-NEXT:    bne s2, a6, .LBB1_2
+; RV32IZFINXZDINX-NEXT:    li a4, 1
+; RV32IZFINXZDINX-NEXT:    lui a3, 524288
+; RV32IZFINXZDINX-NEXT:    bne a2, a4, .LBB1_2
 ; RV32IZFINXZDINX-NEXT:  # %bb.1:
-; RV32IZFINXZDINX-NEXT:    mv a4, a1
+; RV32IZFINXZDINX-NEXT:    mv a3, a1
 ; RV32IZFINXZDINX-NEXT:  .LBB1_2:
-; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    beqz a3, .LBB1_4
+; RV32IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI1_1)
+; RV32IZFINXZDINX-NEXT:    lw a6, %lo(.LCPI1_1)(a1)
+; RV32IZFINXZDINX-NEXT:    lw a7, %lo(.LCPI1_1+4)(a1)
+; RV32IZFINXZDINX-NEXT:    flt.d a4, a6, s0
+; RV32IZFINXZDINX-NEXT:    beqz a4, .LBB1_4
 ; RV32IZFINXZDINX-NEXT:  # %bb.3:
-; RV32IZFINXZDINX-NEXT:    addi a4, a5, -1
+; RV32IZFINXZDINX-NEXT:    addi a3, a5, -1
 ; RV32IZFINXZDINX-NEXT:  .LBB1_4:
-; RV32IZFINXZDINX-NEXT:    and a1, a2, a4
-; RV32IZFINXZDINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    addi sp, sp, 32
+; RV32IZFINXZDINX-NEXT:    feq.d a1, s0, s0
+; RV32IZFINXZDINX-NEXT:    neg a5, a1
+; RV32IZFINXZDINX-NEXT:    and a1, a5, a3
+; RV32IZFINXZDINX-NEXT:    neg a2, a2
+; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    neg a2, a4
+; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    and a0, a5, a0
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
 ; RV32IZFINXZDINX-NEXT:    ret
 ;
 ; RV64IZFINXZDINX-LABEL: test_floor_si64:
@@ -299,48 +289,42 @@ define signext i32 @test_ceil_si32(double %x) {
 define i64 @test_ceil_si64(double %x) nounwind {
 ; RV32IFD-LABEL: test_ceil_si64:
 ; RV32IFD:       # %bb.0:
-; RV32IFD-NEXT:    addi sp, sp, -32
-; RV32IFD-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    fsd fs0, 0(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    call ceil
 ; RV32IFD-NEXT:    lui a0, %hi(.LCPI5_0)
 ; RV32IFD-NEXT:    fld fa5, %lo(.LCPI5_0)(a0)
-; RV32IFD-NEXT:    lui a0, %hi(.LCPI5_1)
-; RV32IFD-NEXT:    fld fa4, %lo(.LCPI5_1)(a0)
 ; RV32IFD-NEXT:    fmv.d fs0, fa0
-; RV32IFD-NEXT:    flt.d s0, fa5, fa0
-; RV32IFD-NEXT:    neg s1, s0
-; RV32IFD-NEXT:    fle.d s2, fa4, fa0
-; RV32IFD-NEXT:    neg s3, s2
+; RV32IFD-NEXT:    fle.d s0, fa5, fa0
 ; RV32IFD-NEXT:    call __fixdfdi
-; RV32IFD-NEXT:    and a0, s3, a0
-; RV32IFD-NEXT:    or a0, s1, a0
-; RV32IFD-NEXT:    feq.d a2, fs0, fs0
-; RV32IFD-NEXT:    neg a2, a2
-; RV32IFD-NEXT:    lui a4, 524288
-; RV32IFD-NEXT:    li a5, 1
 ; RV32IFD-NEXT:    lui a3, 524288
-; RV32IFD-NEXT:    bne s2, a5, .LBB5_2
+; RV32IFD-NEXT:    li a4, 1
+; RV32IFD-NEXT:    lui a2, 524288
+; RV32IFD-NEXT:    bne s0, a4, .LBB5_2
 ; RV32IFD-NEXT:  # %bb.1:
-; RV32IFD-NEXT:    mv a3, a1
+; RV32IFD-NEXT:    mv a2, a1
 ; RV32IFD-NEXT:  .LBB5_2:
-; RV32IFD-NEXT:    and a0, a2, a0
-; RV32IFD-NEXT:    beqz s0, .LBB5_4
+; RV32IFD-NEXT:    lui a1, %hi(.LCPI5_1)
+; RV32IFD-NEXT:    fld fa5, %lo(.LCPI5_1)(a1)
+; RV32IFD-NEXT:    flt.d a4, fa5, fs0
+; RV32IFD-NEXT:    beqz a4, .LBB5_4
 ; RV32IFD-NEXT:  # %bb.3:
-; RV32IFD-NEXT:    addi a3, a4, -1
+; RV32IFD-NEXT:    addi a2, a3, -1
 ; RV32IFD-NEXT:  .LBB5_4:
-; RV32IFD-NEXT:    and a1, a2, a3
-; RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    feq.d a1, fs0, fs0
+; RV32IFD-NEXT:    neg a3, a1
+; RV32IFD-NEXT:    and a1, a3, a2
+; RV32IFD-NEXT:    neg a2, a4
+; RV32IFD-NEXT:    neg a4, s0
+; RV32IFD-NEXT:    and a0, a4, a0
+; RV32IFD-NEXT:    or a0, a2, a0
+; RV32IFD-NEXT:    and a0, a3, a0
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
-; RV32IFD-NEXT:    addi sp, sp, 32
+; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: test_ceil_si64:
@@ -354,49 +338,45 @@ define i64 @test_ceil_si64(double %x) nounwind {
 ;
 ; RV32IZFINXZDINX-LABEL: test_ceil_si64:
 ; RV32IZFINXZDINX:       # %bb.0:
-; RV32IZFINXZDINX-NEXT:    addi sp, sp, -32
-; RV32IZFINXZDINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32IZFINXZDINX-NEXT:    call ceil
-; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI5_0)
-; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI5_0+4)(a2)
-; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI5_0)(a2)
 ; RV32IZFINXZDINX-NEXT:    mv s0, a0
 ; RV32IZFINXZDINX-NEXT:    mv s1, a1
-; RV32IZFINXZDINX-NEXT:    fle.d s2, a2, s0
-; RV32IZFINXZDINX-NEXT:    neg s3, s2
 ; RV32IZFINXZDINX-NEXT:    call __fixdfdi
-; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI5_1)
-; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI5_1+4)(a2)
-; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI5_1)(a2)
-; RV32IZFINXZDINX-NEXT:    and a0, s3, a0
-; RV32IZFINXZDINX-NEXT:    flt.d a3, a2, s0
-; RV32IZFINXZDINX-NEXT:    neg a2, a3
-; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    feq.d a2, s0, s0
-; RV32IZFINXZDINX-NEXT:    neg a2, a2
+; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI5_0)
+; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI5_0+4)(a2)
+; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI5_0)(a2)
+; RV32IZFINXZDINX-NEXT:    fle.d a2, a2, s0
 ; RV32IZFINXZDINX-NEXT:    lui a5, 524288
-; RV32IZFINXZDINX-NEXT:    li a6, 1
-; RV32IZFINXZDINX-NEXT:    lui a4, 524288
-; RV32IZFINXZDINX-NEXT:    bne s2, a6, .LBB5_2
+; RV32IZFINXZDINX-NEXT:    li a4, 1
+; RV32IZFINXZDINX-NEXT:    lui a3, 524288
+; RV32IZFINXZDINX-NEXT:    bne a2, a4, .LBB5_2
 ; RV32IZFINXZDINX-NEXT:  # %bb.1:
-; RV32IZFINXZDINX-NEXT:    mv a4, a1
+; RV32IZFINXZDINX-NEXT:    mv a3, a1
 ; RV32IZFINXZDINX-NEXT:  .LBB5_2:
-; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    beqz a3, .LBB5_4
+; RV32IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI5_1)
+; RV32IZFINXZDINX-NEXT:    lw a6, %lo(.LCPI5_1)(a1)
+; RV32IZFINXZDINX-NEXT:    lw a7, %lo(.LCPI5_1+4)(a1)
+; RV32IZFINXZDINX-NEXT:    flt.d a4, a6, s0
+; RV32IZFINXZDINX-NEXT:    beqz a4, .LBB5_4
 ; RV32IZFINXZDINX-NEXT:  # %bb.3:
-; RV32IZFINXZDINX-NEXT:    addi a4, a5, -1
+; RV32IZFINXZDINX-NEXT:    addi a3, a5, -1
 ; RV32IZFINXZDINX-NEXT:  .LBB5_4:
-; RV32IZFINXZDINX-NEXT:    and a1, a2, a4
-; RV32IZFINXZDINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    addi sp, sp, 32
+; RV32IZFINXZDINX-NEXT:    feq.d a1, s0, s0
+; RV32IZFINXZDINX-NEXT:    neg a5, a1
+; RV32IZFINXZDINX-NEXT:    and a1, a5, a3
+; RV32IZFINXZDINX-NEXT:    neg a2, a2
+; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    neg a2, a4
+; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    and a0, a5, a0
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
 ; RV32IZFINXZDINX-NEXT:    ret
 ;
 ; RV64IZFINXZDINX-LABEL: test_ceil_si64:
@@ -555,48 +535,42 @@ define signext i32 @test_trunc_si32(double %x) {
 define i64 @test_trunc_si64(double %x) nounwind {
 ; RV32IFD-LABEL: test_trunc_si64:
 ; RV32IFD:       # %bb.0:
-; RV32IFD-NEXT:    addi sp, sp, -32
-; RV32IFD-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    fsd fs0, 0(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    call trunc
 ; RV32IFD-NEXT:    lui a0, %hi(.LCPI9_0)
 ; RV32IFD-NEXT:    fld fa5, %lo(.LCPI9_0)(a0)
-; RV32IFD-NEXT:    lui a0, %hi(.LCPI9_1)
-; RV32IFD-NEXT:    fld fa4, %lo(.LCPI9_1)(a0)
 ; RV32IFD-NEXT:    fmv.d fs0, fa0
-; RV32IFD-NEXT:    flt.d s0, fa5, fa0
-; RV32IFD-NEXT:    neg s1, s0
-; RV32IFD-NEXT:    fle.d s2, fa4, fa0
-; RV32IFD-NEXT:    neg s3, s2
+; RV32IFD-NEXT:    fle.d s0, fa5, fa0
 ; RV32IFD-NEXT:    call __fixdfdi
-; RV32IFD-NEXT:    and a0, s3, a0
-; RV32IFD-NEXT:    or a0, s1, a0
-; RV32IFD-NEXT:    feq.d a2, fs0, fs0
-; RV32IFD-NEXT:    neg a2, a2
-; RV32IFD-NEXT:    lui a4, 524288
-; RV32IFD-NEXT:    li a5, 1
 ; RV32IFD-NEXT:    lui a3, 524288
-; RV32IFD-NEXT:    bne s2, a5, .LBB9_2
+; RV32IFD-NEXT:    li a4, 1
+; RV32IFD-NEXT:    lui a2, 524288
+; RV32IFD-NEXT:    bne s0, a4, .LBB9_2
 ; RV32IFD-NEXT:  # %bb.1:
-; RV32IFD-NEXT:    mv a3, a1
+; RV32IFD-NEXT:    mv a2, a1
 ; RV32IFD-NEXT:  .LBB9_2:
-; RV32IFD-NEXT:    and a0, a2, a0
-; RV32IFD-NEXT:    beqz s0, .LBB9_4
+; RV32IFD-NEXT:    lui a1, %hi(.LCPI9_1)
+; RV32IFD-NEXT:    fld fa5, %lo(.LCPI9_1)(a1)
+; RV32IFD-NEXT:    flt.d a4, fa5, fs0
+; RV32IFD-NEXT:    beqz a4, .LBB9_4
 ; RV32IFD-NEXT:  # %bb.3:
-; RV32IFD-NEXT:    addi a3, a4, -1
+; RV32IFD-NEXT:    addi a2, a3, -1
 ; RV32IFD-NEXT:  .LBB9_4:
-; RV32IFD-NEXT:    and a1, a2, a3
-; RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    feq.d a1, fs0, fs0
+; RV32IFD-NEXT:    neg a3, a1
+; RV32IFD-NEXT:    and a1, a3, a2
+; RV32IFD-NEXT:    neg a2, a4
+; RV32IFD-NEXT:    neg a4, s0
+; RV32IFD-NEXT:    and a0, a4, a0
+; RV32IFD-NEXT:    or a0, a2, a0
+; RV32IFD-NEXT:    and a0, a3, a0
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
-; RV32IFD-NEXT:    addi sp, sp, 32
+; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: test_trunc_si64:
@@ -610,49 +584,45 @@ define i64 @test_trunc_si64(double %x) nounwind {
 ;
 ; RV32IZFINXZDINX-LABEL: test_trunc_si64:
 ; RV32IZFINXZDINX:       # %bb.0:
-; RV32IZFINXZDINX-NEXT:    addi sp, sp, -32
-; RV32IZFINXZDINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32IZFINXZDINX-NEXT:    call trunc
-; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI9_0)
-; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI9_0+4)(a2)
-; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI9_0)(a2)
 ; RV32IZFINXZDINX-NEXT:    mv s0, a0
 ; RV32IZFINXZDINX-NEXT:    mv s1, a1
-; RV32IZFINXZDINX-NEXT:    fle.d s2, a2, s0
-; RV32IZFINXZDINX-NEXT:    neg s3, s2
 ; RV32IZFINXZDINX-NEXT:    call __fixdfdi
-; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI9_1)
-; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI9_1+4)(a2)
-; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI9_1)(a2)
-; RV32IZFINXZDINX-NEXT:    and a0, s3, a0
-; RV32IZFINXZDINX-NEXT:    flt.d a3, a2, s0
-; RV32IZFINXZDINX-NEXT:    neg a2, a3
-; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    feq.d a2, s0, s0
-; RV32IZFINXZDINX-NEXT:    neg a2, a2
+; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI9_0)
+; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI9_0+4)(a2)
+; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI9_0)(a2)
+; RV32IZFINXZDINX-NEXT:    fle.d a2, a2, s0
 ; RV32IZFINXZDINX-NEXT:    lui a5, 524288
-; RV32IZFINXZDINX-NEXT:    li a6, 1
-; RV32IZFINXZDINX-NEXT:    lui a4, 524288
-; RV32IZFINXZDINX-NEXT:    bne s2, a6, .LBB9_2
+; RV32IZFINXZDINX-NEXT:    li a4, 1
+; RV32IZFINXZDINX-NEXT:    lui a3, 524288
+; RV32IZFINXZDINX-NEXT:    bne a2, a4, .LBB9_2
 ; RV32IZFINXZDINX-NEXT:  # %bb.1:
-; RV32IZFINXZDINX-NEXT:    mv a4, a1
+; RV32IZFINXZDINX-NEXT:    mv a3, a1
 ; RV32IZFINXZDINX-NEXT:  .LBB9_2:
-; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    beqz a3, .LBB9_4
+; RV32IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI9_1)
+; RV32IZFINXZDINX-NEXT:    lw a6, %lo(.LCPI9_1)(a1)
+; RV32IZFINXZDINX-NEXT:    lw a7, %lo(.LCPI9_1+4)(a1)
+; RV32IZFINXZDINX-NEXT:    flt.d a4, a6, s0
+; RV32IZFINXZDINX-NEXT:    beqz a4, .LBB9_4
 ; RV32IZFINXZDINX-NEXT:  # %bb.3:
-; RV32IZFINXZDINX-NEXT:    addi a4, a5, -1
+; RV32IZFINXZDINX-NEXT:    addi a3, a5, -1
 ; RV32IZFINXZDINX-NEXT:  .LBB9_4:
-; RV32IZFINXZDINX-NEXT:    and a1, a2, a4
-; RV32IZFINXZDINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    addi sp, sp, 32
+; RV32IZFINXZDINX-NEXT:    feq.d a1, s0, s0
+; RV32IZFINXZDINX-NEXT:    neg a5, a1
+; RV32IZFINXZDINX-NEXT:    and a1, a5, a3
+; RV32IZFINXZDINX-NEXT:    neg a2, a2
+; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    neg a2, a4
+; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    and a0, a5, a0
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
 ; RV32IZFINXZDINX-NEXT:    ret
 ;
 ; RV64IZFINXZDINX-LABEL: test_trunc_si64:
@@ -811,48 +781,42 @@ define signext i32 @test_round_si32(double %x) {
 define i64 @test_round_si64(double %x) nounwind {
 ; RV32IFD-LABEL: test_round_si64:
 ; RV32IFD:       # %bb.0:
-; RV32IFD-NEXT:    addi sp, sp, -32
-; RV32IFD-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    fsd fs0, 0(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    call round
 ; RV32IFD-NEXT:    lui a0, %hi(.LCPI13_0)
 ; RV32IFD-NEXT:    fld fa5, %lo(.LCPI13_0)(a0)
-; RV32IFD-NEXT:    lui a0, %hi(.LCPI13_1)
-; RV32IFD-NEXT:    fld fa4, %lo(.LCPI13_1)(a0)
 ; RV32IFD-NEXT:    fmv.d fs0, fa0
-; RV32IFD-NEXT:    flt.d s0, fa5, fa0
-; RV32IFD-NEXT:    neg s1, s0
-; RV32IFD-NEXT:    fle.d s2, fa4, fa0
-; RV32IFD-NEXT:    neg s3, s2
+; RV32IFD-NEXT:    fle.d s0, fa5, fa0
 ; RV32IFD-NEXT:    call __fixdfdi
-; RV32IFD-NEXT:    and a0, s3, a0
-; RV32IFD-NEXT:    or a0, s1, a0
-; RV32IFD-NEXT:    feq.d a2, fs0, fs0
-; RV32IFD-NEXT:    neg a2, a2
-; RV32IFD-NEXT:    lui a4, 524288
-; RV32IFD-NEXT:    li a5, 1
 ; RV32IFD-NEXT:    lui a3, 524288
-; RV32IFD-NEXT:    bne s2, a5, .LBB13_2
+; RV32IFD-NEXT:    li a4, 1
+; RV32IFD-NEXT:    lui a2, 524288
+; RV32IFD-NEXT:    bne s0, a4, .LBB13_2
 ; RV32IFD-NEXT:  # %bb.1:
-; RV32IFD-NEXT:    mv a3, a1
+; RV32IFD-NEXT:    mv a2, a1
 ; RV32IFD-NEXT:  .LBB13_2:
-; RV32IFD-NEXT:    and a0, a2, a0
-; RV32IFD-NEXT:    beqz s0, .LBB13_4
+; RV32IFD-NEXT:    lui a1, %hi(.LCPI13_1)
+; RV32IFD-NEXT:    fld fa5, %lo(.LCPI13_1)(a1)
+; RV32IFD-NEXT:    flt.d a4, fa5, fs0
+; RV32IFD-NEXT:    beqz a4, .LBB13_4
 ; RV32IFD-NEXT:  # %bb.3:
-; RV32IFD-NEXT:    addi a3, a4, -1
+; RV32IFD-NEXT:    addi a2, a3, -1
 ; RV32IFD-NEXT:  .LBB13_4:
-; RV32IFD-NEXT:    and a1, a2, a3
-; RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    feq.d a1, fs0, fs0
+; RV32IFD-NEXT:    neg a3, a1
+; RV32IFD-NEXT:    and a1, a3, a2
+; RV32IFD-NEXT:    neg a2, a4
+; RV32IFD-NEXT:    neg a4, s0
+; RV32IFD-NEXT:    and a0, a4, a0
+; RV32IFD-NEXT:    or a0, a2, a0
+; RV32IFD-NEXT:    and a0, a3, a0
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
-; RV32IFD-NEXT:    addi sp, sp, 32
+; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: test_round_si64:
@@ -866,49 +830,45 @@ define i64 @test_round_si64(double %x) nounwind {
 ;
 ; RV32IZFINXZDINX-LABEL: test_round_si64:
 ; RV32IZFINXZDINX:       # %bb.0:
-; RV32IZFINXZDINX-NEXT:    addi sp, sp, -32
-; RV32IZFINXZDINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32IZFINXZDINX-NEXT:    call round
-; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI13_0)
-; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI13_0+4)(a2)
-; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI13_0)(a2)
 ; RV32IZFINXZDINX-NEXT:    mv s0, a0
 ; RV32IZFINXZDINX-NEXT:    mv s1, a1
-; RV32IZFINXZDINX-NEXT:    fle.d s2, a2, s0
-; RV32IZFINXZDINX-NEXT:    neg s3, s2
 ; RV32IZFINXZDINX-NEXT:    call __fixdfdi
-; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI13_1)
-; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI13_1+4)(a2)
-; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI13_1)(a2)
-; RV32IZFINXZDINX-NEXT:    and a0, s3, a0
-; RV32IZFINXZDINX-NEXT:    flt.d a3, a2, s0
-; RV32IZFINXZDINX-NEXT:    neg a2, a3
-; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    feq.d a2, s0, s0
-; RV32IZFINXZDINX-NEXT:    neg a2, a2
+; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI13_0)
+; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI13_0+4)(a2)
+; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI13_0)(a2)
+; RV32IZFINXZDINX-NEXT:    fle.d a2, a2, s0
 ; RV32IZFINXZDINX-NEXT:    lui a5, 524288
-; RV32IZFINXZDINX-NEXT:    li a6, 1
-; RV32IZFINXZDINX-NEXT:    lui a4, 524288
-; RV32IZFINXZDINX-NEXT:    bne s2, a6, .LBB13_2
+; RV32IZFINXZDINX-NEXT:    li a4, 1
+; RV32IZFINXZDINX-NEXT:    lui a3, 524288
+; RV32IZFINXZDINX-NEXT:    bne a2, a4, .LBB13_2
 ; RV32IZFINXZDINX-NEXT:  # %bb.1:
-; RV32IZFINXZDINX-NEXT:    mv a4, a1
+; RV32IZFINXZDINX-NEXT:    mv a3, a1
 ; RV32IZFINXZDINX-NEXT:  .LBB13_2:
-; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    beqz a3, .LBB13_4
+; RV32IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI13_1)
+; RV32IZFINXZDINX-NEXT:    lw a6, %lo(.LCPI13_1)(a1)
+; RV32IZFINXZDINX-NEXT:    lw a7, %lo(.LCPI13_1+4)(a1)
+; RV32IZFINXZDINX-NEXT:    flt.d a4, a6, s0
+; RV32IZFINXZDINX-NEXT:    beqz a4, .LBB13_4
 ; RV32IZFINXZDINX-NEXT:  # %bb.3:
-; RV32IZFINXZDINX-NEXT:    addi a4, a5, -1
+; RV32IZFINXZDINX-NEXT:    addi a3, a5, -1
 ; RV32IZFINXZDINX-NEXT:  .LBB13_4:
-; RV32IZFINXZDINX-NEXT:    and a1, a2, a4
-; RV32IZFINXZDINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    addi sp, sp, 32
+; RV32IZFINXZDINX-NEXT:    feq.d a1, s0, s0
+; RV32IZFINXZDINX-NEXT:    neg a5, a1
+; RV32IZFINXZDINX-NEXT:    and a1, a5, a3
+; RV32IZFINXZDINX-NEXT:    neg a2, a2
+; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    neg a2, a4
+; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    and a0, a5, a0
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
 ; RV32IZFINXZDINX-NEXT:    ret
 ;
 ; RV64IZFINXZDINX-LABEL: test_round_si64:
@@ -1067,48 +1027,42 @@ define signext i32 @test_roundeven_si32(double %x) {
 define i64 @test_roundeven_si64(double %x) nounwind {
 ; RV32IFD-LABEL: test_roundeven_si64:
 ; RV32IFD:       # %bb.0:
-; RV32IFD-NEXT:    addi sp, sp, -32
-; RV32IFD-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    fsd fs0, 0(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    call roundeven
 ; RV32IFD-NEXT:    lui a0, %hi(.LCPI17_0)
 ; RV32IFD-NEXT:    fld fa5, %lo(.LCPI17_0)(a0)
-; RV32IFD-NEXT:    lui a0, %hi(.LCPI17_1)
-; RV32IFD-NEXT:    fld fa4, %lo(.LCPI17_1)(a0)
 ; RV32IFD-NEXT:    fmv.d fs0, fa0
-; RV32IFD-NEXT:    flt.d s0, fa5, fa0
-; RV32IFD-NEXT:    neg s1, s0
-; RV32IFD-NEXT:    fle.d s2, fa4, fa0
-; RV32IFD-NEXT:    neg s3, s2
+; RV32IFD-NEXT:    fle.d s0, fa5, fa0
 ; RV32IFD-NEXT:    call __fixdfdi
-; RV32IFD-NEXT:    and a0, s3, a0
-; RV32IFD-NEXT:    or a0, s1, a0
-; RV32IFD-NEXT:    feq.d a2, fs0, fs0
-; RV32IFD-NEXT:    neg a2, a2
-; RV32IFD-NEXT:    lui a4, 524288
-; RV32IFD-NEXT:    li a5, 1
 ; RV32IFD-NEXT:    lui a3, 524288
-; RV32IFD-NEXT:    bne s2, a5, .LBB17_2
+; RV32IFD-NEXT:    li a4, 1
+; RV32IFD-NEXT:    lui a2, 524288
+; RV32IFD-NEXT:    bne s0, a4, .LBB17_2
 ; RV32IFD-NEXT:  # %bb.1:
-; RV32IFD-NEXT:    mv a3, a1
+; RV32IFD-NEXT:    mv a2, a1
 ; RV32IFD-NEXT:  .LBB17_2:
-; RV32IFD-NEXT:    and a0, a2, a0
-; RV32IFD-NEXT:    beqz s0, .LBB17_4
+; RV32IFD-NEXT:    lui a1, %hi(.LCPI17_1)
+; RV32IFD-NEXT:    fld fa5, %lo(.LCPI17_1)(a1)
+; RV32IFD-NEXT:    flt.d a4, fa5, fs0
+; RV32IFD-NEXT:    beqz a4, .LBB17_4
 ; RV32IFD-NEXT:  # %bb.3:
-; RV32IFD-NEXT:    addi a3, a4, -1
+; RV32IFD-NEXT:    addi a2, a3, -1
 ; RV32IFD-NEXT:  .LBB17_4:
-; RV32IFD-NEXT:    and a1, a2, a3
-; RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    feq.d a1, fs0, fs0
+; RV32IFD-NEXT:    neg a3, a1
+; RV32IFD-NEXT:    and a1, a3, a2
+; RV32IFD-NEXT:    neg a2, a4
+; RV32IFD-NEXT:    neg a4, s0
+; RV32IFD-NEXT:    and a0, a4, a0
+; RV32IFD-NEXT:    or a0, a2, a0
+; RV32IFD-NEXT:    and a0, a3, a0
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
-; RV32IFD-NEXT:    addi sp, sp, 32
+; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: test_roundeven_si64:
@@ -1122,49 +1076,45 @@ define i64 @test_roundeven_si64(double %x) nounwind {
 ;
 ; RV32IZFINXZDINX-LABEL: test_roundeven_si64:
 ; RV32IZFINXZDINX:       # %bb.0:
-; RV32IZFINXZDINX-NEXT:    addi sp, sp, -32
-; RV32IZFINXZDINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32IZFINXZDINX-NEXT:    call roundeven
-; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI17_0)
-; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI17_0+4)(a2)
-; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI17_0)(a2)
 ; RV32IZFINXZDINX-NEXT:    mv s0, a0
 ; RV32IZFINXZDINX-NEXT:    mv s1, a1
-; RV32IZFINXZDINX-NEXT:    fle.d s2, a2, s0
-; RV32IZFINXZDINX-NEXT:    neg s3, s2
 ; RV32IZFINXZDINX-NEXT:    call __fixdfdi
-; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI17_1)
-; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI17_1+4)(a2)
-; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI17_1)(a2)
-; RV32IZFINXZDINX-NEXT:    and a0, s3, a0
-; RV32IZFINXZDINX-NEXT:    flt.d a3, a2, s0
-; RV32IZFINXZDINX-NEXT:    neg a2, a3
-; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    feq.d a2, s0, s0
-; RV32IZFINXZDINX-NEXT:    neg a2, a2
+; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI17_0)
+; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI17_0+4)(a2)
+; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI17_0)(a2)
+; RV32IZFINXZDINX-NEXT:    fle.d a2, a2, s0
 ; RV32IZFINXZDINX-NEXT:    lui a5, 524288
-; RV32IZFINXZDINX-NEXT:    li a6, 1
-; RV32IZFINXZDINX-NEXT:    lui a4, 524288
-; RV32IZFINXZDINX-NEXT:    bne s2, a6, .LBB17_2
+; RV32IZFINXZDINX-NEXT:    li a4, 1
+; RV32IZFINXZDINX-NEXT:    lui a3, 524288
+; RV32IZFINXZDINX-NEXT:    bne a2, a4, .LBB17_2
 ; RV32IZFINXZDINX-NEXT:  # %bb.1:
-; RV32IZFINXZDINX-NEXT:    mv a4, a1
+; RV32IZFINXZDINX-NEXT:    mv a3, a1
 ; RV32IZFINXZDINX-NEXT:  .LBB17_2:
-; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    beqz a3, .LBB17_4
+; RV32IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI17_1)
+; RV32IZFINXZDINX-NEXT:    lw a6, %lo(.LCPI17_1)(a1)
+; RV32IZFINXZDINX-NEXT:    lw a7, %lo(.LCPI17_1+4)(a1)
+; RV32IZFINXZDINX-NEXT:    flt.d a4, a6, s0
+; RV32IZFINXZDINX-NEXT:    beqz a4, .LBB17_4
 ; RV32IZFINXZDINX-NEXT:  # %bb.3:
-; RV32IZFINXZDINX-NEXT:    addi a4, a5, -1
+; RV32IZFINXZDINX-NEXT:    addi a3, a5, -1
 ; RV32IZFINXZDINX-NEXT:  .LBB17_4:
-; RV32IZFINXZDINX-NEXT:    and a1, a2, a4
-; RV32IZFINXZDINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    addi sp, sp, 32
+; RV32IZFINXZDINX-NEXT:    feq.d a1, s0, s0
+; RV32IZFINXZDINX-NEXT:    neg a5, a1
+; RV32IZFINXZDINX-NEXT:    and a1, a5, a3
+; RV32IZFINXZDINX-NEXT:    neg a2, a2
+; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    neg a2, a4
+; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    and a0, a5, a0
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
 ; RV32IZFINXZDINX-NEXT:    ret
 ;
 ; RV64IZFINXZDINX-LABEL: test_roundeven_si64:
@@ -1323,48 +1273,42 @@ define signext i32 @test_rint_si32(double %x) {
 define i64 @test_rint_si64(double %x) nounwind {
 ; RV32IFD-LABEL: test_rint_si64:
 ; RV32IFD:       # %bb.0:
-; RV32IFD-NEXT:    addi sp, sp, -32
-; RV32IFD-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
-; RV32IFD-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    fsd fs0, 0(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    call rint
 ; RV32IFD-NEXT:    lui a0, %hi(.LCPI21_0)
 ; RV32IFD-NEXT:    fld fa5, %lo(.LCPI21_0)(a0)
-; RV32IFD-NEXT:    lui a0, %hi(.LCPI21_1)
-; RV32IFD-NEXT:    fld fa4, %lo(.LCPI21_1)(a0)
 ; RV32IFD-NEXT:    fmv.d fs0, fa0
-; RV32IFD-NEXT:    flt.d s0, fa5, fa0
-; RV32IFD-NEXT:    neg s1, s0
-; RV32IFD-NEXT:    fle.d s2, fa4, fa0
-; RV32IFD-NEXT:    neg s3, s2
+; RV32IFD-NEXT:    fle.d s0, fa5, fa0
 ; RV32IFD-NEXT:    call __fixdfdi
-; RV32IFD-NEXT:    and a0, s3, a0
-; RV32IFD-NEXT:    or a0, s1, a0
-; RV32IFD-NEXT:    feq.d a2, fs0, fs0
-; RV32IFD-NEXT:    neg a2, a2
-; RV32IFD-NEXT:    lui a4, 524288
-; RV32IFD-NEXT:    li a5, 1
 ; RV32IFD-NEXT:    lui a3, 524288
-; RV32IFD-NEXT:    bne s2, a5, .LBB21_2
+; RV32IFD-NEXT:    li a4, 1
+; RV32IFD-NEXT:    lui a2, 524288
+; RV32IFD-NEXT:    bne s0, a4, .LBB21_2
 ; RV32IFD-NEXT:  # %bb.1:
-; RV32IFD-NEXT:    mv a3, a1
+; RV32IFD-NEXT:    mv a2, a1
 ; RV32IFD-NEXT:  .LBB21_2:
-; RV32IFD-NEXT:    and a0, a2, a0
-; RV32IFD-NEXT:    beqz s0, .LBB21_4
+; RV32IFD-NEXT:    lui a1, %hi(.LCPI21_1)
+; RV32IFD-NEXT:    fld fa5, %lo(.LCPI21_1)(a1)
+; RV32IFD-NEXT:    flt.d a4, fa5, fs0
+; RV32IFD-NEXT:    beqz a4, .LBB21_4
 ; RV32IFD-NEXT:  # %bb.3:
-; RV32IFD-NEXT:    addi a3, a4, -1
+; RV32IFD-NEXT:    addi a2, a3, -1
 ; RV32IFD-NEXT:  .LBB21_4:
-; RV32IFD-NEXT:    and a1, a2, a3
-; RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
-; RV32IFD-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    feq.d a1, fs0, fs0
+; RV32IFD-NEXT:    neg a3, a1
+; RV32IFD-NEXT:    and a1, a3, a2
+; RV32IFD-NEXT:    neg a2, a4
+; RV32IFD-NEXT:    neg a4, s0
+; RV32IFD-NEXT:    and a0, a4, a0
+; RV32IFD-NEXT:    or a0, a2, a0
+; RV32IFD-NEXT:    and a0, a3, a0
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
-; RV32IFD-NEXT:    addi sp, sp, 32
+; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: test_rint_si64:
@@ -1378,49 +1322,45 @@ define i64 @test_rint_si64(double %x) nounwind {
 ;
 ; RV32IZFINXZDINX-LABEL: test_rint_si64:
 ; RV32IZFINXZDINX:       # %bb.0:
-; RV32IZFINXZDINX-NEXT:    addi sp, sp, -32
-; RV32IZFINXZDINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
-; RV32IZFINXZDINX-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, -16
+; RV32IZFINXZDINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IZFINXZDINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32IZFINXZDINX-NEXT:    call rint
-; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI21_0)
-; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI21_0+4)(a2)
-; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI21_0)(a2)
 ; RV32IZFINXZDINX-NEXT:    mv s0, a0
 ; RV32IZFINXZDINX-NEXT:    mv s1, a1
-; RV32IZFINXZDINX-NEXT:    fle.d s2, a2, s0
-; RV32IZFINXZDINX-NEXT:    neg s3, s2
 ; RV32IZFINXZDINX-NEXT:    call __fixdfdi
-; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI21_1)
-; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI21_1+4)(a2)
-; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI21_1)(a2)
-; RV32IZFINXZDINX-NEXT:    and a0, s3, a0
-; RV32IZFINXZDINX-NEXT:    flt.d a3, a2, s0
-; RV32IZFINXZDINX-NEXT:    neg a2, a3
-; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    feq.d a2, s0, s0
-; RV32IZFINXZDINX-NEXT:    neg a2, a2
+; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI21_0)
+; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI21_0+4)(a2)
+; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI21_0)(a2)
+; RV32IZFINXZDINX-NEXT:    fle.d a2, a2, s0
 ; RV32IZFINXZDINX-NEXT:    lui a5, 524288
-; RV32IZFINXZDINX-NEXT:    li a6, 1
-; RV32IZFINXZDINX-NEXT:    lui a4, 524288
-; RV32IZFINXZDINX-NEXT:    bne s2, a6, .LBB21_2
+; RV32IZFINXZDINX-NEXT:    li a4, 1
+; RV32IZFINXZDINX-NEXT:    lui a3, 524288
+; RV32IZFINXZDINX-NEXT:    bne a2, a4, .LBB21_2
 ; RV32IZFINXZDINX-NEXT:  # %bb.1:
-; RV32IZFINXZDINX-NEXT:    mv a4, a1
+; RV32IZFINXZDINX-NEXT:    mv a3, a1
 ; RV32IZFINXZDINX-NEXT:  .LBB21_2:
-; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    beqz a3, .LBB21_4
+; RV32IZFINXZDINX-NEXT:    lui a1, %hi(.LCPI21_1)
+; RV32IZFINXZDINX-NEXT:    lw a6, %lo(.LCPI21_1)(a1)
+; RV32IZFINXZDINX-NEXT:    lw a7, %lo(.LCPI21_1+4)(a1)
+; RV32IZFINXZDINX-NEXT:    flt.d a4, a6, s0
+; RV32IZFINXZDINX-NEXT:    beqz a4, .LBB21_4
 ; RV32IZFINXZDINX-NEXT:  # %bb.3:
-; RV32IZFINXZDINX-NEXT:    addi a4, a5, -1
+; RV32IZFINXZDINX-NEXT:    addi a3, a5, -1
 ; RV32IZFINXZDINX-NEXT:  .LBB21_4:
-; RV32IZFINXZDINX-NEXT:    and a1, a2, a4
-; RV32IZFINXZDINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
-; RV32IZFINXZDINX-NEXT:    addi sp, sp, 32
+; RV32IZFINXZDINX-NEXT:    feq.d a1, s0, s0
+; RV32IZFINXZDINX-NEXT:    neg a5, a1
+; RV32IZFINXZDINX-NEXT:    and a1, a5, a3
+; RV32IZFINXZDINX-NEXT:    neg a2, a2
+; RV32IZFINXZDINX-NEXT:    and a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    neg a2, a4
+; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    and a0, a5, a0
+; RV32IZFINXZDINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IZFINXZDINX-NEXT:    addi sp, sp, 16
 ; RV32IZFINXZDINX-NEXT:    ret
 ;
 ; RV64IZFINXZDINX-LABEL: test_rint_si64:
diff --git a/llvm/test/CodeGen/RISCV/float-convert.ll b/llvm/test/CodeGen/RISCV/float-convert.ll
index 9fb78d4c4d52..2c7315fbe59f 100644
--- a/llvm/test/CodeGen/RISCV/float-convert.ll
+++ b/llvm/test/CodeGen/RISCV/float-convert.ll
@@ -629,23 +629,23 @@ define i64 @fcvt_l_s_sat(float %a) nounwind {
 ; RV32IF-NEXT:    lui a2, %hi(.LCPI12_0)
 ; RV32IF-NEXT:    flw fa5, %lo(.LCPI12_0)(a2)
 ; RV32IF-NEXT:    and a0, s1, a0
-; RV32IF-NEXT:    flt.s a4, fa5, fs0
-; RV32IF-NEXT:    neg a2, a4
+; RV32IF-NEXT:    flt.s a3, fa5, fs0
+; RV32IF-NEXT:    neg a2, a3
 ; RV32IF-NEXT:    or a0, a2, a0
 ; RV32IF-NEXT:    feq.s a2, fs0, fs0
 ; RV32IF-NEXT:    neg a2, a2
 ; RV32IF-NEXT:    lui a5, 524288
-; RV32IF-NEXT:    lui a3, 524288
+; RV32IF-NEXT:    lui a4, 524288
 ; RV32IF-NEXT:    beqz s0, .LBB12_2
 ; RV32IF-NEXT:  # %bb.1: # %start
-; RV32IF-NEXT:    mv a3, a1
+; RV32IF-NEXT:    mv a4, a1
 ; RV32IF-NEXT:  .LBB12_2: # %start
 ; RV32IF-NEXT:    and a0, a2, a0
-; RV32IF-NEXT:    beqz a4, .LBB12_4
+; RV32IF-NEXT:    beqz a3, .LBB12_4
 ; RV32IF-NEXT:  # %bb.3:
-; RV32IF-NEXT:    addi a3, a5, -1
+; RV32IF-NEXT:    addi a4, a5, -1
 ; RV32IF-NEXT:  .LBB12_4: # %start
-; RV32IF-NEXT:    and a1, a2, a3
+; RV32IF-NEXT:    and a1, a2, a4
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -668,37 +668,35 @@ define i64 @fcvt_l_s_sat(float %a) nounwind {
 ; RV32IZFINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IZFINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32IZFINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
-; RV32IZFINX-NEXT:    sw s2, 0(sp) # 4-byte Folded Spill
 ; RV32IZFINX-NEXT:    mv s0, a0
 ; RV32IZFINX-NEXT:    lui a0, 913408
 ; RV32IZFINX-NEXT:    fle.s s1, a0, s0
-; RV32IZFINX-NEXT:    neg s2, s1
 ; RV32IZFINX-NEXT:    mv a0, s0
 ; RV32IZFINX-NEXT:    call __fixsfdi
-; RV32IZFINX-NEXT:    lui a2, %hi(.LCPI12_0)
-; RV32IZFINX-NEXT:    lw a2, %lo(.LCPI12_0)(a2)
-; RV32IZFINX-NEXT:    and a0, s2, a0
-; RV32IZFINX-NEXT:    flt.s a4, a2, s0
-; RV32IZFINX-NEXT:    neg a2, a4
-; RV32IZFINX-NEXT:    or a0, a2, a0
-; RV32IZFINX-NEXT:    feq.s a2, s0, s0
-; RV32IZFINX-NEXT:    neg a2, a2
-; RV32IZFINX-NEXT:    lui a5, 524288
-; RV32IZFINX-NEXT:    lui a3, 524288
+; RV32IZFINX-NEXT:    lui a4, 524288
+; RV32IZFINX-NEXT:    lui a2, 524288
 ; RV32IZFINX-NEXT:    beqz s1, .LBB12_2
 ; RV32IZFINX-NEXT:  # %bb.1: # %start
-; RV32IZFINX-NEXT:    mv a3, a1
+; RV32IZFINX-NEXT:    mv a2, a1
 ; RV32IZFINX-NEXT:  .LBB12_2: # %start
-; RV32IZFINX-NEXT:    and a0, a2, a0
-; RV32IZFINX-NEXT:    beqz a4, .LBB12_4
+; RV32IZFINX-NEXT:    lui a1, %hi(.LCPI12_0)
+; RV32IZFINX-NEXT:    lw a1, %lo(.LCPI12_0)(a1)
+; RV32IZFINX-NEXT:    flt.s a3, a1, s0
+; RV32IZFINX-NEXT:    beqz a3, .LBB12_4
 ; RV32IZFINX-NEXT:  # %bb.3:
-; RV32IZFINX-NEXT:    addi a3, a5, -1
+; RV32IZFINX-NEXT:    addi a2, a4, -1
 ; RV32IZFINX-NEXT:  .LBB12_4: # %start
-; RV32IZFINX-NEXT:    and a1, a2, a3
+; RV32IZFINX-NEXT:    feq.s a1, s0, s0
+; RV32IZFINX-NEXT:    neg a4, a1
+; RV32IZFINX-NEXT:    and a1, a4, a2
+; RV32IZFINX-NEXT:    neg a2, s1
+; RV32IZFINX-NEXT:    and a0, a2, a0
+; RV32IZFINX-NEXT:    neg a2, a3
+; RV32IZFINX-NEXT:    or a0, a2, a0
+; RV32IZFINX-NEXT:    and a0, a4, a0
 ; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
-; RV32IZFINX-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
 ; RV32IZFINX-NEXT:    addi sp, sp, 16
 ; RV32IZFINX-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/float-round-conv-sat.ll b/llvm/test/CodeGen/RISCV/float-round-conv-sat.ll
index c72e69c92a13..4f747c278da0 100644
--- a/llvm/test/CodeGen/RISCV/float-round-conv-sat.ll
+++ b/llvm/test/CodeGen/RISCV/float-round-conv-sat.ll
@@ -59,23 +59,23 @@ define i64 @test_floor_si64(float %x) nounwind {
 ; RV32IF-NEXT:    lui a2, %hi(.LCPI1_0)
 ; RV32IF-NEXT:    flw fa5, %lo(.LCPI1_0)(a2)
 ; RV32IF-NEXT:    and a0, s1, a0
-; RV32IF-NEXT:    flt.s a4, fa5, fs0
-; RV32IF-NEXT:    neg a2, a4
+; RV32IF-NEXT:    flt.s a3, fa5, fs0
+; RV32IF-NEXT:    neg a2, a3
 ; RV32IF-NEXT:    or a0, a2, a0
 ; RV32IF-NEXT:    feq.s a2, fs0, fs0
 ; RV32IF-NEXT:    neg a2, a2
 ; RV32IF-NEXT:    lui a5, 524288
-; RV32IF-NEXT:    lui a3, 524288
+; RV32IF-NEXT:    lui a4, 524288
 ; RV32IF-NEXT:    beqz s0, .LBB1_4
 ; RV32IF-NEXT:  # %bb.3:
-; RV32IF-NEXT:    mv a3, a1
+; RV32IF-NEXT:    mv a4, a1
 ; RV32IF-NEXT:  .LBB1_4:
 ; RV32IF-NEXT:    and a0, a2, a0
-; RV32IF-NEXT:    beqz a4, .LBB1_6
+; RV32IF-NEXT:    beqz a3, .LBB1_6
 ; RV32IF-NEXT:  # %bb.5:
-; RV32IF-NEXT:    addi a3, a5, -1
+; RV32IF-NEXT:    addi a4, a5, -1
 ; RV32IF-NEXT:  .LBB1_6:
-; RV32IF-NEXT:    and a1, a2, a3
+; RV32IF-NEXT:    and a1, a2, a4
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -117,23 +117,23 @@ define i64 @test_floor_si64(float %x) nounwind {
 ; RV32IZFINX-NEXT:    lui a2, %hi(.LCPI1_0)
 ; RV32IZFINX-NEXT:    lw a2, %lo(.LCPI1_0)(a2)
 ; RV32IZFINX-NEXT:    and a0, s2, a0
-; RV32IZFINX-NEXT:    flt.s a4, a2, s0
-; RV32IZFINX-NEXT:    neg a2, a4
+; RV32IZFINX-NEXT:    flt.s a3, a2, s0
+; RV32IZFINX-NEXT:    neg a2, a3
 ; RV32IZFINX-NEXT:    or a0, a2, a0
 ; RV32IZFINX-NEXT:    feq.s a2, s0, s0
 ; RV32IZFINX-NEXT:    neg a2, a2
 ; RV32IZFINX-NEXT:    lui a5, 524288
-; RV32IZFINX-NEXT:    lui a3, 524288
+; RV32IZFINX-NEXT:    lui a4, 524288
 ; RV32IZFINX-NEXT:    beqz s1, .LBB1_4
 ; RV32IZFINX-NEXT:  # %bb.3:
-; RV32IZFINX-NEXT:    mv a3, a1
+; RV32IZFINX-NEXT:    mv a4, a1
 ; RV32IZFINX-NEXT:  .LBB1_4:
 ; RV32IZFINX-NEXT:    and a0, a2, a0
-; RV32IZFINX-NEXT:    beqz a4, .LBB1_6
+; RV32IZFINX-NEXT:    beqz a3, .LBB1_6
 ; RV32IZFINX-NEXT:  # %bb.5:
-; RV32IZFINX-NEXT:    addi a3, a5, -1
+; RV32IZFINX-NEXT:    addi a4, a5, -1
 ; RV32IZFINX-NEXT:  .LBB1_6:
-; RV32IZFINX-NEXT:    and a1, a2, a3
+; RV32IZFINX-NEXT:    and a1, a2, a4
 ; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -321,23 +321,23 @@ define i64 @test_ceil_si64(float %x) nounwind {
 ; RV32IF-NEXT:    lui a2, %hi(.LCPI5_0)
 ; RV32IF-NEXT:    flw fa5, %lo(.LCPI5_0)(a2)
 ; RV32IF-NEXT:    and a0, s1, a0
-; RV32IF-NEXT:    flt.s a4, fa5, fs0
-; RV32IF-NEXT:    neg a2, a4
+; RV32IF-NEXT:    flt.s a3, fa5, fs0
+; RV32IF-NEXT:    neg a2, a3
 ; RV32IF-NEXT:    or a0, a2, a0
 ; RV32IF-NEXT:    feq.s a2, fs0, fs0
 ; RV32IF-NEXT:    neg a2, a2
 ; RV32IF-NEXT:    lui a5, 524288
-; RV32IF-NEXT:    lui a3, 524288
+; RV32IF-NEXT:    lui a4, 524288
 ; RV32IF-NEXT:    beqz s0, .LBB5_4
 ; RV32IF-NEXT:  # %bb.3:
-; RV32IF-NEXT:    mv a3, a1
+; RV32IF-NEXT:    mv a4, a1
 ; RV32IF-NEXT:  .LBB5_4:
 ; RV32IF-NEXT:    and a0, a2, a0
-; RV32IF-NEXT:    beqz a4, .LBB5_6
+; RV32IF-NEXT:    beqz a3, .LBB5_6
 ; RV32IF-NEXT:  # %bb.5:
-; RV32IF-NEXT:    addi a3, a5, -1
+; RV32IF-NEXT:    addi a4, a5, -1
 ; RV32IF-NEXT:  .LBB5_6:
-; RV32IF-NEXT:    and a1, a2, a3
+; RV32IF-NEXT:    and a1, a2, a4
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -379,23 +379,23 @@ define i64 @test_ceil_si64(float %x) nounwind {
 ; RV32IZFINX-NEXT:    lui a2, %hi(.LCPI5_0)
 ; RV32IZFINX-NEXT:    lw a2, %lo(.LCPI5_0)(a2)
 ; RV32IZFINX-NEXT:    and a0, s2, a0
-; RV32IZFINX-NEXT:    flt.s a4, a2, s0
-; RV32IZFINX-NEXT:    neg a2, a4
+; RV32IZFINX-NEXT:    flt.s a3, a2, s0
+; RV32IZFINX-NEXT:    neg a2, a3
 ; RV32IZFINX-NEXT:    or a0, a2, a0
 ; RV32IZFINX-NEXT:    feq.s a2, s0, s0
 ; RV32IZFINX-NEXT:    neg a2, a2
 ; RV32IZFINX-NEXT:    lui a5, 524288
-; RV32IZFINX-NEXT:    lui a3, 524288
+; RV32IZFINX-NEXT:    lui a4, 524288
 ; RV32IZFINX-NEXT:    beqz s1, .LBB5_4
 ; RV32IZFINX-NEXT:  # %bb.3:
-; RV32IZFINX-NEXT:    mv a3, a1
+; RV32IZFINX-NEXT:    mv a4, a1
 ; RV32IZFINX-NEXT:  .LBB5_4:
 ; RV32IZFINX-NEXT:    and a0, a2, a0
-; RV32IZFINX-NEXT:    beqz a4, .LBB5_6
+; RV32IZFINX-NEXT:    beqz a3, .LBB5_6
 ; RV32IZFINX-NEXT:  # %bb.5:
-; RV32IZFINX-NEXT:    addi a3, a5, -1
+; RV32IZFINX-NEXT:    addi a4, a5, -1
 ; RV32IZFINX-NEXT:  .LBB5_6:
-; RV32IZFINX-NEXT:    and a1, a2, a3
+; RV32IZFINX-NEXT:    and a1, a2, a4
 ; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -583,23 +583,23 @@ define i64 @test_trunc_si64(float %x) nounwind {
 ; RV32IF-NEXT:    lui a2, %hi(.LCPI9_0)
 ; RV32IF-NEXT:    flw fa5, %lo(.LCPI9_0)(a2)
 ; RV32IF-NEXT:    and a0, s1, a0
-; RV32IF-NEXT:    flt.s a4, fa5, fs0
-; RV32IF-NEXT:    neg a2, a4
+; RV32IF-NEXT:    flt.s a3, fa5, fs0
+; RV32IF-NEXT:    neg a2, a3
 ; RV32IF-NEXT:    or a0, a2, a0
 ; RV32IF-NEXT:    feq.s a2, fs0, fs0
 ; RV32IF-NEXT:    neg a2, a2
 ; RV32IF-NEXT:    lui a5, 524288
-; RV32IF-NEXT:    lui a3, 524288
+; RV32IF-NEXT:    lui a4, 524288
 ; RV32IF-NEXT:    beqz s0, .LBB9_4
 ; RV32IF-NEXT:  # %bb.3:
-; RV32IF-NEXT:    mv a3, a1
+; RV32IF-NEXT:    mv a4, a1
 ; RV32IF-NEXT:  .LBB9_4:
 ; RV32IF-NEXT:    and a0, a2, a0
-; RV32IF-NEXT:    beqz a4, .LBB9_6
+; RV32IF-NEXT:    beqz a3, .LBB9_6
 ; RV32IF-NEXT:  # %bb.5:
-; RV32IF-NEXT:    addi a3, a5, -1
+; RV32IF-NEXT:    addi a4, a5, -1
 ; RV32IF-NEXT:  .LBB9_6:
-; RV32IF-NEXT:    and a1, a2, a3
+; RV32IF-NEXT:    and a1, a2, a4
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -641,23 +641,23 @@ define i64 @test_trunc_si64(float %x) nounwind {
 ; RV32IZFINX-NEXT:    lui a2, %hi(.LCPI9_0)
 ; RV32IZFINX-NEXT:    lw a2, %lo(.LCPI9_0)(a2)
 ; RV32IZFINX-NEXT:    and a0, s2, a0
-; RV32IZFINX-NEXT:    flt.s a4, a2, s0
-; RV32IZFINX-NEXT:    neg a2, a4
+; RV32IZFINX-NEXT:    flt.s a3, a2, s0
+; RV32IZFINX-NEXT:    neg a2, a3
 ; RV32IZFINX-NEXT:    or a0, a2, a0
 ; RV32IZFINX-NEXT:    feq.s a2, s0, s0
 ; RV32IZFINX-NEXT:    neg a2, a2
 ; RV32IZFINX-NEXT:    lui a5, 524288
-; RV32IZFINX-NEXT:    lui a3, 524288
+; RV32IZFINX-NEXT:    lui a4, 524288
 ; RV32IZFINX-NEXT:    beqz s1, .LBB9_4
 ; RV32IZFINX-NEXT:  # %bb.3:
-; RV32IZFINX-NEXT:    mv a3, a1
+; RV32IZFINX-NEXT:    mv a4, a1
 ; RV32IZFINX-NEXT:  .LBB9_4:
 ; RV32IZFINX-NEXT:    and a0, a2, a0
-; RV32IZFINX-NEXT:    beqz a4, .LBB9_6
+; RV32IZFINX-NEXT:    beqz a3, .LBB9_6
 ; RV32IZFINX-NEXT:  # %bb.5:
-; RV32IZFINX-NEXT:    addi a3, a5, -1
+; RV32IZFINX-NEXT:    addi a4, a5, -1
 ; RV32IZFINX-NEXT:  .LBB9_6:
-; RV32IZFINX-NEXT:    and a1, a2, a3
+; RV32IZFINX-NEXT:    and a1, a2, a4
 ; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -845,23 +845,23 @@ define i64 @test_round_si64(float %x) nounwind {
 ; RV32IF-NEXT:    lui a2, %hi(.LCPI13_0)
 ; RV32IF-NEXT:    flw fa5, %lo(.LCPI13_0)(a2)
 ; RV32IF-NEXT:    and a0, s1, a0
-; RV32IF-NEXT:    flt.s a4, fa5, fs0
-; RV32IF-NEXT:    neg a2, a4
+; RV32IF-NEXT:    flt.s a3, fa5, fs0
+; RV32IF-NEXT:    neg a2, a3
 ; RV32IF-NEXT:    or a0, a2, a0
 ; RV32IF-NEXT:    feq.s a2, fs0, fs0
 ; RV32IF-NEXT:    neg a2, a2
 ; RV32IF-NEXT:    lui a5, 524288
-; RV32IF-NEXT:    lui a3, 524288
+; RV32IF-NEXT:    lui a4, 524288
 ; RV32IF-NEXT:    beqz s0, .LBB13_4
 ; RV32IF-NEXT:  # %bb.3:
-; RV32IF-NEXT:    mv a3, a1
+; RV32IF-NEXT:    mv a4, a1
 ; RV32IF-NEXT:  .LBB13_4:
 ; RV32IF-NEXT:    and a0, a2, a0
-; RV32IF-NEXT:    beqz a4, .LBB13_6
+; RV32IF-NEXT:    beqz a3, .LBB13_6
 ; RV32IF-NEXT:  # %bb.5:
-; RV32IF-NEXT:    addi a3, a5, -1
+; RV32IF-NEXT:    addi a4, a5, -1
 ; RV32IF-NEXT:  .LBB13_6:
-; RV32IF-NEXT:    and a1, a2, a3
+; RV32IF-NEXT:    and a1, a2, a4
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -903,23 +903,23 @@ define i64 @test_round_si64(float %x) nounwind {
 ; RV32IZFINX-NEXT:    lui a2, %hi(.LCPI13_0)
 ; RV32IZFINX-NEXT:    lw a2, %lo(.LCPI13_0)(a2)
 ; RV32IZFINX-NEXT:    and a0, s2, a0
-; RV32IZFINX-NEXT:    flt.s a4, a2, s0
-; RV32IZFINX-NEXT:    neg a2, a4
+; RV32IZFINX-NEXT:    flt.s a3, a2, s0
+; RV32IZFINX-NEXT:    neg a2, a3
 ; RV32IZFINX-NEXT:    or a0, a2, a0
 ; RV32IZFINX-NEXT:    feq.s a2, s0, s0
 ; RV32IZFINX-NEXT:    neg a2, a2
 ; RV32IZFINX-NEXT:    lui a5, 524288
-; RV32IZFINX-NEXT:    lui a3, 524288
+; RV32IZFINX-NEXT:    lui a4, 524288
 ; RV32IZFINX-NEXT:    beqz s1, .LBB13_4
 ; RV32IZFINX-NEXT:  # %bb.3:
-; RV32IZFINX-NEXT:    mv a3, a1
+; RV32IZFINX-NEXT:    mv a4, a1
 ; RV32IZFINX-NEXT:  .LBB13_4:
 ; RV32IZFINX-NEXT:    and a0, a2, a0
-; RV32IZFINX-NEXT:    beqz a4, .LBB13_6
+; RV32IZFINX-NEXT:    beqz a3, .LBB13_6
 ; RV32IZFINX-NEXT:  # %bb.5:
-; RV32IZFINX-NEXT:    addi a3, a5, -1
+; RV32IZFINX-NEXT:    addi a4, a5, -1
 ; RV32IZFINX-NEXT:  .LBB13_6:
-; RV32IZFINX-NEXT:    and a1, a2, a3
+; RV32IZFINX-NEXT:    and a1, a2, a4
 ; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -1107,23 +1107,23 @@ define i64 @test_roundeven_si64(float %x) nounwind {
 ; RV32IF-NEXT:    lui a2, %hi(.LCPI17_0)
 ; RV32IF-NEXT:    flw fa5, %lo(.LCPI17_0)(a2)
 ; RV32IF-NEXT:    and a0, s1, a0
-; RV32IF-NEXT:    flt.s a4, fa5, fs0
-; RV32IF-NEXT:    neg a2, a4
+; RV32IF-NEXT:    flt.s a3, fa5, fs0
+; RV32IF-NEXT:    neg a2, a3
 ; RV32IF-NEXT:    or a0, a2, a0
 ; RV32IF-NEXT:    feq.s a2, fs0, fs0
 ; RV32IF-NEXT:    neg a2, a2
 ; RV32IF-NEXT:    lui a5, 524288
-; RV32IF-NEXT:    lui a3, 524288
+; RV32IF-NEXT:    lui a4, 524288
 ; RV32IF-NEXT:    beqz s0, .LBB17_4
 ; RV32IF-NEXT:  # %bb.3:
-; RV32IF-NEXT:    mv a3, a1
+; RV32IF-NEXT:    mv a4, a1
 ; RV32IF-NEXT:  .LBB17_4:
 ; RV32IF-NEXT:    and a0, a2, a0
-; RV32IF-NEXT:    beqz a4, .LBB17_6
+; RV32IF-NEXT:    beqz a3, .LBB17_6
 ; RV32IF-NEXT:  # %bb.5:
-; RV32IF-NEXT:    addi a3, a5, -1
+; RV32IF-NEXT:    addi a4, a5, -1
 ; RV32IF-NEXT:  .LBB17_6:
-; RV32IF-NEXT:    and a1, a2, a3
+; RV32IF-NEXT:    and a1, a2, a4
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -1165,23 +1165,23 @@ define i64 @test_roundeven_si64(float %x) nounwind {
 ; RV32IZFINX-NEXT:    lui a2, %hi(.LCPI17_0)
 ; RV32IZFINX-NEXT:    lw a2, %lo(.LCPI17_0)(a2)
 ; RV32IZFINX-NEXT:    and a0, s2, a0
-; RV32IZFINX-NEXT:    flt.s a4, a2, s0
-; RV32IZFINX-NEXT:    neg a2, a4
+; RV32IZFINX-NEXT:    flt.s a3, a2, s0
+; RV32IZFINX-NEXT:    neg a2, a3
 ; RV32IZFINX-NEXT:    or a0, a2, a0
 ; RV32IZFINX-NEXT:    feq.s a2, s0, s0
 ; RV32IZFINX-NEXT:    neg a2, a2
 ; RV32IZFINX-NEXT:    lui a5, 524288
-; RV32IZFINX-NEXT:    lui a3, 524288
+; RV32IZFINX-NEXT:    lui a4, 524288
 ; RV32IZFINX-NEXT:    beqz s1, .LBB17_4
 ; RV32IZFINX-NEXT:  # %bb.3:
-; RV32IZFINX-NEXT:    mv a3, a1
+; RV32IZFINX-NEXT:    mv a4, a1
 ; RV32IZFINX-NEXT:  .LBB17_4:
 ; RV32IZFINX-NEXT:    and a0, a2, a0
-; RV32IZFINX-NEXT:    beqz a4, .LBB17_6
+; RV32IZFINX-NEXT:    beqz a3, .LBB17_6
 ; RV32IZFINX-NEXT:  # %bb.5:
-; RV32IZFINX-NEXT:    addi a3, a5, -1
+; RV32IZFINX-NEXT:    addi a4, a5, -1
 ; RV32IZFINX-NEXT:  .LBB17_6:
-; RV32IZFINX-NEXT:    and a1, a2, a3
+; RV32IZFINX-NEXT:    and a1, a2, a4
 ; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -1369,23 +1369,23 @@ define i64 @test_rint_si64(float %x) nounwind {
 ; RV32IF-NEXT:    lui a2, %hi(.LCPI21_0)
 ; RV32IF-NEXT:    flw fa5, %lo(.LCPI21_0)(a2)
 ; RV32IF-NEXT:    and a0, s1, a0
-; RV32IF-NEXT:    flt.s a4, fa5, fs0
-; RV32IF-NEXT:    neg a2, a4
+; RV32IF-NEXT:    flt.s a3, fa5, fs0
+; RV32IF-NEXT:    neg a2, a3
 ; RV32IF-NEXT:    or a0, a2, a0
 ; RV32IF-NEXT:    feq.s a2, fs0, fs0
 ; RV32IF-NEXT:    neg a2, a2
 ; RV32IF-NEXT:    lui a5, 524288
-; RV32IF-NEXT:    lui a3, 524288
+; RV32IF-NEXT:    lui a4, 524288
 ; RV32IF-NEXT:    beqz s0, .LBB21_4
 ; RV32IF-NEXT:  # %bb.3:
-; RV32IF-NEXT:    mv a3, a1
+; RV32IF-NEXT:    mv a4, a1
 ; RV32IF-NEXT:  .LBB21_4:
 ; RV32IF-NEXT:    and a0, a2, a0
-; RV32IF-NEXT:    beqz a4, .LBB21_6
+; RV32IF-NEXT:    beqz a3, .LBB21_6
 ; RV32IF-NEXT:  # %bb.5:
-; RV32IF-NEXT:    addi a3, a5, -1
+; RV32IF-NEXT:    addi a4, a5, -1
 ; RV32IF-NEXT:  .LBB21_6:
-; RV32IF-NEXT:    and a1, a2, a3
+; RV32IF-NEXT:    and a1, a2, a4
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -1427,23 +1427,23 @@ define i64 @test_rint_si64(float %x) nounwind {
 ; RV32IZFINX-NEXT:    lui a2, %hi(.LCPI21_0)
 ; RV32IZFINX-NEXT:    lw a2, %lo(.LCPI21_0)(a2)
 ; RV32IZFINX-NEXT:    and a0, s2, a0
-; RV32IZFINX-NEXT:    flt.s a4, a2, s0
-; RV32IZFINX-NEXT:    neg a2, a4
+; RV32IZFINX-NEXT:    flt.s a3, a2, s0
+; RV32IZFINX-NEXT:    neg a2, a3
 ; RV32IZFINX-NEXT:    or a0, a2, a0
 ; RV32IZFINX-NEXT:    feq.s a2, s0, s0
 ; RV32IZFINX-NEXT:    neg a2, a2
 ; RV32IZFINX-NEXT:    lui a5, 524288
-; RV32IZFINX-NEXT:    lui a3, 524288
+; RV32IZFINX-NEXT:    lui a4, 524288
 ; RV32IZFINX-NEXT:    beqz s1, .LBB21_4
 ; RV32IZFINX-NEXT:  # %bb.3:
-; RV32IZFINX-NEXT:    mv a3, a1
+; RV32IZFINX-NEXT:    mv a4, a1
 ; RV32IZFINX-NEXT:  .LBB21_4:
 ; RV32IZFINX-NEXT:    and a0, a2, a0
-; RV32IZFINX-NEXT:    beqz a4, .LBB21_6
+; RV32IZFINX-NEXT:    beqz a3, .LBB21_6
 ; RV32IZFINX-NEXT:  # %bb.5:
-; RV32IZFINX-NEXT:    addi a3, a5, -1
+; RV32IZFINX-NEXT:    addi a4, a5, -1
 ; RV32IZFINX-NEXT:  .LBB21_6:
-; RV32IZFINX-NEXT:    and a1, a2, a3
+; RV32IZFINX-NEXT:    and a1, a2, a4
 ; RV32IZFINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZFINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZFINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/half-convert.ll b/llvm/test/CodeGen/RISCV/half-convert.ll
index bc63b3961952..16c096290720 100644
--- a/llvm/test/CodeGen/RISCV/half-convert.ll
+++ b/llvm/test/CodeGen/RISCV/half-convert.ll
@@ -2460,47 +2460,42 @@ define i64 @fcvt_l_h_sat(half %a) nounwind {
 ;
 ; RV32ID-ILP32-LABEL: fcvt_l_h_sat:
 ; RV32ID-ILP32:       # %bb.0: # %start
-; RV32ID-ILP32-NEXT:    addi sp, sp, -32
-; RV32ID-ILP32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32ID-ILP32-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32ID-ILP32-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
-; RV32ID-ILP32-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
-; RV32ID-ILP32-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32ID-ILP32-NEXT:    addi sp, sp, -16
+; RV32ID-ILP32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32ID-ILP32-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32ID-ILP32-NEXT:    call __extendhfsf2
-; RV32ID-ILP32-NEXT:    lui a1, %hi(.LCPI10_0)
-; RV32ID-ILP32-NEXT:    flw fa5, %lo(.LCPI10_0)(a1)
 ; RV32ID-ILP32-NEXT:    fmv.w.x fa4, a0
-; RV32ID-ILP32-NEXT:    fsw fa4, 8(sp) # 4-byte Folded Spill
-; RV32ID-ILP32-NEXT:    flt.s s0, fa5, fa4
-; RV32ID-ILP32-NEXT:    neg s1, s0
 ; RV32ID-ILP32-NEXT:    lui a1, 913408
 ; RV32ID-ILP32-NEXT:    fmv.w.x fa5, a1
-; RV32ID-ILP32-NEXT:    fle.s s2, fa5, fa4
-; RV32ID-ILP32-NEXT:    neg s3, s2
+; RV32ID-ILP32-NEXT:    fsw fa4, 4(sp) # 4-byte Folded Spill
+; RV32ID-ILP32-NEXT:    fle.s s0, fa5, fa4
 ; RV32ID-ILP32-NEXT:    call __fixsfdi
-; RV32ID-ILP32-NEXT:    and a0, s3, a0
-; RV32ID-ILP32-NEXT:    or a0, s1, a0
-; RV32ID-ILP32-NEXT:    flw fa5, 8(sp) # 4-byte Folded Reload
-; RV32ID-ILP32-NEXT:    feq.s a2, fa5, fa5
-; RV32ID-ILP32-NEXT:    neg a2, a2
 ; RV32ID-ILP32-NEXT:    lui a4, 524288
-; RV32ID-ILP32-NEXT:    lui a3, 524288
-; RV32ID-ILP32-NEXT:    beqz s2, .LBB10_2
+; RV32ID-ILP32-NEXT:    lui a2, 524288
+; RV32ID-ILP32-NEXT:    beqz s0, .LBB10_2
 ; RV32ID-ILP32-NEXT:  # %bb.1: # %start
-; RV32ID-ILP32-NEXT:    mv a3, a1
+; RV32ID-ILP32-NEXT:    mv a2, a1
 ; RV32ID-ILP32-NEXT:  .LBB10_2: # %start
-; RV32ID-ILP32-NEXT:    and a0, a2, a0
-; RV32ID-ILP32-NEXT:    beqz s0, .LBB10_4
+; RV32ID-ILP32-NEXT:    lui a1, %hi(.LCPI10_0)
+; RV32ID-ILP32-NEXT:    flw fa5, %lo(.LCPI10_0)(a1)
+; RV32ID-ILP32-NEXT:    flw fa4, 4(sp) # 4-byte Folded Reload
+; RV32ID-ILP32-NEXT:    flt.s a3, fa5, fa4
+; RV32ID-ILP32-NEXT:    fmv.s fa5, fa4
+; RV32ID-ILP32-NEXT:    beqz a3, .LBB10_4
 ; RV32ID-ILP32-NEXT:  # %bb.3:
-; RV32ID-ILP32-NEXT:    addi a3, a4, -1
+; RV32ID-ILP32-NEXT:    addi a2, a4, -1
 ; RV32ID-ILP32-NEXT:  .LBB10_4: # %start
-; RV32ID-ILP32-NEXT:    and a1, a2, a3
-; RV32ID-ILP32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32ID-ILP32-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32ID-ILP32-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
-; RV32ID-ILP32-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
-; RV32ID-ILP32-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
-; RV32ID-ILP32-NEXT:    addi sp, sp, 32
+; RV32ID-ILP32-NEXT:    feq.s a1, fa5, fa5
+; RV32ID-ILP32-NEXT:    neg a4, a1
+; RV32ID-ILP32-NEXT:    and a1, a4, a2
+; RV32ID-ILP32-NEXT:    neg a2, a3
+; RV32ID-ILP32-NEXT:    neg a3, s0
+; RV32ID-ILP32-NEXT:    and a0, a3, a0
+; RV32ID-ILP32-NEXT:    or a0, a2, a0
+; RV32ID-ILP32-NEXT:    and a0, a4, a0
+; RV32ID-ILP32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32ID-ILP32-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32ID-ILP32-NEXT:    addi sp, sp, 16
 ; RV32ID-ILP32-NEXT:    ret
 ;
 ; RV64ID-LP64-LABEL: fcvt_l_h_sat:
diff --git a/llvm/test/CodeGen/RISCV/half-round-conv-sat.ll b/llvm/test/CodeGen/RISCV/half-round-conv-sat.ll
index dd1115b20225..9c95210bfa7c 100644
--- a/llvm/test/CodeGen/RISCV/half-round-conv-sat.ll
+++ b/llvm/test/CodeGen/RISCV/half-round-conv-sat.ll
@@ -120,16 +120,16 @@ define i64 @test_floor_si64(half %x) nounwind {
 ; RV32IZFH-NEXT:    lui a2, %hi(.LCPI1_1)
 ; RV32IZFH-NEXT:    flw fa5, %lo(.LCPI1_1)(a2)
 ; RV32IZFH-NEXT:    and a0, s1, a0
-; RV32IZFH-NEXT:    flt.s a4, fa5, fs0
-; RV32IZFH-NEXT:    neg a2, a4
+; RV32IZFH-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFH-NEXT:    neg a2, a3
 ; RV32IZFH-NEXT:    or a0, a2, a0
 ; RV32IZFH-NEXT:    feq.s a2, fs0, fs0
 ; RV32IZFH-NEXT:    neg a2, a2
 ; RV32IZFH-NEXT:    lui a5, 524288
-; RV32IZFH-NEXT:    lui a3, 524288
+; RV32IZFH-NEXT:    lui a4, 524288
 ; RV32IZFH-NEXT:    beqz s0, .LBB1_4
 ; RV32IZFH-NEXT:  # %bb.3:
-; RV32IZFH-NEXT:    mv a3, a1
+; RV32IZFH-NEXT:    mv a4, a1
 ; RV32IZFH-NEXT:  .LBB1_4:
 ; RV32IZFH-NEXT:    and a0, a2, a0
 ; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -137,11 +137,11 @@ define i64 @test_floor_si64(half %x) nounwind {
 ; RV32IZFH-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    flw fs0, 0(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    addi sp, sp, 16
-; RV32IZFH-NEXT:    beqz a4, .LBB1_6
+; RV32IZFH-NEXT:    beqz a3, .LBB1_6
 ; RV32IZFH-NEXT:  # %bb.5:
-; RV32IZFH-NEXT:    addi a3, a5, -1
+; RV32IZFH-NEXT:    addi a4, a5, -1
 ; RV32IZFH-NEXT:  .LBB1_6:
-; RV32IZFH-NEXT:    and a1, a2, a3
+; RV32IZFH-NEXT:    and a1, a2, a4
 ; RV32IZFH-NEXT:    ret
 ;
 ; RV64IZFH-LABEL: test_floor_si64:
@@ -179,16 +179,16 @@ define i64 @test_floor_si64(half %x) nounwind {
 ; RV32IZHINX-NEXT:    lui a2, %hi(.LCPI1_1)
 ; RV32IZHINX-NEXT:    lw a2, %lo(.LCPI1_1)(a2)
 ; RV32IZHINX-NEXT:    and a0, s2, a0
-; RV32IZHINX-NEXT:    flt.s a4, a2, s0
-; RV32IZHINX-NEXT:    neg a2, a4
+; RV32IZHINX-NEXT:    flt.s a3, a2, s0
+; RV32IZHINX-NEXT:    neg a2, a3
 ; RV32IZHINX-NEXT:    or a0, a2, a0
 ; RV32IZHINX-NEXT:    feq.s a2, s0, s0
 ; RV32IZHINX-NEXT:    neg a2, a2
 ; RV32IZHINX-NEXT:    lui a5, 524288
-; RV32IZHINX-NEXT:    lui a3, 524288
+; RV32IZHINX-NEXT:    lui a4, 524288
 ; RV32IZHINX-NEXT:    beqz s1, .LBB1_4
 ; RV32IZHINX-NEXT:  # %bb.3:
-; RV32IZHINX-NEXT:    mv a3, a1
+; RV32IZHINX-NEXT:    mv a4, a1
 ; RV32IZHINX-NEXT:  .LBB1_4:
 ; RV32IZHINX-NEXT:    and a0, a2, a0
 ; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -196,11 +196,11 @@ define i64 @test_floor_si64(half %x) nounwind {
 ; RV32IZHINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZHINX-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
 ; RV32IZHINX-NEXT:    addi sp, sp, 16
-; RV32IZHINX-NEXT:    beqz a4, .LBB1_6
+; RV32IZHINX-NEXT:    beqz a3, .LBB1_6
 ; RV32IZHINX-NEXT:  # %bb.5:
-; RV32IZHINX-NEXT:    addi a3, a5, -1
+; RV32IZHINX-NEXT:    addi a4, a5, -1
 ; RV32IZHINX-NEXT:  .LBB1_6:
-; RV32IZHINX-NEXT:    and a1, a2, a3
+; RV32IZHINX-NEXT:    and a1, a2, a4
 ; RV32IZHINX-NEXT:    ret
 ;
 ; RV64IZHINX-LABEL: test_floor_si64:
@@ -251,16 +251,16 @@ define i64 @test_floor_si64(half %x) nounwind {
 ; RV32IZFHMIN-NEXT:    lui a2, %hi(.LCPI1_0)
 ; RV32IZFHMIN-NEXT:    flw fa5, %lo(.LCPI1_0)(a2)
 ; RV32IZFHMIN-NEXT:    and a0, s1, a0
-; RV32IZFHMIN-NEXT:    flt.s a4, fa5, fs0
-; RV32IZFHMIN-NEXT:    neg a2, a4
+; RV32IZFHMIN-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFHMIN-NEXT:    neg a2, a3
 ; RV32IZFHMIN-NEXT:    or a0, a2, a0
 ; RV32IZFHMIN-NEXT:    feq.s a2, fs0, fs0
 ; RV32IZFHMIN-NEXT:    neg a2, a2
 ; RV32IZFHMIN-NEXT:    lui a5, 524288
-; RV32IZFHMIN-NEXT:    lui a3, 524288
+; RV32IZFHMIN-NEXT:    lui a4, 524288
 ; RV32IZFHMIN-NEXT:    beqz s0, .LBB1_4
 ; RV32IZFHMIN-NEXT:  # %bb.3:
-; RV32IZFHMIN-NEXT:    mv a3, a1
+; RV32IZFHMIN-NEXT:    mv a4, a1
 ; RV32IZFHMIN-NEXT:  .LBB1_4:
 ; RV32IZFHMIN-NEXT:    and a0, a2, a0
 ; RV32IZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -268,11 +268,11 @@ define i64 @test_floor_si64(half %x) nounwind {
 ; RV32IZFHMIN-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    flw fs0, 0(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    addi sp, sp, 16
-; RV32IZFHMIN-NEXT:    beqz a4, .LBB1_6
+; RV32IZFHMIN-NEXT:    beqz a3, .LBB1_6
 ; RV32IZFHMIN-NEXT:  # %bb.5:
-; RV32IZFHMIN-NEXT:    addi a3, a5, -1
+; RV32IZFHMIN-NEXT:    addi a4, a5, -1
 ; RV32IZFHMIN-NEXT:  .LBB1_6:
-; RV32IZFHMIN-NEXT:    and a1, a2, a3
+; RV32IZFHMIN-NEXT:    and a1, a2, a4
 ; RV32IZFHMIN-NEXT:    ret
 ;
 ; RV64IZFHMIN-LABEL: test_floor_si64:
@@ -324,16 +324,16 @@ define i64 @test_floor_si64(half %x) nounwind {
 ; RV32IZHINXMIN-NEXT:    lui a2, %hi(.LCPI1_0)
 ; RV32IZHINXMIN-NEXT:    lw a2, %lo(.LCPI1_0)(a2)
 ; RV32IZHINXMIN-NEXT:    and a0, s2, a0
-; RV32IZHINXMIN-NEXT:    flt.s a4, a2, s0
-; RV32IZHINXMIN-NEXT:    neg a2, a4
+; RV32IZHINXMIN-NEXT:    flt.s a3, a2, s0
+; RV32IZHINXMIN-NEXT:    neg a2, a3
 ; RV32IZHINXMIN-NEXT:    or a0, a2, a0
 ; RV32IZHINXMIN-NEXT:    feq.s a2, s0, s0
 ; RV32IZHINXMIN-NEXT:    neg a2, a2
 ; RV32IZHINXMIN-NEXT:    lui a5, 524288
-; RV32IZHINXMIN-NEXT:    lui a3, 524288
+; RV32IZHINXMIN-NEXT:    lui a4, 524288
 ; RV32IZHINXMIN-NEXT:    beqz s1, .LBB1_4
 ; RV32IZHINXMIN-NEXT:  # %bb.3:
-; RV32IZHINXMIN-NEXT:    mv a3, a1
+; RV32IZHINXMIN-NEXT:    mv a4, a1
 ; RV32IZHINXMIN-NEXT:  .LBB1_4:
 ; RV32IZHINXMIN-NEXT:    and a0, a2, a0
 ; RV32IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -341,11 +341,11 @@ define i64 @test_floor_si64(half %x) nounwind {
 ; RV32IZHINXMIN-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZHINXMIN-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
 ; RV32IZHINXMIN-NEXT:    addi sp, sp, 16
-; RV32IZHINXMIN-NEXT:    beqz a4, .LBB1_6
+; RV32IZHINXMIN-NEXT:    beqz a3, .LBB1_6
 ; RV32IZHINXMIN-NEXT:  # %bb.5:
-; RV32IZHINXMIN-NEXT:    addi a3, a5, -1
+; RV32IZHINXMIN-NEXT:    addi a4, a5, -1
 ; RV32IZHINXMIN-NEXT:  .LBB1_6:
-; RV32IZHINXMIN-NEXT:    and a1, a2, a3
+; RV32IZHINXMIN-NEXT:    and a1, a2, a4
 ; RV32IZHINXMIN-NEXT:    ret
 ;
 ; RV64IZHINXMIN-LABEL: test_floor_si64:
@@ -836,16 +836,16 @@ define i64 @test_ceil_si64(half %x) nounwind {
 ; RV32IZFH-NEXT:    lui a2, %hi(.LCPI5_1)
 ; RV32IZFH-NEXT:    flw fa5, %lo(.LCPI5_1)(a2)
 ; RV32IZFH-NEXT:    and a0, s1, a0
-; RV32IZFH-NEXT:    flt.s a4, fa5, fs0
-; RV32IZFH-NEXT:    neg a2, a4
+; RV32IZFH-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFH-NEXT:    neg a2, a3
 ; RV32IZFH-NEXT:    or a0, a2, a0
 ; RV32IZFH-NEXT:    feq.s a2, fs0, fs0
 ; RV32IZFH-NEXT:    neg a2, a2
 ; RV32IZFH-NEXT:    lui a5, 524288
-; RV32IZFH-NEXT:    lui a3, 524288
+; RV32IZFH-NEXT:    lui a4, 524288
 ; RV32IZFH-NEXT:    beqz s0, .LBB5_4
 ; RV32IZFH-NEXT:  # %bb.3:
-; RV32IZFH-NEXT:    mv a3, a1
+; RV32IZFH-NEXT:    mv a4, a1
 ; RV32IZFH-NEXT:  .LBB5_4:
 ; RV32IZFH-NEXT:    and a0, a2, a0
 ; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -853,11 +853,11 @@ define i64 @test_ceil_si64(half %x) nounwind {
 ; RV32IZFH-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    flw fs0, 0(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    addi sp, sp, 16
-; RV32IZFH-NEXT:    beqz a4, .LBB5_6
+; RV32IZFH-NEXT:    beqz a3, .LBB5_6
 ; RV32IZFH-NEXT:  # %bb.5:
-; RV32IZFH-NEXT:    addi a3, a5, -1
+; RV32IZFH-NEXT:    addi a4, a5, -1
 ; RV32IZFH-NEXT:  .LBB5_6:
-; RV32IZFH-NEXT:    and a1, a2, a3
+; RV32IZFH-NEXT:    and a1, a2, a4
 ; RV32IZFH-NEXT:    ret
 ;
 ; RV64IZFH-LABEL: test_ceil_si64:
@@ -895,16 +895,16 @@ define i64 @test_ceil_si64(half %x) nounwind {
 ; RV32IZHINX-NEXT:    lui a2, %hi(.LCPI5_1)
 ; RV32IZHINX-NEXT:    lw a2, %lo(.LCPI5_1)(a2)
 ; RV32IZHINX-NEXT:    and a0, s2, a0
-; RV32IZHINX-NEXT:    flt.s a4, a2, s0
-; RV32IZHINX-NEXT:    neg a2, a4
+; RV32IZHINX-NEXT:    flt.s a3, a2, s0
+; RV32IZHINX-NEXT:    neg a2, a3
 ; RV32IZHINX-NEXT:    or a0, a2, a0
 ; RV32IZHINX-NEXT:    feq.s a2, s0, s0
 ; RV32IZHINX-NEXT:    neg a2, a2
 ; RV32IZHINX-NEXT:    lui a5, 524288
-; RV32IZHINX-NEXT:    lui a3, 524288
+; RV32IZHINX-NEXT:    lui a4, 524288
 ; RV32IZHINX-NEXT:    beqz s1, .LBB5_4
 ; RV32IZHINX-NEXT:  # %bb.3:
-; RV32IZHINX-NEXT:    mv a3, a1
+; RV32IZHINX-NEXT:    mv a4, a1
 ; RV32IZHINX-NEXT:  .LBB5_4:
 ; RV32IZHINX-NEXT:    and a0, a2, a0
 ; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -912,11 +912,11 @@ define i64 @test_ceil_si64(half %x) nounwind {
 ; RV32IZHINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZHINX-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
 ; RV32IZHINX-NEXT:    addi sp, sp, 16
-; RV32IZHINX-NEXT:    beqz a4, .LBB5_6
+; RV32IZHINX-NEXT:    beqz a3, .LBB5_6
 ; RV32IZHINX-NEXT:  # %bb.5:
-; RV32IZHINX-NEXT:    addi a3, a5, -1
+; RV32IZHINX-NEXT:    addi a4, a5, -1
 ; RV32IZHINX-NEXT:  .LBB5_6:
-; RV32IZHINX-NEXT:    and a1, a2, a3
+; RV32IZHINX-NEXT:    and a1, a2, a4
 ; RV32IZHINX-NEXT:    ret
 ;
 ; RV64IZHINX-LABEL: test_ceil_si64:
@@ -967,16 +967,16 @@ define i64 @test_ceil_si64(half %x) nounwind {
 ; RV32IZFHMIN-NEXT:    lui a2, %hi(.LCPI5_0)
 ; RV32IZFHMIN-NEXT:    flw fa5, %lo(.LCPI5_0)(a2)
 ; RV32IZFHMIN-NEXT:    and a0, s1, a0
-; RV32IZFHMIN-NEXT:    flt.s a4, fa5, fs0
-; RV32IZFHMIN-NEXT:    neg a2, a4
+; RV32IZFHMIN-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFHMIN-NEXT:    neg a2, a3
 ; RV32IZFHMIN-NEXT:    or a0, a2, a0
 ; RV32IZFHMIN-NEXT:    feq.s a2, fs0, fs0
 ; RV32IZFHMIN-NEXT:    neg a2, a2
 ; RV32IZFHMIN-NEXT:    lui a5, 524288
-; RV32IZFHMIN-NEXT:    lui a3, 524288
+; RV32IZFHMIN-NEXT:    lui a4, 524288
 ; RV32IZFHMIN-NEXT:    beqz s0, .LBB5_4
 ; RV32IZFHMIN-NEXT:  # %bb.3:
-; RV32IZFHMIN-NEXT:    mv a3, a1
+; RV32IZFHMIN-NEXT:    mv a4, a1
 ; RV32IZFHMIN-NEXT:  .LBB5_4:
 ; RV32IZFHMIN-NEXT:    and a0, a2, a0
 ; RV32IZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -984,11 +984,11 @@ define i64 @test_ceil_si64(half %x) nounwind {
 ; RV32IZFHMIN-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    flw fs0, 0(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    addi sp, sp, 16
-; RV32IZFHMIN-NEXT:    beqz a4, .LBB5_6
+; RV32IZFHMIN-NEXT:    beqz a3, .LBB5_6
 ; RV32IZFHMIN-NEXT:  # %bb.5:
-; RV32IZFHMIN-NEXT:    addi a3, a5, -1
+; RV32IZFHMIN-NEXT:    addi a4, a5, -1
 ; RV32IZFHMIN-NEXT:  .LBB5_6:
-; RV32IZFHMIN-NEXT:    and a1, a2, a3
+; RV32IZFHMIN-NEXT:    and a1, a2, a4
 ; RV32IZFHMIN-NEXT:    ret
 ;
 ; RV64IZFHMIN-LABEL: test_ceil_si64:
@@ -1040,16 +1040,16 @@ define i64 @test_ceil_si64(half %x) nounwind {
 ; RV32IZHINXMIN-NEXT:    lui a2, %hi(.LCPI5_0)
 ; RV32IZHINXMIN-NEXT:    lw a2, %lo(.LCPI5_0)(a2)
 ; RV32IZHINXMIN-NEXT:    and a0, s2, a0
-; RV32IZHINXMIN-NEXT:    flt.s a4, a2, s0
-; RV32IZHINXMIN-NEXT:    neg a2, a4
+; RV32IZHINXMIN-NEXT:    flt.s a3, a2, s0
+; RV32IZHINXMIN-NEXT:    neg a2, a3
 ; RV32IZHINXMIN-NEXT:    or a0, a2, a0
 ; RV32IZHINXMIN-NEXT:    feq.s a2, s0, s0
 ; RV32IZHINXMIN-NEXT:    neg a2, a2
 ; RV32IZHINXMIN-NEXT:    lui a5, 524288
-; RV32IZHINXMIN-NEXT:    lui a3, 524288
+; RV32IZHINXMIN-NEXT:    lui a4, 524288
 ; RV32IZHINXMIN-NEXT:    beqz s1, .LBB5_4
 ; RV32IZHINXMIN-NEXT:  # %bb.3:
-; RV32IZHINXMIN-NEXT:    mv a3, a1
+; RV32IZHINXMIN-NEXT:    mv a4, a1
 ; RV32IZHINXMIN-NEXT:  .LBB5_4:
 ; RV32IZHINXMIN-NEXT:    and a0, a2, a0
 ; RV32IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -1057,11 +1057,11 @@ define i64 @test_ceil_si64(half %x) nounwind {
 ; RV32IZHINXMIN-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZHINXMIN-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
 ; RV32IZHINXMIN-NEXT:    addi sp, sp, 16
-; RV32IZHINXMIN-NEXT:    beqz a4, .LBB5_6
+; RV32IZHINXMIN-NEXT:    beqz a3, .LBB5_6
 ; RV32IZHINXMIN-NEXT:  # %bb.5:
-; RV32IZHINXMIN-NEXT:    addi a3, a5, -1
+; RV32IZHINXMIN-NEXT:    addi a4, a5, -1
 ; RV32IZHINXMIN-NEXT:  .LBB5_6:
-; RV32IZHINXMIN-NEXT:    and a1, a2, a3
+; RV32IZHINXMIN-NEXT:    and a1, a2, a4
 ; RV32IZHINXMIN-NEXT:    ret
 ;
 ; RV64IZHINXMIN-LABEL: test_ceil_si64:
@@ -1552,16 +1552,16 @@ define i64 @test_trunc_si64(half %x) nounwind {
 ; RV32IZFH-NEXT:    lui a2, %hi(.LCPI9_1)
 ; RV32IZFH-NEXT:    flw fa5, %lo(.LCPI9_1)(a2)
 ; RV32IZFH-NEXT:    and a0, s1, a0
-; RV32IZFH-NEXT:    flt.s a4, fa5, fs0
-; RV32IZFH-NEXT:    neg a2, a4
+; RV32IZFH-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFH-NEXT:    neg a2, a3
 ; RV32IZFH-NEXT:    or a0, a2, a0
 ; RV32IZFH-NEXT:    feq.s a2, fs0, fs0
 ; RV32IZFH-NEXT:    neg a2, a2
 ; RV32IZFH-NEXT:    lui a5, 524288
-; RV32IZFH-NEXT:    lui a3, 524288
+; RV32IZFH-NEXT:    lui a4, 524288
 ; RV32IZFH-NEXT:    beqz s0, .LBB9_4
 ; RV32IZFH-NEXT:  # %bb.3:
-; RV32IZFH-NEXT:    mv a3, a1
+; RV32IZFH-NEXT:    mv a4, a1
 ; RV32IZFH-NEXT:  .LBB9_4:
 ; RV32IZFH-NEXT:    and a0, a2, a0
 ; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -1569,11 +1569,11 @@ define i64 @test_trunc_si64(half %x) nounwind {
 ; RV32IZFH-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    flw fs0, 0(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    addi sp, sp, 16
-; RV32IZFH-NEXT:    beqz a4, .LBB9_6
+; RV32IZFH-NEXT:    beqz a3, .LBB9_6
 ; RV32IZFH-NEXT:  # %bb.5:
-; RV32IZFH-NEXT:    addi a3, a5, -1
+; RV32IZFH-NEXT:    addi a4, a5, -1
 ; RV32IZFH-NEXT:  .LBB9_6:
-; RV32IZFH-NEXT:    and a1, a2, a3
+; RV32IZFH-NEXT:    and a1, a2, a4
 ; RV32IZFH-NEXT:    ret
 ;
 ; RV64IZFH-LABEL: test_trunc_si64:
@@ -1611,16 +1611,16 @@ define i64 @test_trunc_si64(half %x) nounwind {
 ; RV32IZHINX-NEXT:    lui a2, %hi(.LCPI9_1)
 ; RV32IZHINX-NEXT:    lw a2, %lo(.LCPI9_1)(a2)
 ; RV32IZHINX-NEXT:    and a0, s2, a0
-; RV32IZHINX-NEXT:    flt.s a4, a2, s0
-; RV32IZHINX-NEXT:    neg a2, a4
+; RV32IZHINX-NEXT:    flt.s a3, a2, s0
+; RV32IZHINX-NEXT:    neg a2, a3
 ; RV32IZHINX-NEXT:    or a0, a2, a0
 ; RV32IZHINX-NEXT:    feq.s a2, s0, s0
 ; RV32IZHINX-NEXT:    neg a2, a2
 ; RV32IZHINX-NEXT:    lui a5, 524288
-; RV32IZHINX-NEXT:    lui a3, 524288
+; RV32IZHINX-NEXT:    lui a4, 524288
 ; RV32IZHINX-NEXT:    beqz s1, .LBB9_4
 ; RV32IZHINX-NEXT:  # %bb.3:
-; RV32IZHINX-NEXT:    mv a3, a1
+; RV32IZHINX-NEXT:    mv a4, a1
 ; RV32IZHINX-NEXT:  .LBB9_4:
 ; RV32IZHINX-NEXT:    and a0, a2, a0
 ; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -1628,11 +1628,11 @@ define i64 @test_trunc_si64(half %x) nounwind {
 ; RV32IZHINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZHINX-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
 ; RV32IZHINX-NEXT:    addi sp, sp, 16
-; RV32IZHINX-NEXT:    beqz a4, .LBB9_6
+; RV32IZHINX-NEXT:    beqz a3, .LBB9_6
 ; RV32IZHINX-NEXT:  # %bb.5:
-; RV32IZHINX-NEXT:    addi a3, a5, -1
+; RV32IZHINX-NEXT:    addi a4, a5, -1
 ; RV32IZHINX-NEXT:  .LBB9_6:
-; RV32IZHINX-NEXT:    and a1, a2, a3
+; RV32IZHINX-NEXT:    and a1, a2, a4
 ; RV32IZHINX-NEXT:    ret
 ;
 ; RV64IZHINX-LABEL: test_trunc_si64:
@@ -1683,16 +1683,16 @@ define i64 @test_trunc_si64(half %x) nounwind {
 ; RV32IZFHMIN-NEXT:    lui a2, %hi(.LCPI9_0)
 ; RV32IZFHMIN-NEXT:    flw fa5, %lo(.LCPI9_0)(a2)
 ; RV32IZFHMIN-NEXT:    and a0, s1, a0
-; RV32IZFHMIN-NEXT:    flt.s a4, fa5, fs0
-; RV32IZFHMIN-NEXT:    neg a2, a4
+; RV32IZFHMIN-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFHMIN-NEXT:    neg a2, a3
 ; RV32IZFHMIN-NEXT:    or a0, a2, a0
 ; RV32IZFHMIN-NEXT:    feq.s a2, fs0, fs0
 ; RV32IZFHMIN-NEXT:    neg a2, a2
 ; RV32IZFHMIN-NEXT:    lui a5, 524288
-; RV32IZFHMIN-NEXT:    lui a3, 524288
+; RV32IZFHMIN-NEXT:    lui a4, 524288
 ; RV32IZFHMIN-NEXT:    beqz s0, .LBB9_4
 ; RV32IZFHMIN-NEXT:  # %bb.3:
-; RV32IZFHMIN-NEXT:    mv a3, a1
+; RV32IZFHMIN-NEXT:    mv a4, a1
 ; RV32IZFHMIN-NEXT:  .LBB9_4:
 ; RV32IZFHMIN-NEXT:    and a0, a2, a0
 ; RV32IZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -1700,11 +1700,11 @@ define i64 @test_trunc_si64(half %x) nounwind {
 ; RV32IZFHMIN-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    flw fs0, 0(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    addi sp, sp, 16
-; RV32IZFHMIN-NEXT:    beqz a4, .LBB9_6
+; RV32IZFHMIN-NEXT:    beqz a3, .LBB9_6
 ; RV32IZFHMIN-NEXT:  # %bb.5:
-; RV32IZFHMIN-NEXT:    addi a3, a5, -1
+; RV32IZFHMIN-NEXT:    addi a4, a5, -1
 ; RV32IZFHMIN-NEXT:  .LBB9_6:
-; RV32IZFHMIN-NEXT:    and a1, a2, a3
+; RV32IZFHMIN-NEXT:    and a1, a2, a4
 ; RV32IZFHMIN-NEXT:    ret
 ;
 ; RV64IZFHMIN-LABEL: test_trunc_si64:
@@ -1756,16 +1756,16 @@ define i64 @test_trunc_si64(half %x) nounwind {
 ; RV32IZHINXMIN-NEXT:    lui a2, %hi(.LCPI9_0)
 ; RV32IZHINXMIN-NEXT:    lw a2, %lo(.LCPI9_0)(a2)
 ; RV32IZHINXMIN-NEXT:    and a0, s2, a0
-; RV32IZHINXMIN-NEXT:    flt.s a4, a2, s0
-; RV32IZHINXMIN-NEXT:    neg a2, a4
+; RV32IZHINXMIN-NEXT:    flt.s a3, a2, s0
+; RV32IZHINXMIN-NEXT:    neg a2, a3
 ; RV32IZHINXMIN-NEXT:    or a0, a2, a0
 ; RV32IZHINXMIN-NEXT:    feq.s a2, s0, s0
 ; RV32IZHINXMIN-NEXT:    neg a2, a2
 ; RV32IZHINXMIN-NEXT:    lui a5, 524288
-; RV32IZHINXMIN-NEXT:    lui a3, 524288
+; RV32IZHINXMIN-NEXT:    lui a4, 524288
 ; RV32IZHINXMIN-NEXT:    beqz s1, .LBB9_4
 ; RV32IZHINXMIN-NEXT:  # %bb.3:
-; RV32IZHINXMIN-NEXT:    mv a3, a1
+; RV32IZHINXMIN-NEXT:    mv a4, a1
 ; RV32IZHINXMIN-NEXT:  .LBB9_4:
 ; RV32IZHINXMIN-NEXT:    and a0, a2, a0
 ; RV32IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -1773,11 +1773,11 @@ define i64 @test_trunc_si64(half %x) nounwind {
 ; RV32IZHINXMIN-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZHINXMIN-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
 ; RV32IZHINXMIN-NEXT:    addi sp, sp, 16
-; RV32IZHINXMIN-NEXT:    beqz a4, .LBB9_6
+; RV32IZHINXMIN-NEXT:    beqz a3, .LBB9_6
 ; RV32IZHINXMIN-NEXT:  # %bb.5:
-; RV32IZHINXMIN-NEXT:    addi a3, a5, -1
+; RV32IZHINXMIN-NEXT:    addi a4, a5, -1
 ; RV32IZHINXMIN-NEXT:  .LBB9_6:
-; RV32IZHINXMIN-NEXT:    and a1, a2, a3
+; RV32IZHINXMIN-NEXT:    and a1, a2, a4
 ; RV32IZHINXMIN-NEXT:    ret
 ;
 ; RV64IZHINXMIN-LABEL: test_trunc_si64:
@@ -2268,16 +2268,16 @@ define i64 @test_round_si64(half %x) nounwind {
 ; RV32IZFH-NEXT:    lui a2, %hi(.LCPI13_1)
 ; RV32IZFH-NEXT:    flw fa5, %lo(.LCPI13_1)(a2)
 ; RV32IZFH-NEXT:    and a0, s1, a0
-; RV32IZFH-NEXT:    flt.s a4, fa5, fs0
-; RV32IZFH-NEXT:    neg a2, a4
+; RV32IZFH-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFH-NEXT:    neg a2, a3
 ; RV32IZFH-NEXT:    or a0, a2, a0
 ; RV32IZFH-NEXT:    feq.s a2, fs0, fs0
 ; RV32IZFH-NEXT:    neg a2, a2
 ; RV32IZFH-NEXT:    lui a5, 524288
-; RV32IZFH-NEXT:    lui a3, 524288
+; RV32IZFH-NEXT:    lui a4, 524288
 ; RV32IZFH-NEXT:    beqz s0, .LBB13_4
 ; RV32IZFH-NEXT:  # %bb.3:
-; RV32IZFH-NEXT:    mv a3, a1
+; RV32IZFH-NEXT:    mv a4, a1
 ; RV32IZFH-NEXT:  .LBB13_4:
 ; RV32IZFH-NEXT:    and a0, a2, a0
 ; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -2285,11 +2285,11 @@ define i64 @test_round_si64(half %x) nounwind {
 ; RV32IZFH-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    flw fs0, 0(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    addi sp, sp, 16
-; RV32IZFH-NEXT:    beqz a4, .LBB13_6
+; RV32IZFH-NEXT:    beqz a3, .LBB13_6
 ; RV32IZFH-NEXT:  # %bb.5:
-; RV32IZFH-NEXT:    addi a3, a5, -1
+; RV32IZFH-NEXT:    addi a4, a5, -1
 ; RV32IZFH-NEXT:  .LBB13_6:
-; RV32IZFH-NEXT:    and a1, a2, a3
+; RV32IZFH-NEXT:    and a1, a2, a4
 ; RV32IZFH-NEXT:    ret
 ;
 ; RV64IZFH-LABEL: test_round_si64:
@@ -2327,16 +2327,16 @@ define i64 @test_round_si64(half %x) nounwind {
 ; RV32IZHINX-NEXT:    lui a2, %hi(.LCPI13_1)
 ; RV32IZHINX-NEXT:    lw a2, %lo(.LCPI13_1)(a2)
 ; RV32IZHINX-NEXT:    and a0, s2, a0
-; RV32IZHINX-NEXT:    flt.s a4, a2, s0
-; RV32IZHINX-NEXT:    neg a2, a4
+; RV32IZHINX-NEXT:    flt.s a3, a2, s0
+; RV32IZHINX-NEXT:    neg a2, a3
 ; RV32IZHINX-NEXT:    or a0, a2, a0
 ; RV32IZHINX-NEXT:    feq.s a2, s0, s0
 ; RV32IZHINX-NEXT:    neg a2, a2
 ; RV32IZHINX-NEXT:    lui a5, 524288
-; RV32IZHINX-NEXT:    lui a3, 524288
+; RV32IZHINX-NEXT:    lui a4, 524288
 ; RV32IZHINX-NEXT:    beqz s1, .LBB13_4
 ; RV32IZHINX-NEXT:  # %bb.3:
-; RV32IZHINX-NEXT:    mv a3, a1
+; RV32IZHINX-NEXT:    mv a4, a1
 ; RV32IZHINX-NEXT:  .LBB13_4:
 ; RV32IZHINX-NEXT:    and a0, a2, a0
 ; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -2344,11 +2344,11 @@ define i64 @test_round_si64(half %x) nounwind {
 ; RV32IZHINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZHINX-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
 ; RV32IZHINX-NEXT:    addi sp, sp, 16
-; RV32IZHINX-NEXT:    beqz a4, .LBB13_6
+; RV32IZHINX-NEXT:    beqz a3, .LBB13_6
 ; RV32IZHINX-NEXT:  # %bb.5:
-; RV32IZHINX-NEXT:    addi a3, a5, -1
+; RV32IZHINX-NEXT:    addi a4, a5, -1
 ; RV32IZHINX-NEXT:  .LBB13_6:
-; RV32IZHINX-NEXT:    and a1, a2, a3
+; RV32IZHINX-NEXT:    and a1, a2, a4
 ; RV32IZHINX-NEXT:    ret
 ;
 ; RV64IZHINX-LABEL: test_round_si64:
@@ -2399,16 +2399,16 @@ define i64 @test_round_si64(half %x) nounwind {
 ; RV32IZFHMIN-NEXT:    lui a2, %hi(.LCPI13_0)
 ; RV32IZFHMIN-NEXT:    flw fa5, %lo(.LCPI13_0)(a2)
 ; RV32IZFHMIN-NEXT:    and a0, s1, a0
-; RV32IZFHMIN-NEXT:    flt.s a4, fa5, fs0
-; RV32IZFHMIN-NEXT:    neg a2, a4
+; RV32IZFHMIN-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFHMIN-NEXT:    neg a2, a3
 ; RV32IZFHMIN-NEXT:    or a0, a2, a0
 ; RV32IZFHMIN-NEXT:    feq.s a2, fs0, fs0
 ; RV32IZFHMIN-NEXT:    neg a2, a2
 ; RV32IZFHMIN-NEXT:    lui a5, 524288
-; RV32IZFHMIN-NEXT:    lui a3, 524288
+; RV32IZFHMIN-NEXT:    lui a4, 524288
 ; RV32IZFHMIN-NEXT:    beqz s0, .LBB13_4
 ; RV32IZFHMIN-NEXT:  # %bb.3:
-; RV32IZFHMIN-NEXT:    mv a3, a1
+; RV32IZFHMIN-NEXT:    mv a4, a1
 ; RV32IZFHMIN-NEXT:  .LBB13_4:
 ; RV32IZFHMIN-NEXT:    and a0, a2, a0
 ; RV32IZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -2416,11 +2416,11 @@ define i64 @test_round_si64(half %x) nounwind {
 ; RV32IZFHMIN-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    flw fs0, 0(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    addi sp, sp, 16
-; RV32IZFHMIN-NEXT:    beqz a4, .LBB13_6
+; RV32IZFHMIN-NEXT:    beqz a3, .LBB13_6
 ; RV32IZFHMIN-NEXT:  # %bb.5:
-; RV32IZFHMIN-NEXT:    addi a3, a5, -1
+; RV32IZFHMIN-NEXT:    addi a4, a5, -1
 ; RV32IZFHMIN-NEXT:  .LBB13_6:
-; RV32IZFHMIN-NEXT:    and a1, a2, a3
+; RV32IZFHMIN-NEXT:    and a1, a2, a4
 ; RV32IZFHMIN-NEXT:    ret
 ;
 ; RV64IZFHMIN-LABEL: test_round_si64:
@@ -2472,16 +2472,16 @@ define i64 @test_round_si64(half %x) nounwind {
 ; RV32IZHINXMIN-NEXT:    lui a2, %hi(.LCPI13_0)
 ; RV32IZHINXMIN-NEXT:    lw a2, %lo(.LCPI13_0)(a2)
 ; RV32IZHINXMIN-NEXT:    and a0, s2, a0
-; RV32IZHINXMIN-NEXT:    flt.s a4, a2, s0
-; RV32IZHINXMIN-NEXT:    neg a2, a4
+; RV32IZHINXMIN-NEXT:    flt.s a3, a2, s0
+; RV32IZHINXMIN-NEXT:    neg a2, a3
 ; RV32IZHINXMIN-NEXT:    or a0, a2, a0
 ; RV32IZHINXMIN-NEXT:    feq.s a2, s0, s0
 ; RV32IZHINXMIN-NEXT:    neg a2, a2
 ; RV32IZHINXMIN-NEXT:    lui a5, 524288
-; RV32IZHINXMIN-NEXT:    lui a3, 524288
+; RV32IZHINXMIN-NEXT:    lui a4, 524288
 ; RV32IZHINXMIN-NEXT:    beqz s1, .LBB13_4
 ; RV32IZHINXMIN-NEXT:  # %bb.3:
-; RV32IZHINXMIN-NEXT:    mv a3, a1
+; RV32IZHINXMIN-NEXT:    mv a4, a1
 ; RV32IZHINXMIN-NEXT:  .LBB13_4:
 ; RV32IZHINXMIN-NEXT:    and a0, a2, a0
 ; RV32IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -2489,11 +2489,11 @@ define i64 @test_round_si64(half %x) nounwind {
 ; RV32IZHINXMIN-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZHINXMIN-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
 ; RV32IZHINXMIN-NEXT:    addi sp, sp, 16
-; RV32IZHINXMIN-NEXT:    beqz a4, .LBB13_6
+; RV32IZHINXMIN-NEXT:    beqz a3, .LBB13_6
 ; RV32IZHINXMIN-NEXT:  # %bb.5:
-; RV32IZHINXMIN-NEXT:    addi a3, a5, -1
+; RV32IZHINXMIN-NEXT:    addi a4, a5, -1
 ; RV32IZHINXMIN-NEXT:  .LBB13_6:
-; RV32IZHINXMIN-NEXT:    and a1, a2, a3
+; RV32IZHINXMIN-NEXT:    and a1, a2, a4
 ; RV32IZHINXMIN-NEXT:    ret
 ;
 ; RV64IZHINXMIN-LABEL: test_round_si64:
@@ -2984,16 +2984,16 @@ define i64 @test_roundeven_si64(half %x) nounwind {
 ; RV32IZFH-NEXT:    lui a2, %hi(.LCPI17_1)
 ; RV32IZFH-NEXT:    flw fa5, %lo(.LCPI17_1)(a2)
 ; RV32IZFH-NEXT:    and a0, s1, a0
-; RV32IZFH-NEXT:    flt.s a4, fa5, fs0
-; RV32IZFH-NEXT:    neg a2, a4
+; RV32IZFH-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFH-NEXT:    neg a2, a3
 ; RV32IZFH-NEXT:    or a0, a2, a0
 ; RV32IZFH-NEXT:    feq.s a2, fs0, fs0
 ; RV32IZFH-NEXT:    neg a2, a2
 ; RV32IZFH-NEXT:    lui a5, 524288
-; RV32IZFH-NEXT:    lui a3, 524288
+; RV32IZFH-NEXT:    lui a4, 524288
 ; RV32IZFH-NEXT:    beqz s0, .LBB17_4
 ; RV32IZFH-NEXT:  # %bb.3:
-; RV32IZFH-NEXT:    mv a3, a1
+; RV32IZFH-NEXT:    mv a4, a1
 ; RV32IZFH-NEXT:  .LBB17_4:
 ; RV32IZFH-NEXT:    and a0, a2, a0
 ; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -3001,11 +3001,11 @@ define i64 @test_roundeven_si64(half %x) nounwind {
 ; RV32IZFH-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    flw fs0, 0(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    addi sp, sp, 16
-; RV32IZFH-NEXT:    beqz a4, .LBB17_6
+; RV32IZFH-NEXT:    beqz a3, .LBB17_6
 ; RV32IZFH-NEXT:  # %bb.5:
-; RV32IZFH-NEXT:    addi a3, a5, -1
+; RV32IZFH-NEXT:    addi a4, a5, -1
 ; RV32IZFH-NEXT:  .LBB17_6:
-; RV32IZFH-NEXT:    and a1, a2, a3
+; RV32IZFH-NEXT:    and a1, a2, a4
 ; RV32IZFH-NEXT:    ret
 ;
 ; RV64IZFH-LABEL: test_roundeven_si64:
@@ -3043,16 +3043,16 @@ define i64 @test_roundeven_si64(half %x) nounwind {
 ; RV32IZHINX-NEXT:    lui a2, %hi(.LCPI17_1)
 ; RV32IZHINX-NEXT:    lw a2, %lo(.LCPI17_1)(a2)
 ; RV32IZHINX-NEXT:    and a0, s2, a0
-; RV32IZHINX-NEXT:    flt.s a4, a2, s0
-; RV32IZHINX-NEXT:    neg a2, a4
+; RV32IZHINX-NEXT:    flt.s a3, a2, s0
+; RV32IZHINX-NEXT:    neg a2, a3
 ; RV32IZHINX-NEXT:    or a0, a2, a0
 ; RV32IZHINX-NEXT:    feq.s a2, s0, s0
 ; RV32IZHINX-NEXT:    neg a2, a2
 ; RV32IZHINX-NEXT:    lui a5, 524288
-; RV32IZHINX-NEXT:    lui a3, 524288
+; RV32IZHINX-NEXT:    lui a4, 524288
 ; RV32IZHINX-NEXT:    beqz s1, .LBB17_4
 ; RV32IZHINX-NEXT:  # %bb.3:
-; RV32IZHINX-NEXT:    mv a3, a1
+; RV32IZHINX-NEXT:    mv a4, a1
 ; RV32IZHINX-NEXT:  .LBB17_4:
 ; RV32IZHINX-NEXT:    and a0, a2, a0
 ; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -3060,11 +3060,11 @@ define i64 @test_roundeven_si64(half %x) nounwind {
 ; RV32IZHINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZHINX-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
 ; RV32IZHINX-NEXT:    addi sp, sp, 16
-; RV32IZHINX-NEXT:    beqz a4, .LBB17_6
+; RV32IZHINX-NEXT:    beqz a3, .LBB17_6
 ; RV32IZHINX-NEXT:  # %bb.5:
-; RV32IZHINX-NEXT:    addi a3, a5, -1
+; RV32IZHINX-NEXT:    addi a4, a5, -1
 ; RV32IZHINX-NEXT:  .LBB17_6:
-; RV32IZHINX-NEXT:    and a1, a2, a3
+; RV32IZHINX-NEXT:    and a1, a2, a4
 ; RV32IZHINX-NEXT:    ret
 ;
 ; RV64IZHINX-LABEL: test_roundeven_si64:
@@ -3115,16 +3115,16 @@ define i64 @test_roundeven_si64(half %x) nounwind {
 ; RV32IZFHMIN-NEXT:    lui a2, %hi(.LCPI17_0)
 ; RV32IZFHMIN-NEXT:    flw fa5, %lo(.LCPI17_0)(a2)
 ; RV32IZFHMIN-NEXT:    and a0, s1, a0
-; RV32IZFHMIN-NEXT:    flt.s a4, fa5, fs0
-; RV32IZFHMIN-NEXT:    neg a2, a4
+; RV32IZFHMIN-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFHMIN-NEXT:    neg a2, a3
 ; RV32IZFHMIN-NEXT:    or a0, a2, a0
 ; RV32IZFHMIN-NEXT:    feq.s a2, fs0, fs0
 ; RV32IZFHMIN-NEXT:    neg a2, a2
 ; RV32IZFHMIN-NEXT:    lui a5, 524288
-; RV32IZFHMIN-NEXT:    lui a3, 524288
+; RV32IZFHMIN-NEXT:    lui a4, 524288
 ; RV32IZFHMIN-NEXT:    beqz s0, .LBB17_4
 ; RV32IZFHMIN-NEXT:  # %bb.3:
-; RV32IZFHMIN-NEXT:    mv a3, a1
+; RV32IZFHMIN-NEXT:    mv a4, a1
 ; RV32IZFHMIN-NEXT:  .LBB17_4:
 ; RV32IZFHMIN-NEXT:    and a0, a2, a0
 ; RV32IZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -3132,11 +3132,11 @@ define i64 @test_roundeven_si64(half %x) nounwind {
 ; RV32IZFHMIN-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    flw fs0, 0(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    addi sp, sp, 16
-; RV32IZFHMIN-NEXT:    beqz a4, .LBB17_6
+; RV32IZFHMIN-NEXT:    beqz a3, .LBB17_6
 ; RV32IZFHMIN-NEXT:  # %bb.5:
-; RV32IZFHMIN-NEXT:    addi a3, a5, -1
+; RV32IZFHMIN-NEXT:    addi a4, a5, -1
 ; RV32IZFHMIN-NEXT:  .LBB17_6:
-; RV32IZFHMIN-NEXT:    and a1, a2, a3
+; RV32IZFHMIN-NEXT:    and a1, a2, a4
 ; RV32IZFHMIN-NEXT:    ret
 ;
 ; RV64IZFHMIN-LABEL: test_roundeven_si64:
@@ -3188,16 +3188,16 @@ define i64 @test_roundeven_si64(half %x) nounwind {
 ; RV32IZHINXMIN-NEXT:    lui a2, %hi(.LCPI17_0)
 ; RV32IZHINXMIN-NEXT:    lw a2, %lo(.LCPI17_0)(a2)
 ; RV32IZHINXMIN-NEXT:    and a0, s2, a0
-; RV32IZHINXMIN-NEXT:    flt.s a4, a2, s0
-; RV32IZHINXMIN-NEXT:    neg a2, a4
+; RV32IZHINXMIN-NEXT:    flt.s a3, a2, s0
+; RV32IZHINXMIN-NEXT:    neg a2, a3
 ; RV32IZHINXMIN-NEXT:    or a0, a2, a0
 ; RV32IZHINXMIN-NEXT:    feq.s a2, s0, s0
 ; RV32IZHINXMIN-NEXT:    neg a2, a2
 ; RV32IZHINXMIN-NEXT:    lui a5, 524288
-; RV32IZHINXMIN-NEXT:    lui a3, 524288
+; RV32IZHINXMIN-NEXT:    lui a4, 524288
 ; RV32IZHINXMIN-NEXT:    beqz s1, .LBB17_4
 ; RV32IZHINXMIN-NEXT:  # %bb.3:
-; RV32IZHINXMIN-NEXT:    mv a3, a1
+; RV32IZHINXMIN-NEXT:    mv a4, a1
 ; RV32IZHINXMIN-NEXT:  .LBB17_4:
 ; RV32IZHINXMIN-NEXT:    and a0, a2, a0
 ; RV32IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -3205,11 +3205,11 @@ define i64 @test_roundeven_si64(half %x) nounwind {
 ; RV32IZHINXMIN-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZHINXMIN-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
 ; RV32IZHINXMIN-NEXT:    addi sp, sp, 16
-; RV32IZHINXMIN-NEXT:    beqz a4, .LBB17_6
+; RV32IZHINXMIN-NEXT:    beqz a3, .LBB17_6
 ; RV32IZHINXMIN-NEXT:  # %bb.5:
-; RV32IZHINXMIN-NEXT:    addi a3, a5, -1
+; RV32IZHINXMIN-NEXT:    addi a4, a5, -1
 ; RV32IZHINXMIN-NEXT:  .LBB17_6:
-; RV32IZHINXMIN-NEXT:    and a1, a2, a3
+; RV32IZHINXMIN-NEXT:    and a1, a2, a4
 ; RV32IZHINXMIN-NEXT:    ret
 ;
 ; RV64IZHINXMIN-LABEL: test_roundeven_si64:
@@ -3700,16 +3700,16 @@ define i64 @test_rint_si64(half %x) nounwind {
 ; RV32IZFH-NEXT:    lui a2, %hi(.LCPI21_1)
 ; RV32IZFH-NEXT:    flw fa5, %lo(.LCPI21_1)(a2)
 ; RV32IZFH-NEXT:    and a0, s1, a0
-; RV32IZFH-NEXT:    flt.s a4, fa5, fs0
-; RV32IZFH-NEXT:    neg a2, a4
+; RV32IZFH-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFH-NEXT:    neg a2, a3
 ; RV32IZFH-NEXT:    or a0, a2, a0
 ; RV32IZFH-NEXT:    feq.s a2, fs0, fs0
 ; RV32IZFH-NEXT:    neg a2, a2
 ; RV32IZFH-NEXT:    lui a5, 524288
-; RV32IZFH-NEXT:    lui a3, 524288
+; RV32IZFH-NEXT:    lui a4, 524288
 ; RV32IZFH-NEXT:    beqz s0, .LBB21_4
 ; RV32IZFH-NEXT:  # %bb.3:
-; RV32IZFH-NEXT:    mv a3, a1
+; RV32IZFH-NEXT:    mv a4, a1
 ; RV32IZFH-NEXT:  .LBB21_4:
 ; RV32IZFH-NEXT:    and a0, a2, a0
 ; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -3717,11 +3717,11 @@ define i64 @test_rint_si64(half %x) nounwind {
 ; RV32IZFH-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    flw fs0, 0(sp) # 4-byte Folded Reload
 ; RV32IZFH-NEXT:    addi sp, sp, 16
-; RV32IZFH-NEXT:    beqz a4, .LBB21_6
+; RV32IZFH-NEXT:    beqz a3, .LBB21_6
 ; RV32IZFH-NEXT:  # %bb.5:
-; RV32IZFH-NEXT:    addi a3, a5, -1
+; RV32IZFH-NEXT:    addi a4, a5, -1
 ; RV32IZFH-NEXT:  .LBB21_6:
-; RV32IZFH-NEXT:    and a1, a2, a3
+; RV32IZFH-NEXT:    and a1, a2, a4
 ; RV32IZFH-NEXT:    ret
 ;
 ; RV64IZFH-LABEL: test_rint_si64:
@@ -3759,16 +3759,16 @@ define i64 @test_rint_si64(half %x) nounwind {
 ; RV32IZHINX-NEXT:    lui a2, %hi(.LCPI21_1)
 ; RV32IZHINX-NEXT:    lw a2, %lo(.LCPI21_1)(a2)
 ; RV32IZHINX-NEXT:    and a0, s2, a0
-; RV32IZHINX-NEXT:    flt.s a4, a2, s0
-; RV32IZHINX-NEXT:    neg a2, a4
+; RV32IZHINX-NEXT:    flt.s a3, a2, s0
+; RV32IZHINX-NEXT:    neg a2, a3
 ; RV32IZHINX-NEXT:    or a0, a2, a0
 ; RV32IZHINX-NEXT:    feq.s a2, s0, s0
 ; RV32IZHINX-NEXT:    neg a2, a2
 ; RV32IZHINX-NEXT:    lui a5, 524288
-; RV32IZHINX-NEXT:    lui a3, 524288
+; RV32IZHINX-NEXT:    lui a4, 524288
 ; RV32IZHINX-NEXT:    beqz s1, .LBB21_4
 ; RV32IZHINX-NEXT:  # %bb.3:
-; RV32IZHINX-NEXT:    mv a3, a1
+; RV32IZHINX-NEXT:    mv a4, a1
 ; RV32IZHINX-NEXT:  .LBB21_4:
 ; RV32IZHINX-NEXT:    and a0, a2, a0
 ; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -3776,11 +3776,11 @@ define i64 @test_rint_si64(half %x) nounwind {
 ; RV32IZHINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZHINX-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
 ; RV32IZHINX-NEXT:    addi sp, sp, 16
-; RV32IZHINX-NEXT:    beqz a4, .LBB21_6
+; RV32IZHINX-NEXT:    beqz a3, .LBB21_6
 ; RV32IZHINX-NEXT:  # %bb.5:
-; RV32IZHINX-NEXT:    addi a3, a5, -1
+; RV32IZHINX-NEXT:    addi a4, a5, -1
 ; RV32IZHINX-NEXT:  .LBB21_6:
-; RV32IZHINX-NEXT:    and a1, a2, a3
+; RV32IZHINX-NEXT:    and a1, a2, a4
 ; RV32IZHINX-NEXT:    ret
 ;
 ; RV64IZHINX-LABEL: test_rint_si64:
@@ -3831,16 +3831,16 @@ define i64 @test_rint_si64(half %x) nounwind {
 ; RV32IZFHMIN-NEXT:    lui a2, %hi(.LCPI21_0)
 ; RV32IZFHMIN-NEXT:    flw fa5, %lo(.LCPI21_0)(a2)
 ; RV32IZFHMIN-NEXT:    and a0, s1, a0
-; RV32IZFHMIN-NEXT:    flt.s a4, fa5, fs0
-; RV32IZFHMIN-NEXT:    neg a2, a4
+; RV32IZFHMIN-NEXT:    flt.s a3, fa5, fs0
+; RV32IZFHMIN-NEXT:    neg a2, a3
 ; RV32IZFHMIN-NEXT:    or a0, a2, a0
 ; RV32IZFHMIN-NEXT:    feq.s a2, fs0, fs0
 ; RV32IZFHMIN-NEXT:    neg a2, a2
 ; RV32IZFHMIN-NEXT:    lui a5, 524288
-; RV32IZFHMIN-NEXT:    lui a3, 524288
+; RV32IZFHMIN-NEXT:    lui a4, 524288
 ; RV32IZFHMIN-NEXT:    beqz s0, .LBB21_4
 ; RV32IZFHMIN-NEXT:  # %bb.3:
-; RV32IZFHMIN-NEXT:    mv a3, a1
+; RV32IZFHMIN-NEXT:    mv a4, a1
 ; RV32IZFHMIN-NEXT:  .LBB21_4:
 ; RV32IZFHMIN-NEXT:    and a0, a2, a0
 ; RV32IZFHMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -3848,11 +3848,11 @@ define i64 @test_rint_si64(half %x) nounwind {
 ; RV32IZFHMIN-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    flw fs0, 0(sp) # 4-byte Folded Reload
 ; RV32IZFHMIN-NEXT:    addi sp, sp, 16
-; RV32IZFHMIN-NEXT:    beqz a4, .LBB21_6
+; RV32IZFHMIN-NEXT:    beqz a3, .LBB21_6
 ; RV32IZFHMIN-NEXT:  # %bb.5:
-; RV32IZFHMIN-NEXT:    addi a3, a5, -1
+; RV32IZFHMIN-NEXT:    addi a4, a5, -1
 ; RV32IZFHMIN-NEXT:  .LBB21_6:
-; RV32IZFHMIN-NEXT:    and a1, a2, a3
+; RV32IZFHMIN-NEXT:    and a1, a2, a4
 ; RV32IZFHMIN-NEXT:    ret
 ;
 ; RV64IZFHMIN-LABEL: test_rint_si64:
@@ -3904,16 +3904,16 @@ define i64 @test_rint_si64(half %x) nounwind {
 ; RV32IZHINXMIN-NEXT:    lui a2, %hi(.LCPI21_0)
 ; RV32IZHINXMIN-NEXT:    lw a2, %lo(.LCPI21_0)(a2)
 ; RV32IZHINXMIN-NEXT:    and a0, s2, a0
-; RV32IZHINXMIN-NEXT:    flt.s a4, a2, s0
-; RV32IZHINXMIN-NEXT:    neg a2, a4
+; RV32IZHINXMIN-NEXT:    flt.s a3, a2, s0
+; RV32IZHINXMIN-NEXT:    neg a2, a3
 ; RV32IZHINXMIN-NEXT:    or a0, a2, a0
 ; RV32IZHINXMIN-NEXT:    feq.s a2, s0, s0
 ; RV32IZHINXMIN-NEXT:    neg a2, a2
 ; RV32IZHINXMIN-NEXT:    lui a5, 524288
-; RV32IZHINXMIN-NEXT:    lui a3, 524288
+; RV32IZHINXMIN-NEXT:    lui a4, 524288
 ; RV32IZHINXMIN-NEXT:    beqz s1, .LBB21_4
 ; RV32IZHINXMIN-NEXT:  # %bb.3:
-; RV32IZHINXMIN-NEXT:    mv a3, a1
+; RV32IZHINXMIN-NEXT:    mv a4, a1
 ; RV32IZHINXMIN-NEXT:  .LBB21_4:
 ; RV32IZHINXMIN-NEXT:    and a0, a2, a0
 ; RV32IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -3921,11 +3921,11 @@ define i64 @test_rint_si64(half %x) nounwind {
 ; RV32IZHINXMIN-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
 ; RV32IZHINXMIN-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
 ; RV32IZHINXMIN-NEXT:    addi sp, sp, 16
-; RV32IZHINXMIN-NEXT:    beqz a4, .LBB21_6
+; RV32IZHINXMIN-NEXT:    beqz a3, .LBB21_6
 ; RV32IZHINXMIN-NEXT:  # %bb.5:
-; RV32IZHINXMIN-NEXT:    addi a3, a5, -1
+; RV32IZHINXMIN-NEXT:    addi a4, a5, -1
 ; RV32IZHINXMIN-NEXT:  .LBB21_6:
-; RV32IZHINXMIN-NEXT:    and a1, a2, a3
+; RV32IZHINXMIN-NEXT:    and a1, a2, a4
 ; RV32IZHINXMIN-NEXT:    ret
 ;
 ; RV64IZHINXMIN-LABEL: test_rint_si64:
-- 
GitLab


From edfa97a8fbd5d8a763c30b083771c679d6a51a31 Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Mon, 25 Mar 2024 14:17:59 -0400
Subject: [PATCH 167/404] [C99] Claim conformance to "new block scopes for
 selection and iteration statements"

I was unable to definitively locate a document number for this, but it
is potentially WG14 N759 based on the description in the C99 rationale
document.

We do implement the C89 and C99 scope rules for iteration and selection
statements, so claiming conformance here.
---
 clang/test/C/C99/block-scopes.c | 34 +++++++++++++++++++++++++++++++++
 clang/www/c_status.html         |  2 +-
 2 files changed, 35 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/C/C99/block-scopes.c

diff --git a/clang/test/C/C99/block-scopes.c b/clang/test/C/C99/block-scopes.c
new file mode 100644
index 000000000000..589047df3e52
--- /dev/null
+++ b/clang/test/C/C99/block-scopes.c
@@ -0,0 +1,34 @@
+// RUN: %clang_cc1 -std=c89 -verify %s
+// RUN: %clang_cc1 -std=c99 -verify %s
+// RUN: %clang_cc1 -std=c11 -verify %s
+// RUN: %clang_cc1 -std=c17 -verify %s
+// RUN: %clang_cc1 -std=c23 -verify %s
+
+// expected-no-diagnostics
+
+/* WG14 ???: yes
+ * new block scopes for selection and iteration statements
+ *
+ * This is referenced in the C99 front matter as new changes to C99, but it is
+ * not clear which document number introduced the changes. It's possible this
+ * is WG14 N759, based on discussion in the C99 rationale document that claims
+ * these changes were made in response to surprising issues with the lifetime
+ * of compound literals in compound statements vs non-compound statements.
+ */
+
+enum {a, b};
+void different(void) {
+  if (sizeof(enum {b, a}) != sizeof(int))
+    _Static_assert(a == 1, "");
+  /* In C89, the 'b' found here would have been from the enum declaration in
+   * the controlling expression of the selection statement, not from the global
+   * declaration. In C99 and later, that enumeration is scoped to the 'if'
+   * statement and the global declaration is what's found.
+   */
+  #if __STDC_VERSION__ >= 199901L
+    _Static_assert(b == 1, "");
+  #else
+    _Static_assert(b == 0, "");
+  #endif
+}
+
diff --git a/clang/www/c_status.html b/clang/www/c_status.html
index 435897277c49..60f48aba2788 100644
--- a/clang/www/c_status.html
+++ b/clang/www/c_status.html
@@ -260,7 +260,7 @@ conformance.</p>
     <tr>
       <td>new block scopes for selection and iteration statements</td>
       <td>Unknown</td>
-      <td class="unknown" align="center">Unknown</td>
+      <td class="full" align="center">Yes</td>
     </tr>
     <tr>
       <td>integer constant type rules</td>
-- 
GitLab


From 3dcf62b5eea950153be5f2ddb4b79b3c6362148b Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 25 Mar 2024 18:22:15 +0000
Subject: [PATCH 168/404] [X86] HSW/BDW - vector splat shifts don't use Port5
 when loading the shift amount

Noticed while trying to compare splat vs per-element shift perf stats for #39424

Confirmed with uops.info
---
 llvm/lib/Target/X86/X86SchedBroadwell.td      |  6 ++--
 llvm/lib/Target/X86/X86SchedHaswell.td        |  6 ++--
 .../llvm-mca/X86/Broadwell/resources-avx1.s   | 34 +++++++++----------
 .../llvm-mca/X86/Broadwell/resources-sse2.s   | 34 +++++++++----------
 .../llvm-mca/X86/Haswell/resources-avx1.s     | 34 +++++++++----------
 .../llvm-mca/X86/Haswell/resources-sse2.s     | 34 +++++++++----------
 6 files changed, 76 insertions(+), 72 deletions(-)

diff --git a/llvm/lib/Target/X86/X86SchedBroadwell.td b/llvm/lib/Target/X86/X86SchedBroadwell.td
index 61a8832000e2..0027de851df7 100644
--- a/llvm/lib/Target/X86/X86SchedBroadwell.td
+++ b/llvm/lib/Target/X86/X86SchedBroadwell.td
@@ -471,9 +471,11 @@ defm : X86WriteResPairUnsupported<WritePSADBWZ>;
 defm : BWWriteResPair<WritePHMINPOS, [BWPort0], 5>; // Vector PHMINPOS.
 
 // Vector integer shifts.
-defm : BWWriteResPair<WriteVecShift,     [BWPort0], 1, [1], 1, 5>;
-defm : BWWriteResPair<WriteVecShiftX,    [BWPort0,BWPort5],  2, [1,1], 2, 5>;
+defm : X86WriteRes<WriteVecShift,        [BWPort0], 1, [1], 1>;
+defm : X86WriteRes<WriteVecShiftX,       [BWPort0,BWPort5],  2, [1,1], 2>;
 defm : X86WriteRes<WriteVecShiftY,       [BWPort0,BWPort5],  4, [1,1], 2>;
+defm : X86WriteRes<WriteVecShiftLd,      [BWPort0,BWPort23], 6, [1,1], 2>;
+defm : X86WriteRes<WriteVecShiftXLd,     [BWPort0,BWPort23], 7, [1,1], 2>;
 defm : X86WriteRes<WriteVecShiftYLd,     [BWPort0,BWPort23], 7, [1,1], 2>;
 defm : X86WriteResPairUnsupported<WriteVecShiftZ>;
 
diff --git a/llvm/lib/Target/X86/X86SchedHaswell.td b/llvm/lib/Target/X86/X86SchedHaswell.td
index 8795ca95c559..a11b470b1f51 100644
--- a/llvm/lib/Target/X86/X86SchedHaswell.td
+++ b/llvm/lib/Target/X86/X86SchedHaswell.td
@@ -469,10 +469,12 @@ defm : HWWriteResPair<WritePSADBWZ, [HWPort0], 5, [1], 1, 7>; // Unsupported = 1
 defm : HWWriteResPair<WritePHMINPOS, [HWPort0],  5, [1], 1, 6>;
 
 // Vector integer shifts.
-defm : HWWriteResPair<WriteVecShift,     [HWPort0], 1, [1], 1, 5>;
-defm : HWWriteResPair<WriteVecShiftX,    [HWPort0,HWPort5],  2, [1,1], 2, 6>;
+defm : X86WriteRes<WriteVecShift,        [HWPort0], 1, [1], 1>;
+defm : X86WriteRes<WriteVecShiftX,       [HWPort0,HWPort5],  2, [1,1], 2>;
 defm : X86WriteRes<WriteVecShiftY,       [HWPort0,HWPort5],  4, [1,1], 2>;
 defm : X86WriteRes<WriteVecShiftZ,       [HWPort0,HWPort5],  4, [1,1], 2>; // Unsupported = 1
+defm : X86WriteRes<WriteVecShiftLd,      [HWPort0,HWPort23], 6, [1,1], 2>;
+defm : X86WriteRes<WriteVecShiftXLd,     [HWPort0,HWPort23], 8, [1,1], 2>;
 defm : X86WriteRes<WriteVecShiftYLd,     [HWPort0,HWPort23], 8, [1,1], 2>;
 defm : X86WriteRes<WriteVecShiftZLd,     [HWPort0,HWPort23], 8, [1,1], 2>; // Unsupported = 1
 
diff --git a/llvm/test/tools/llvm-mca/X86/Broadwell/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/Broadwell/resources-avx1.s
index c33cc79bd6c1..98b8619f2e04 100644
--- a/llvm/test/tools/llvm-mca/X86/Broadwell/resources-avx1.s
+++ b/llvm/test/tools/llvm-mca/X86/Broadwell/resources-avx1.s
@@ -1564,30 +1564,30 @@ vzeroupper
 # CHECK-NEXT:  2      6     0.50    *                   vpsignw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpslld	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      2     1.00                        vpslld	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  3      7     1.00    *                   vpslld	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   vpslld	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpslldq	$1, %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpsllq	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      2     1.00                        vpsllq	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  3      7     1.00    *                   vpsllq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   vpsllq	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpsllw	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      2     1.00                        vpsllw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  3      7     1.00    *                   vpsllw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   vpsllw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpsrad	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      2     1.00                        vpsrad	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  3      7     1.00    *                   vpsrad	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   vpsrad	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpsraw	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      2     1.00                        vpsraw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  3      7     1.00    *                   vpsraw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   vpsraw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpsrld	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      2     1.00                        vpsrld	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  3      7     1.00    *                   vpsrld	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   vpsrld	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpsrldq	$1, %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpsrlq	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      2     1.00                        vpsrlq	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  3      7     1.00    *                   vpsrlq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   vpsrlq	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpsrlw	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      2     1.00                        vpsrlw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  3      7     1.00    *                   vpsrlw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   vpsrlw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpsubb	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      6     0.50    *                   vpsubb	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpsubd	%xmm0, %xmm1, %xmm2
@@ -1736,7 +1736,7 @@ vzeroupper
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     257.00 215.25 235.25 176.17 176.17 38.00  432.25 2.25   12.67
+# CHECK-NEXT:  -     257.00 215.25 235.25 176.17 176.17 38.00  424.25 2.25   12.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -2274,30 +2274,30 @@ vzeroupper
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     vpsignw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpslld	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vpslld	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpslld	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpslld	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpslldq	$1, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpsllq	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vpsllq	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpsllq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpsllq	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpsllw	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vpsllw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpsllw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpsllw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpsrad	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vpsrad	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpsrad	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpsrad	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpsraw	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vpsraw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpsraw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpsraw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpsrld	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vpsrld	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpsrld	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpsrld	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpsrldq	$1, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpsrlq	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vpsrlq	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpsrlq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpsrlq	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpsrlw	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vpsrlw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpsrlw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpsrlw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     vpsubb	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     vpsubb	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     vpsubd	%xmm0, %xmm1, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/Broadwell/resources-sse2.s b/llvm/test/tools/llvm-mca/X86/Broadwell/resources-sse2.s
index 5bed312b0fe1..e76d90521afa 100644
--- a/llvm/test/tools/llvm-mca/X86/Broadwell/resources-sse2.s
+++ b/llvm/test/tools/llvm-mca/X86/Broadwell/resources-sse2.s
@@ -596,30 +596,30 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      6     1.00    *                   pshuflw	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        pslld	$1, %xmm2
 # CHECK-NEXT:  2      2     1.00                        pslld	%xmm0, %xmm2
-# CHECK-NEXT:  3      7     1.00    *                   pslld	(%rax), %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   pslld	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        pslldq	$1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        psllq	$1, %xmm2
 # CHECK-NEXT:  2      2     1.00                        psllq	%xmm0, %xmm2
-# CHECK-NEXT:  3      7     1.00    *                   psllq	(%rax), %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   psllq	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        psllw	$1, %xmm2
 # CHECK-NEXT:  2      2     1.00                        psllw	%xmm0, %xmm2
-# CHECK-NEXT:  3      7     1.00    *                   psllw	(%rax), %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   psllw	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        psrad	$1, %xmm2
 # CHECK-NEXT:  2      2     1.00                        psrad	%xmm0, %xmm2
-# CHECK-NEXT:  3      7     1.00    *                   psrad	(%rax), %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   psrad	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        psraw	$1, %xmm2
 # CHECK-NEXT:  2      2     1.00                        psraw	%xmm0, %xmm2
-# CHECK-NEXT:  3      7     1.00    *                   psraw	(%rax), %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   psraw	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        psrld	$1, %xmm2
 # CHECK-NEXT:  2      2     1.00                        psrld	%xmm0, %xmm2
-# CHECK-NEXT:  3      7     1.00    *                   psrld	(%rax), %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   psrld	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        psrldq	$1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        psrlq	$1, %xmm2
 # CHECK-NEXT:  2      2     1.00                        psrlq	%xmm0, %xmm2
-# CHECK-NEXT:  3      7     1.00    *                   psrlq	(%rax), %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   psrlq	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        psrlw	$1, %xmm2
 # CHECK-NEXT:  2      2     1.00                        psrlw	%xmm0, %xmm2
-# CHECK-NEXT:  3      7     1.00    *                   psrlw	(%rax), %xmm2
+# CHECK-NEXT:  2      7     1.00    *                   psrlw	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        psubb	%xmm0, %xmm2
 # CHECK-NEXT:  2      6     0.50    *                   psubb	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        psubd	%xmm0, %xmm2
@@ -689,7 +689,7 @@ xorpd       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     78.00  70.75  95.75  63.17  63.17  14.00  127.25 2.25   4.67
+# CHECK-NEXT:  -     78.00  70.75  95.75  63.17  63.17  14.00  119.25 2.25   4.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -882,30 +882,30 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     pshuflw	$1, (%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     pslld	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     pslld	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     pslld	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     pslld	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     pslldq	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     psllq	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     psllq	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     psllq	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     psllq	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     psllw	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     psllw	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     psllw	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     psllw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     psrad	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     psrad	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     psrad	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     psrad	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     psraw	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     psraw	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     psraw	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     psraw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     psrld	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     psrld	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     psrld	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     psrld	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     psrldq	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     psrlq	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     psrlq	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     psrlq	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     psrlq	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     psrlw	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     psrlw	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     psrlw	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     psrlw	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     psubb	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     psubb	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     psubd	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/Haswell/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/Haswell/resources-avx1.s
index 3da547de54e3..376070d7f4e0 100644
--- a/llvm/test/tools/llvm-mca/X86/Haswell/resources-avx1.s
+++ b/llvm/test/tools/llvm-mca/X86/Haswell/resources-avx1.s
@@ -1564,30 +1564,30 @@ vzeroupper
 # CHECK-NEXT:  2      7     0.50    *                   vpsignw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpslld	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      2     1.00                        vpslld	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  3      8     1.00    *                   vpslld	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      8     1.00    *                   vpslld	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpslldq	$1, %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpsllq	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      2     1.00                        vpsllq	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  3      8     1.00    *                   vpsllq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      8     1.00    *                   vpsllq	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpsllw	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      2     1.00                        vpsllw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  3      8     1.00    *                   vpsllw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      8     1.00    *                   vpsllw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpsrad	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      2     1.00                        vpsrad	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  3      8     1.00    *                   vpsrad	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      8     1.00    *                   vpsrad	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpsraw	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      2     1.00                        vpsraw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  3      8     1.00    *                   vpsraw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      8     1.00    *                   vpsraw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpsrld	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      2     1.00                        vpsrld	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  3      8     1.00    *                   vpsrld	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      8     1.00    *                   vpsrld	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpsrldq	$1, %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpsrlq	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      2     1.00                        vpsrlq	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  3      8     1.00    *                   vpsrlq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      8     1.00    *                   vpsrlq	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpsrlw	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      2     1.00                        vpsrlw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  3      8     1.00    *                   vpsrlw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      8     1.00    *                   vpsrlw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpsubb	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   vpsubb	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpsubd	%xmm0, %xmm1, %xmm2
@@ -1736,7 +1736,7 @@ vzeroupper
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     336.00 214.58 236.58 176.17 176.17 38.00  435.58 2.25   12.67
+# CHECK-NEXT:  -     336.00 214.58 236.58 176.17 176.17 38.00  427.58 2.25   12.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -2274,30 +2274,30 @@ vzeroupper
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     vpsignw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpslld	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vpslld	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpslld	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpslld	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpslldq	$1, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpsllq	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vpsllq	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpsllq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpsllq	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpsllw	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vpsllw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpsllw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpsllw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpsrad	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vpsrad	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpsrad	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpsrad	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpsraw	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vpsraw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpsraw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpsraw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpsrld	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vpsrld	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpsrld	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpsrld	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpsrldq	$1, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpsrlq	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vpsrlq	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpsrlq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpsrlq	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpsrlw	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vpsrlw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpsrlw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpsrlw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     vpsubb	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     vpsubb	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     vpsubd	%xmm0, %xmm1, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/Haswell/resources-sse2.s b/llvm/test/tools/llvm-mca/X86/Haswell/resources-sse2.s
index 3813ef4707a8..3b4aeb37968f 100644
--- a/llvm/test/tools/llvm-mca/X86/Haswell/resources-sse2.s
+++ b/llvm/test/tools/llvm-mca/X86/Haswell/resources-sse2.s
@@ -596,30 +596,30 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      7     1.00    *                   pshuflw	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        pslld	$1, %xmm2
 # CHECK-NEXT:  2      2     1.00                        pslld	%xmm0, %xmm2
-# CHECK-NEXT:  3      8     1.00    *                   pslld	(%rax), %xmm2
+# CHECK-NEXT:  2      8     1.00    *                   pslld	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        pslldq	$1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        psllq	$1, %xmm2
 # CHECK-NEXT:  2      2     1.00                        psllq	%xmm0, %xmm2
-# CHECK-NEXT:  3      8     1.00    *                   psllq	(%rax), %xmm2
+# CHECK-NEXT:  2      8     1.00    *                   psllq	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        psllw	$1, %xmm2
 # CHECK-NEXT:  2      2     1.00                        psllw	%xmm0, %xmm2
-# CHECK-NEXT:  3      8     1.00    *                   psllw	(%rax), %xmm2
+# CHECK-NEXT:  2      8     1.00    *                   psllw	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        psrad	$1, %xmm2
 # CHECK-NEXT:  2      2     1.00                        psrad	%xmm0, %xmm2
-# CHECK-NEXT:  3      8     1.00    *                   psrad	(%rax), %xmm2
+# CHECK-NEXT:  2      8     1.00    *                   psrad	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        psraw	$1, %xmm2
 # CHECK-NEXT:  2      2     1.00                        psraw	%xmm0, %xmm2
-# CHECK-NEXT:  3      8     1.00    *                   psraw	(%rax), %xmm2
+# CHECK-NEXT:  2      8     1.00    *                   psraw	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        psrld	$1, %xmm2
 # CHECK-NEXT:  2      2     1.00                        psrld	%xmm0, %xmm2
-# CHECK-NEXT:  3      8     1.00    *                   psrld	(%rax), %xmm2
+# CHECK-NEXT:  2      8     1.00    *                   psrld	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        psrldq	$1, %xmm2
 # CHECK-NEXT:  1      1     1.00                        psrlq	$1, %xmm2
 # CHECK-NEXT:  2      2     1.00                        psrlq	%xmm0, %xmm2
-# CHECK-NEXT:  3      8     1.00    *                   psrlq	(%rax), %xmm2
+# CHECK-NEXT:  2      8     1.00    *                   psrlq	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        psrlw	$1, %xmm2
 # CHECK-NEXT:  2      2     1.00                        psrlw	%xmm0, %xmm2
-# CHECK-NEXT:  3      8     1.00    *                   psrlw	(%rax), %xmm2
+# CHECK-NEXT:  2      8     1.00    *                   psrlw	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        psubb	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   psubb	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        psubd	%xmm0, %xmm2
@@ -689,7 +689,7 @@ xorpd       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     112.00 70.75  95.75  63.17  63.17  14.00  127.25 2.25   4.67
+# CHECK-NEXT:  -     112.00 70.75  95.75  63.17  63.17  14.00  119.25 2.25   4.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -882,30 +882,30 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     pshuflw	$1, (%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     pslld	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     pslld	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     pslld	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     pslld	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     pslldq	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     psllq	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     psllq	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     psllq	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     psllq	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     psllw	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     psllw	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     psllw	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     psllw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     psrad	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     psrad	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     psrad	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     psrad	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     psraw	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     psraw	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     psraw	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     psraw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     psrld	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     psrld	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     psrld	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     psrld	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     psrldq	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     psrlq	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     psrlq	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     psrlq	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     psrlq	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     psrlw	$1, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     psrlw	%xmm0, %xmm2
-# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     psrlw	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     psrlw	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     psubb	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     psubb	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     psubd	%xmm0, %xmm2
-- 
GitLab


From bc547028dc3551dc5b78bbebd3413c0129215c59 Mon Sep 17 00:00:00 2001
From: Daniil Kovalev <dkovalev@accesssoftek.com>
Date: Mon, 25 Mar 2024 21:23:28 +0300
Subject: [PATCH 169/404] [llvm-readobj,test] Test
 R_AARCH64_AUTH_{ABS64,RELATIVE} (#86511)

Add missing tests for AUTH relocs defined in #74874
---
 llvm/test/tools/llvm-readobj/ELF/reloc-types-aarch64.test | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/llvm/test/tools/llvm-readobj/ELF/reloc-types-aarch64.test b/llvm/test/tools/llvm-readobj/ELF/reloc-types-aarch64.test
index 729128281104..cf95b5170026 100644
--- a/llvm/test/tools/llvm-readobj/ELF/reloc-types-aarch64.test
+++ b/llvm/test/tools/llvm-readobj/ELF/reloc-types-aarch64.test
@@ -119,6 +119,7 @@
 # CHECK: Type: R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC (571)
 # CHECK: Type: R_AARCH64_TLSLD_LDST128_DTPREL_LO12 (572)
 # CHECK: Type: R_AARCH64_TLSLD_LDST128_DTPREL_LO12_NC (573)
+# CHECK: Type: R_AARCH64_AUTH_ABS64 (580)
 # CHECK: Type: R_AARCH64_COPY (1024)
 # CHECK: Type: R_AARCH64_GLOB_DAT (1025)
 # CHECK: Type: R_AARCH64_JUMP_SLOT (1026)
@@ -128,6 +129,7 @@
 # CHECK: Type: R_AARCH64_TLS_TPREL64 (1030)
 # CHECK: Type: R_AARCH64_TLSDESC (1031)
 # CHECK: Type: R_AARCH64_IRELATIVE (1032)
+# CHECK: Type: R_AARCH64_AUTH_RELATIVE (1041)
 
 --- !ELF
 FileHeader:
@@ -254,6 +256,7 @@ Sections:
       - Type: R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC
       - Type: R_AARCH64_TLSLD_LDST128_DTPREL_LO12
       - Type: R_AARCH64_TLSLD_LDST128_DTPREL_LO12_NC
+      - Type: R_AARCH64_AUTH_ABS64
       - Type: R_AARCH64_COPY
       - Type: R_AARCH64_GLOB_DAT
       - Type: R_AARCH64_JUMP_SLOT
@@ -263,3 +266,4 @@ Sections:
       - Type: R_AARCH64_TLS_TPREL64
       - Type: R_AARCH64_TLSDESC
       - Type: R_AARCH64_IRELATIVE
+      - Type: R_AARCH64_AUTH_RELATIVE
-- 
GitLab


From c8d70e94c4b69e809142054e75b9725ed70418af Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Mon, 25 Mar 2024 11:24:44 -0700
Subject: [PATCH 170/404] [LangRef] Fix spelling mistakes (#86013)

---
 llvm/docs/LangRef.rst | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 391fd2d960dd..030b76935c7b 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -2571,7 +2571,7 @@ are grouped into a single :ref:`attribute group <attrgrp>`.
 ``sanitize_memtag``
     This attribute indicates that the global variable should have AArch64 memory
     tags (MTE) instrumentation applied to it. This attribute causes the
-    suppression of certain optimisations, like GlobalMerge, as well as ensuring
+    suppression of certain optimizations, like GlobalMerge, as well as ensuring
     extra directives are emitted in the assembly and extra bits of metadata are
     placed in the object file so that the linker can ensure the accesses are
     protected by MTE. This attribute is added by clang when
@@ -3638,7 +3638,7 @@ floating-point transformations.
 
 ``contract``
    Allow floating-point contraction (e.g. fusing a multiply followed by an
-   addition into a fused multiply-and-add). This does not enable reassociating
+   addition into a fused multiply-and-add). This does not enable reassociation
    to form arbitrary contractions. For example, ``(a*b) + (c*d) + e`` can not
    be transformed into ``(a*b) + ((c*d) + e)`` to create two fma operations.
 
@@ -7365,7 +7365,7 @@ matches the ``llvm.loop.parallel_accesses`` list.
 If all memory-accessing instructions in a loop have
 ``llvm.access.group`` metadata that each refer to one of the access
 groups of a loop's ``llvm.loop.parallel_accesses`` metadata, then the
-loop has no loop carried memory dependences and is considered to be a
+loop has no loop carried memory dependencies and is considered to be a
 parallel loop.
 
 Note that if not all memory access instructions belong to an access
@@ -13577,12 +13577,12 @@ Overview:
 """""""""
 
 The '``llvm.seh.try.begin``' and '``llvm.seh.try.end``' intrinsics mark
-the boundary of a _try region for Windows SEH Asynchrous Exception Handling.
+the boundary of a _try region for Windows SEH Asynchronous Exception Handling.
 
 Semantics:
 """"""""""
 
-When a C-function is compiled with Windows SEH Asynchrous Exception option,
+When a C-function is compiled with Windows SEH Asynchronous Exception option,
 -feh_asynch (aka MSVC -EHa), these two intrinsics are injected to mark _try
 boundary and to prevent potential exceptions from being moved across boundary.
 Any set of operations can then be confined to the region by reading their leaf
@@ -13603,7 +13603,7 @@ Overview:
 """""""""
 
 The '``llvm.seh.scope.begin``' and '``llvm.seh.scope.end``' intrinsics mark
-the boundary of a CPP object lifetime for Windows SEH Asynchrous Exception
+the boundary of a CPP object lifetime for Windows SEH Asynchronous Exception
 Handling (MSVC option -EHa).
 
 Semantics:
@@ -17713,7 +17713,7 @@ Examples
       %res = call i4 @llvm.udiv.fix.sat.i4(i4 8, i4 2, i32 2)  ; %res = 15 (2 / 0.5 = 4 => 3.75)
 
 
-Specialised Arithmetic Intrinsics
+Specialized Arithmetic Intrinsics
 ---------------------------------
 
 .. _i_intr_llvm_canonicalize:
@@ -18037,9 +18037,9 @@ The '``llvm.loop.decrement.reg.*``' intrinsics do an integer ``SUB`` of its
 two operands, which is not allowed to wrap. They return the remaining number of
 iterations still to be executed, and can be used together with a ``PHI``,
 ``ICMP`` and ``BR`` to control the number of loop iterations executed. Any
-optimisations are allowed to treat it is a ``SUB``, and it is supported by
+optimizations are allowed to treat it is a ``SUB``, and it is supported by
 SCEV, so it's the backends responsibility to handle cases where it may be
-optimised. These intrinsics are marked as ``IntrNoDuplicate`` to avoid
+optimized. These intrinsics are marked as ``IntrNoDuplicate`` to avoid
 optimizers duplicating these instructions.
 
 
@@ -18741,7 +18741,7 @@ Arguments:
 The first argument is the vector to be counted. This argument must be a vector
 with integer element type. The return type must also be an integer type which is
 wide enough to hold the maximum number of elements of the source vector. The
-behaviour of this intrinsic is undefined if the return type is not wide enough
+behavior of this intrinsic is undefined if the return type is not wide enough
 for the number of elements in the input vector.
 
 The second argument is a constant flag that indicates whether the intrinsic
@@ -22026,7 +22026,7 @@ and ``evl2`` are unsigned integers indicating the explicit vector lengths of
 ``vec1`` and ``vec2`` respectively.  ``imm``, ``evl1`` and ``evl2`` should
 respect the following constraints: ``-evl1 <= imm < evl1``, ``0 <= evl1 <= VL``
 and ``0 <= evl2 <= VL``, where ``VL`` is the runtime vector factor. If these
-constraints are not satisfied the intrinsic has undefined behaviour.
+constraints are not satisfied the intrinsic has undefined behavior.
 
 Semantics:
 """"""""""
@@ -24442,7 +24442,7 @@ operand. The pointer alignment defaults to 1.
 Semantics:
 """"""""""
 
-The '``llvm.masked.compressstore``' intrinsic is designed for compressing data in memory. It allows to collect elements from possibly non-adjacent lanes of a vector and store them contiguously in memory in one IR operation. It is useful for targets that support compressing store operations and allows vectorizing loops with cross-iteration dependences like in the following example:
+The '``llvm.masked.compressstore``' intrinsic is designed for compressing data in memory. It allows to collect elements from possibly non-adjacent lanes of a vector and store them contiguously in memory in one IR operation. It is useful for targets that support compressing store operations and allows vectorizing loops with cross-iteration dependencies like in the following example:
 
 .. code-block:: c
 
@@ -26584,7 +26584,7 @@ Semantics:
 
 The '``llvm.set.fpenv``' intrinsic sets the current floating-point environment
 to the state specified by the argument. The state may be previously obtained by a
-call to '``llvm.get.fpenv``' or synthesised in a platform-dependent way.
+call to '``llvm.get.fpenv``' or synthesized in a platform-dependent way.
 
 
 '``llvm.reset.fpenv``' Intrinsic
@@ -27013,7 +27013,7 @@ This intrinsic is lowered to code which is intended to cause an execution trap,
 embedding the argument into encoding of that trap somehow to discriminate
 crashes if possible.
 
-Equivalent to ``@llvm.trap`` for targets that do not support this behaviour.
+Equivalent to ``@llvm.trap`` for targets that do not support this behavior.
 
 '``llvm.stackprotector``' Intrinsic
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -27954,7 +27954,7 @@ Arguments:
 The first three arguments are the same as they are in the :ref:`@llvm.memcpy <int_memcpy>`
 intrinsic, with the added constraint that ``len`` is required to be a positive integer
 multiple of the ``element_size``. If ``len`` is not a positive integer multiple of
-``element_size``, then the behaviour of the intrinsic is undefined.
+``element_size``, then the behavior of the intrinsic is undefined.
 
 ``element_size`` must be a compile-time constant positive power of two no greater than
 target-specific atomic access size limit.
@@ -28030,7 +28030,7 @@ The first three arguments are the same as they are in the
 :ref:`@llvm.memmove <int_memmove>` intrinsic, with the added constraint that
 ``len`` is required to be a positive integer multiple of the ``element_size``.
 If ``len`` is not a positive integer multiple of ``element_size``, then the
-behaviour of the intrinsic is undefined.
+behavior of the intrinsic is undefined.
 
 ``element_size`` must be a compile-time constant positive power of two no
 greater than a target-specific atomic access size limit.
@@ -28109,7 +28109,7 @@ Arguments:
 The first three arguments are the same as they are in the :ref:`@llvm.memset <int_memset>`
 intrinsic, with the added constraint that ``len`` is required to be a positive integer
 multiple of the ``element_size``. If ``len`` is not a positive integer multiple of
-``element_size``, then the behaviour of the intrinsic is undefined.
+``element_size``, then the behavior of the intrinsic is undefined.
 
 ``element_size`` must be a compile-time constant positive power of two no greater than
 target-specific atomic access size limit.
-- 
GitLab


From 6ac54105ad5a217c0cf587824dbdffde5eba4e97 Mon Sep 17 00:00:00 2001
From: Justin Bogner <mail@justinbogner.com>
Date: Mon, 25 Mar 2024 11:42:09 -0700
Subject: [PATCH 171/404] [DirectX] disable experimental debuginfo iterators in
 debug-info test (#86538)

We'll need to address this differently long term, but for now just
revert to the old behaviour to keep this test from failing.

Fixes #85884
---
 llvm/test/tools/dxil-dis/debug-info.ll | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/test/tools/dxil-dis/debug-info.ll b/llvm/test/tools/dxil-dis/debug-info.ll
index 92dc65481e82..96e023338e5c 100644
--- a/llvm/test/tools/dxil-dis/debug-info.ll
+++ b/llvm/test/tools/dxil-dis/debug-info.ll
@@ -1,4 +1,4 @@
-; RUN: llc --filetype=obj %s -o - | dxil-dis -o - | FileCheck %s
+; RUN: llc --filetype=obj %s -o  - -experimental-debuginfo-iterators=false | dxil-dis -o - | FileCheck %s
 target triple = "dxil-unknown-shadermodel6.7-library"
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
-- 
GitLab


From 9056ce88042616b431f988758fff73b82fa58a08 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Mon, 25 Mar 2024 11:45:33 -0700
Subject: [PATCH 172/404] Revert "[RISCV][GISEL] Legalize G_VSCALE"

This reverts commit 47681506ded30fada68f180b5e80f740bc76abcd. It is not
consistent with SelectionDAG.
---
 .../CodeGen/GlobalISel/MachineIRBuilder.h     |  11 -
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    |  51 +---
 .../CodeGen/GlobalISel/MachineIRBuilder.cpp   |   7 -
 .../Target/RISCV/GISel/RISCVLegalizerInfo.cpp |  48 ---
 .../Target/RISCV/GISel/RISCVLegalizerInfo.h   |   1 -
 llvm/lib/Target/RISCV/RISCVInstrGISel.td      |   8 -
 .../legalizer/rvv/legalize-vscale-rv32.mir    | 283 ------------------
 .../legalizer/rvv/legalize-vscale-rv64.mir    | 120 --------
 8 files changed, 1 insertion(+), 528 deletions(-)
 delete mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv32.mir
 delete mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv64.mir

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 16a7fc446fbe..3ba036ae713f 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -1165,17 +1165,6 @@ public:
   /// \return a MachineInstrBuilder for the newly created instruction.
   MachineInstrBuilder buildVScale(const DstOp &Res, const ConstantInt &MinElts);
 
-  /// Build and insert \p Res = G_VSCALE \p MinElts
-  ///
-  /// G_VSCALE puts the value of the runtime vscale multiplied by \p MinElts
-  /// into \p Res.
-  ///
-  /// \pre setBasicBlock or setMI must have been called.
-  /// \pre \p Res must be a generic virtual register with scalar type.
-  ///
-  /// \return a MachineInstrBuilder for the newly created instruction.
-  MachineInstrBuilder buildVScale(const DstOp &Res, const APInt &MinElts);
-
   /// Build and insert a G_INTRINSIC instruction.
   ///
   /// There are four different opcodes based on combinations of whether the
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 28b73599e411..1b25da8833e4 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -1699,36 +1699,6 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
   case TargetOpcode::G_FLDEXP:
   case TargetOpcode::G_STRICT_FLDEXP:
     return narrowScalarFLDEXP(MI, TypeIdx, NarrowTy);
-  case TargetOpcode::G_VSCALE: {
-    LLT Ty = MRI.getType(MI.getOperand(0).getReg());
-    const APInt &Val = MI.getOperand(1).getCImm()->getValue();
-    unsigned TotalSize = Ty.getSizeInBits();
-    unsigned NarrowSize = NarrowTy.getSizeInBits();
-    int NumParts = TotalSize / NarrowSize;
-
-    SmallVector<Register, 4> PartRegs;
-    for (int I = 0; I != NumParts; ++I) {
-      unsigned Offset = I * NarrowSize;
-      auto K =
-          MIRBuilder.buildVScale(NarrowTy, Val.lshr(Offset).trunc(NarrowSize));
-      PartRegs.push_back(K.getReg(0));
-    }
-    LLT LeftoverTy;
-    unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
-    SmallVector<Register, 1> LeftoverRegs;
-    if (LeftoverBits != 0) {
-      LeftoverTy = LLT::scalar(LeftoverBits);
-      auto K = MIRBuilder.buildVScale(
-          LeftoverTy, Val.lshr(NumParts * NarrowSize).trunc(LeftoverBits));
-      LeftoverRegs.push_back(K.getReg(0));
-    }
-
-    insertParts(MI.getOperand(0).getReg(), Ty, NarrowTy, PartRegs, LeftoverTy,
-                LeftoverRegs);
-
-    MI.eraseFromParent();
-    return Legalized;
-  }
   }
 }
 
@@ -2996,7 +2966,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
   case TargetOpcode::G_VECREDUCE_FMIN:
   case TargetOpcode::G_VECREDUCE_FMAX:
   case TargetOpcode::G_VECREDUCE_FMINIMUM:
-  case TargetOpcode::G_VECREDUCE_FMAXIMUM: {
+  case TargetOpcode::G_VECREDUCE_FMAXIMUM:
     if (TypeIdx != 0)
       return UnableToLegalize;
     Observer.changingInstr(MI);
@@ -3010,25 +2980,6 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     Observer.changedInstr(MI);
     return Legalized;
   }
-  case TargetOpcode::G_VSCALE: {
-    MachineOperand &SrcMO = MI.getOperand(1);
-    LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
-    unsigned ExtOpc = LI.getExtOpcodeForWideningConstant(
-        MRI.getType(MI.getOperand(0).getReg()));
-    assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT ||
-            ExtOpc == TargetOpcode::G_ANYEXT) &&
-           "Illegal Extend");
-    const APInt &SrcVal = SrcMO.getCImm()->getValue();
-    const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT)
-                           ? SrcVal.sext(WideTy.getSizeInBits())
-                           : SrcVal.zext(WideTy.getSizeInBits());
-    Observer.changingInstr(MI);
-    SrcMO.setCImm(ConstantInt::get(Ctx, Val));
-    widenScalarDst(MI, WideTy);
-    Observer.changedInstr(MI);
-    return Legalized;
-  }
-  }
 }
 
 static void getUnmergePieces(SmallVectorImpl<Register> &Pieces,
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index f2665e25d195..f7aaa0f02efc 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -811,13 +811,6 @@ MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
   return VScale;
 }
 
-MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
-                                                  const APInt &MinElts) {
-  ConstantInt *CI =
-      ConstantInt::get(getMF().getFunction().getContext(), MinElts);
-  return buildVScale(Res, *CI);
-}
-
 static unsigned getIntrinsicOpcode(bool HasSideEffects, bool IsConvergent) {
   if (HasSideEffects && IsConvergent)
     return TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS;
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 9a388f4cd271..941ad752bfd9 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -408,10 +408,6 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
       .clampScalar(0, s32, sXLen)
       .lowerForCartesianProduct({s32, sXLen, p0}, {p0});
 
-  getActionDefinitionsBuilder(G_VSCALE)
-      .clampScalar(0, sXLen, sXLen)
-      .customFor({sXLen});
-
   getLegacyLegalizerInfo().computeTables();
 }
 
@@ -533,48 +529,6 @@ bool RISCVLegalizerInfo::shouldBeInConstantPool(APInt APImm,
   return !(!SeqLo.empty() && (SeqLo.size() + 2) <= STI.getMaxBuildIntsCost());
 }
 
-bool RISCVLegalizerInfo::legalizeVScale(MachineInstr &MI,
-                                        MachineIRBuilder &MIB) const {
-  const LLT XLenTy(STI.getXLenVT());
-  Register Dst = MI.getOperand(0).getReg();
-
-  // We define our scalable vector types for lmul=1 to use a 64 bit known
-  // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
-  // vscale as VLENB / 8.
-  static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
-  if (STI.getRealMinVLen() < RISCV::RVVBitsPerBlock)
-    // Support for VLEN==32 is incomplete.
-    return false;
-
-  // We assume VLENB is a multiple of 8. We manually choose the best shift
-  // here because SimplifyDemandedBits isn't always able to simplify it.
-  uint64_t Val = MI.getOperand(1).getCImm()->getZExtValue();
-  if (isPowerOf2_64(Val)) {
-    uint64_t Log2 = Log2_64(Val);
-    if (Log2 < 3) {
-      auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {XLenTy}, {});
-      MIB.buildLShr(Dst, VLENB, MIB.buildConstant(XLenTy, 3 - Log2));
-    } else if (Log2 > 3) {
-      auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {XLenTy}, {});
-      MIB.buildShl(Dst, VLENB, MIB.buildConstant(XLenTy, Log2 - 3));
-    } else {
-      MIB.buildInstr(RISCV::G_READ_VLENB, {Dst}, {});
-    }
-  } else if ((Val % 8) == 0) {
-    // If the multiplier is a multiple of 8, scale it down to avoid needing
-    // to shift the VLENB value.
-    auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {XLenTy}, {});
-    MIB.buildMul(Dst, VLENB, MIB.buildConstant(XLenTy, Val / 8));
-  } else {
-    auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {XLenTy}, {});
-    auto VScale = MIB.buildLShr(XLenTy, VLENB, MIB.buildConstant(XLenTy, 3));
-    MIB.buildMul(Dst, VScale, MIB.buildConstant(XLenTy, Val));
-  }
-
-  MI.eraseFromParent();
-  return true;
-}
-
 bool RISCVLegalizerInfo::legalizeCustom(
     LegalizerHelper &Helper, MachineInstr &MI,
     LostDebugLocObserver &LocObserver) const {
@@ -632,8 +586,6 @@ bool RISCVLegalizerInfo::legalizeCustom(
   }
   case TargetOpcode::G_VASTART:
     return legalizeVAStart(MI, MIRBuilder);
-  case TargetOpcode::G_VSCALE:
-    return legalizeVScale(MI, MIRBuilder);
   }
 
   llvm_unreachable("expected switch to return");
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
index e2a98c8d2c73..323426034827 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
@@ -42,7 +42,6 @@ private:
                            GISelChangeObserver &Observer) const;
 
   bool legalizeVAStart(MachineInstr &MI, MachineIRBuilder &MIRBuilder) const;
-  bool legalizeVScale(MachineInstr &MI, MachineIRBuilder &MIB) const;
 };
 } // end namespace llvm
 #endif
diff --git a/llvm/lib/Target/RISCV/RISCVInstrGISel.td b/llvm/lib/Target/RISCV/RISCVInstrGISel.td
index 54e22d625781..ede8c9809833 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrGISel.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrGISel.td
@@ -24,11 +24,3 @@ def G_FCLASS : RISCVGenericInstruction {
   let hasSideEffects = false;
 }
 def : GINodeEquiv<G_FCLASS, riscv_fclass>;
-
-// Pseudo equivalent to a RISCVISD::READ_VLENB.
-def G_READ_VLENB : RISCVGenericInstruction {
-  let OutOperandList = (outs type0:$dst);
-  let InOperandList = (ins);
-  let hasSideEffects = false;
-}
-def : GINodeEquiv<G_READ_VLENB, riscv_read_vlenb>;
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv32.mir
deleted file mode 100644
index 396349542236..000000000000
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv32.mir
+++ /dev/null
@@ -1,283 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=riscv32 -mattr=+v -run-pass=legalizer %s -o - | FileCheck %s
-
----
-name:            test_1_s32
-body:             |
-  bb.0.entry:
-
-    ; CHECK-LABEL: name: test_1_s32
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s32) = G_VSCALE i32 1
-    $x10 = COPY %0
-    PseudoRET implicit $x10
-...
----
-name:            test_2_s32
-body:             |
-  bb.0.entry:
-
-    ; CHECK-LABEL: name: test_2_s32
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s32) = G_VSCALE i32 2
-    $x10 = COPY %0
-    PseudoRET implicit $x10
-...
----
-name:            test_3_s32
-body:             |
-  bb.0.entry:
-
-    ; CHECK-LABEL: name: test_3_s32
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
-    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s32) = G_VSCALE i32 3
-    $x10 = COPY %0
-    PseudoRET implicit $x10
-...
----
-name:            test_4_s32
-body:             |
-  bb.0.entry:
-
-    ; CHECK-LABEL: name: test_4_s32
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s32) = G_VSCALE i32 4
-    $x10 = COPY %0
-    PseudoRET implicit $x10
-...
----
-name:            test_8_s32
-body:             |
-  bb.0.entry:
-
-    ; CHECK-LABEL: name: test_8_s32
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s32) = G_VSCALE i32 8
-    $x10 = COPY %0
-    PseudoRET implicit $x10
-...
----
-name:            test_16_s32
-body:             |
-  bb.0.entry:
-
-    ; CHECK-LABEL: name: test_16_s32
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: $x10 = COPY [[SHL]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s32) = G_VSCALE i32 16
-    $x10 = COPY %0
-    PseudoRET implicit $x10
-...
----
-name:            test_40_s32
-body:             |
-  bb.0.entry:
-
-    ; CHECK-LABEL: name: test_40_s32
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
-    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s32) = G_VSCALE i32 40
-    $x10 = COPY %0
-    PseudoRET implicit $x10
-...
-
----
-name:            test_1_s64
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_1_s64
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s64) = G_VSCALE i64 1
-    %1:_(s32) = G_TRUNC %0
-    $x10 = COPY %1
-    PseudoRET implicit $x10
-...
----
-name:            test_2_s64
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_2_s64
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s64) = G_VSCALE i64 2
-    %1:_(s32) = G_TRUNC %0
-    $x10 = COPY %1
-    PseudoRET implicit $x10
-...
----
-name:            test_3_s64
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_3_s64
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
-    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C2]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s64) = G_VSCALE i64 3
-    %1:_(s32) = G_TRUNC %0
-    $x10 = COPY %1
-    PseudoRET implicit $x10
-...
----
-name:            test_4_s64
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_4_s64
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s64) = G_VSCALE i64 4
-    %1:_(s32) = G_TRUNC %0
-    $x10 = COPY %1
-    PseudoRET implicit $x10
-...
----
-name:            test_8_s64
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_8_s64
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s64) = G_VSCALE i64 8
-    %1:_(s32) = G_TRUNC %0
-    $x10 = COPY %1
-    PseudoRET implicit $x10
-...
----
-name:            test_16_s64
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_16_s64
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[SHL]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s64) = G_VSCALE i64 16
-    %1:_(s32) = G_TRUNC %0
-    $x10 = COPY %1
-    PseudoRET implicit $x10
-...
----
-name:            test_40_s64
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_40_s64
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
-    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:_(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s64) = G_VSCALE i64 40
-    %1:_(s32) = G_TRUNC %0
-    $x10 = COPY %1
-    PseudoRET implicit $x10
-...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv64.mir
deleted file mode 100644
index 3e140a5ef72a..000000000000
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv64.mir
+++ /dev/null
@@ -1,120 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=riscv64 -mattr=+v -run-pass=legalizer %s -o - | FileCheck %s
-
----
-name:            test_1
-body:             |
-  bb.0.entry:
-
-    ; CHECK-LABEL: name: test_1
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s64) = G_VSCALE i64 1
-    $x10 = COPY %0
-    PseudoRET implicit $x10
-...
----
-name:            test_2
-body:             |
-  bb.0.entry:
-
-    ; CHECK-LABEL: name: test_2
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s64) = G_VSCALE i64 2
-    $x10 = COPY %0
-    PseudoRET implicit $x10
-...
----
-name:            test_3
-body:             |
-  bb.0.entry:
-
-    ; CHECK-LABEL: name: test_3
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
-    ; CHECK-NEXT: $x11 = COPY [[C1]](s64)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
-    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s64) = G_VSCALE i64 3
-    $x10 = COPY %0
-    PseudoRET implicit $x10
-...
----
-name:            test_4
-body:             |
-  bb.0.entry:
-
-    ; CHECK-LABEL: name: test_4
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s64) = G_VSCALE i64 4
-    $x10 = COPY %0
-    PseudoRET implicit $x10
-...
----
-name:            test_8
-body:             |
-  bb.0.entry:
-
-    ; CHECK-LABEL: name: test_8
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s64)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s64) = G_VSCALE i64 8
-    $x10 = COPY %0
-    PseudoRET implicit $x10
-...
----
-name:            test_16
-body:             |
-  bb.0.entry:
-
-    ; CHECK-LABEL: name: test_16
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[READ_VLENB]], [[C]](s64)
-    ; CHECK-NEXT: $x10 = COPY [[SHL]](s64)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s64) = G_VSCALE i64 16
-    $x10 = COPY %0
-    PseudoRET implicit $x10
-...
----
-name:            test_40
-body:             |
-  bb.0.entry:
-
-    ; CHECK-LABEL: name: test_40
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 5
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s64)
-    ; CHECK-NEXT: $x11 = COPY [[C]](s64)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
-    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s64) = G_VSCALE i64 40
-    $x10 = COPY %0
-    PseudoRET implicit $x10
-...
-
-
-- 
GitLab


From 668687f8a8ccba847e4847597d13124bd9bfdf4a Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Mon, 25 Mar 2024 11:50:21 -0700
Subject: [PATCH 173/404] Revert "[RISCV][GISEL] Add regbankselect tests for
 G_VSCALE"

This reverts commit a2476c99b745381380eab245fc9499a4ecf0b39e. It is not
consistent with SelectionDAG
---
 .../regbankselect/rvv/vscale-rv32.mir         | 409 ------------------
 .../regbankselect/rvv/vscale-rv64.mir         | 158 -------
 2 files changed, 567 deletions(-)
 delete mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv32.mir
 delete mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv64.mir

diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv32.mir
deleted file mode 100644
index 2bc33f94978c..000000000000
--- a/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv32.mir
+++ /dev/null
@@ -1,409 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=riscv32 -mattr=+m,+v -run-pass=regbankselect \
-# RUN:   -disable-gisel-legality-check -simplify-mir -verify-machineinstrs %s \
-# RUN:   -o - | FileCheck %s
-
----
-name:            test_1_s32
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_1_s32
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 3
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %1:_(s32) = G_READ_VLENB
-    %2:_(s32) = G_CONSTANT i32 3
-    %0:_(s32) = G_LSHR %1, %2(s32)
-    $x10 = COPY %0(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_2_s32
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_2_s32
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 2
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %1:_(s32) = G_READ_VLENB
-    %2:_(s32) = G_CONSTANT i32 2
-    %0:_(s32) = G_LSHR %1, %2(s32)
-    $x10 = COPY %0(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_3_s32
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_3_s32
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 3
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 3
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(s32) = COPY $x10
-    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %1:_(s32) = G_READ_VLENB
-    %2:_(s32) = G_CONSTANT i32 3
-    %3:_(s32) = G_LSHR %1, %2(s32)
-    %4:_(s32) = G_CONSTANT i32 3
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %3(s32)
-    $x11 = COPY %4(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    %0:_(s32) = COPY $x10
-    $x10 = COPY %0(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_4_s32
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_4_s32
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %1:_(s32) = G_READ_VLENB
-    %2:_(s32) = G_CONSTANT i32 1
-    %0:_(s32) = G_LSHR %1, %2(s32)
-    $x10 = COPY %0(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_8_s32
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_8_s32
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s32) = G_READ_VLENB
-    $x10 = COPY %0(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_16_s32
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_16_s32
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:gprb(s32) = G_SHL [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: $x10 = COPY [[SHL]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %1:_(s32) = G_READ_VLENB
-    %2:_(s32) = G_CONSTANT i32 1
-    %0:_(s32) = G_SHL %1, %2(s32)
-    $x10 = COPY %0(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_40_s32
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_40_s32
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 5
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(s32) = COPY $x10
-    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %1:_(s32) = G_READ_VLENB
-    %2:_(s32) = G_CONSTANT i32 5
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %1(s32)
-    $x11 = COPY %2(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    %0:_(s32) = COPY $x10
-    $x10 = COPY %0(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_1_s64
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_1_s64
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 3
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %6:_(s32) = G_READ_VLENB
-    %7:_(s32) = G_CONSTANT i32 3
-    %2:_(s32) = G_LSHR %6, %7(s32)
-    %4:_(s32) = G_READ_VLENB
-    %5:_(s32) = G_CONSTANT i32 0
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %4(s32)
-    $x11 = COPY %5(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %2(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_2_s64
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_2_s64
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 2
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %6:_(s32) = G_READ_VLENB
-    %7:_(s32) = G_CONSTANT i32 2
-    %2:_(s32) = G_LSHR %6, %7(s32)
-    %4:_(s32) = G_READ_VLENB
-    %5:_(s32) = G_CONSTANT i32 0
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %4(s32)
-    $x11 = COPY %5(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %2(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_3_s64
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_3_s64
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 3
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 3
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(s32) = COPY $x10
-    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C2]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %6:_(s32) = G_READ_VLENB
-    %7:_(s32) = G_CONSTANT i32 3
-    %8:_(s32) = G_LSHR %6, %7(s32)
-    %9:_(s32) = G_CONSTANT i32 3
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %8(s32)
-    $x11 = COPY %9(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    %2:_(s32) = COPY $x10
-    %4:_(s32) = G_READ_VLENB
-    %5:_(s32) = G_CONSTANT i32 0
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %4(s32)
-    $x11 = COPY %5(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %2(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_4_s64
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_4_s64
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %6:_(s32) = G_READ_VLENB
-    %7:_(s32) = G_CONSTANT i32 1
-    %2:_(s32) = G_LSHR %6, %7(s32)
-    %4:_(s32) = G_READ_VLENB
-    %5:_(s32) = G_CONSTANT i32 0
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %4(s32)
-    $x11 = COPY %5(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %2(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_8_s64
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_8_s64
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %2:_(s32) = G_READ_VLENB
-    %4:_(s32) = G_READ_VLENB
-    %5:_(s32) = G_CONSTANT i32 0
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %4(s32)
-    $x11 = COPY %5(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %2(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_16_s64
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_16_s64
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:gprb(s32) = G_SHL [[READ_VLENB]], [[C]](s32)
-    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[SHL]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %6:_(s32) = G_READ_VLENB
-    %7:_(s32) = G_CONSTANT i32 1
-    %2:_(s32) = G_SHL %6, %7(s32)
-    %4:_(s32) = G_READ_VLENB
-    %5:_(s32) = G_CONSTANT i32 0
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %4(s32)
-    $x11 = COPY %5(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %2(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_40_s64
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_40_s64
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 5
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(s32) = COPY $x10
-    ; CHECK-NEXT: [[READ_VLENB1:%[0-9]+]]:gprb(s32) = G_READ_VLENB
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB1]](s32)
-    ; CHECK-NEXT: $x11 = COPY [[C1]](s32)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[COPY]](s32)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %6:_(s32) = G_READ_VLENB
-    %7:_(s32) = G_CONSTANT i32 5
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %6(s32)
-    $x11 = COPY %7(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    %2:_(s32) = COPY $x10
-    %4:_(s32) = G_READ_VLENB
-    %5:_(s32) = G_CONSTANT i32 0
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %4(s32)
-    $x11 = COPY %5(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %2(s32)
-    PseudoRET implicit $x10
-
-...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv64.mir
deleted file mode 100644
index 45a5a1d84646..000000000000
--- a/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv64.mir
+++ /dev/null
@@ -1,158 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=riscv64 -mattr=+m,+v -run-pass=regbankselect \
-# RUN:   -disable-gisel-legality-check -simplify-mir -verify-machineinstrs %s \
-# RUN:   -o - | FileCheck %s
-
----
-name:            test_1
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_1
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s64) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 3
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %1:_(s64) = G_READ_VLENB
-    %2:_(s64) = G_CONSTANT i64 3
-    %0:_(s64) = G_LSHR %1, %2(s64)
-    $x10 = COPY %0(s64)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_2
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_2
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s64) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 2
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %1:_(s64) = G_READ_VLENB
-    %2:_(s64) = G_CONSTANT i64 2
-    %0:_(s64) = G_LSHR %1, %2(s64)
-    $x10 = COPY %0(s64)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_3
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_3
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s64) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 3
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
-    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 3
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
-    ; CHECK-NEXT: $x11 = COPY [[C1]](s64)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(s64) = COPY $x10
-    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %1:_(s64) = G_READ_VLENB
-    %2:_(s64) = G_CONSTANT i64 3
-    %3:_(s64) = G_LSHR %1, %2(s64)
-    %4:_(s64) = G_CONSTANT i64 3
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %3(s64)
-    $x11 = COPY %4(s64)
-    PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    %0:_(s64) = COPY $x10
-    $x10 = COPY %0(s64)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_4
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_4
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s64) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
-    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %1:_(s64) = G_READ_VLENB
-    %2:_(s64) = G_CONSTANT i64 1
-    %0:_(s64) = G_LSHR %1, %2(s64)
-    $x10 = COPY %0(s64)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_8
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_8
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s64) = G_READ_VLENB
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s64)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:_(s64) = G_READ_VLENB
-    $x10 = COPY %0(s64)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_16
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_16
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s64) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 1
-    ; CHECK-NEXT: [[SHL:%[0-9]+]]:gprb(s64) = G_SHL [[READ_VLENB]], [[C]](s64)
-    ; CHECK-NEXT: $x10 = COPY [[SHL]](s64)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %1:_(s64) = G_READ_VLENB
-    %2:_(s64) = G_CONSTANT i64 1
-    %0:_(s64) = G_SHL %1, %2(s64)
-    $x10 = COPY %0(s64)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_40
-legalized:       true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_40
-    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s64) = G_READ_VLENB
-    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 5
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s64)
-    ; CHECK-NEXT: $x11 = COPY [[C]](s64)
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(s64) = COPY $x10
-    ; CHECK-NEXT: $x10 = COPY [[COPY]](s64)
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %1:_(s64) = G_READ_VLENB
-    %2:_(s64) = G_CONSTANT i64 5
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %1(s64)
-    $x11 = COPY %2(s64)
-    PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    %0:_(s64) = COPY $x10
-    $x10 = COPY %0(s64)
-    PseudoRET implicit $x10
-
-...
-- 
GitLab


From 8b9c3b57b158beeedf69810c2a546ab350a91e57 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Mon, 25 Mar 2024 11:50:57 -0700
Subject: [PATCH 174/404] Revert "[RISCV][GISEL] Add instruction select tests
 for G_VSCALE"

This reverts commit c00a5ab8c4be14f63735ec61c5c9245c233cbcfc. It is not
consistent with SelectionDAG.
---
 .../instruction-select/rvv/vscale32.mir       | 412 ------------------
 .../instruction-select/rvv/vscale64.mir       | 140 ------
 2 files changed, 552 deletions(-)
 delete mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale32.mir
 delete mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale64.mir

diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale32.mir
deleted file mode 100644
index 084d5531c459..000000000000
--- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale32.mir
+++ /dev/null
@@ -1,412 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=riscv32 -mattr=+v -run-pass=instruction-select \
-# RUN:   -simplify-mir -verify-machineinstrs %s -o - | FileCheck  %s
-
----
-name:            test_1_s32
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_1_s32
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
-    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s32) = G_READ_VLENB
-    %1:gprb(s32) = G_CONSTANT i32 3
-    %2:gprb(s32) = G_LSHR %0, %1(s32)
-    $x10 = COPY %2(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_2_s32
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_2_s32
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 2
-    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s32) = G_READ_VLENB
-    %1:gprb(s32) = G_CONSTANT i32 2
-    %2:gprb(s32) = G_LSHR %0, %1(s32)
-    $x10 = COPY %2(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_3_s32
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_3_s32
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
-    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 3
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
-    ; CHECK-NEXT: $x11 = COPY [[ADDI]]
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
-    ; CHECK-NEXT: $x10 = COPY [[COPY]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s32) = G_READ_VLENB
-    %1:gprb(s32) = G_CONSTANT i32 3
-    %2:gprb(s32) = G_LSHR %0, %1(s32)
-    %3:gprb(s32) = G_CONSTANT i32 3
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %2(s32)
-    $x11 = COPY %3(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    %4:gprb(s32) = COPY $x10
-    $x10 = COPY %4(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_4_s32
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_4_s32
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 1
-    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s32) = G_READ_VLENB
-    %1:gprb(s32) = G_CONSTANT i32 1
-    %2:gprb(s32) = G_LSHR %0, %1(s32)
-    $x10 = COPY %2(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_8_s32
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_8_s32
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s32) = G_READ_VLENB
-    $x10 = COPY %0(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_16_s32
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_16_s32
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[SLLI:%[0-9]+]]:gpr = SLLI [[PseudoReadVLENB]], 1
-    ; CHECK-NEXT: $x10 = COPY [[SLLI]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s32) = G_READ_VLENB
-    %1:gprb(s32) = G_CONSTANT i32 1
-    %2:gprb(s32) = G_SHL %0, %1(s32)
-    $x10 = COPY %2(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_40_s32
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_40_s32
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 5
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB]]
-    ; CHECK-NEXT: $x11 = COPY [[ADDI]]
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
-    ; CHECK-NEXT: $x10 = COPY [[COPY]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s32) = G_READ_VLENB
-    %1:gprb(s32) = G_CONSTANT i32 5
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %0(s32)
-    $x11 = COPY %1(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    %2:gprb(s32) = COPY $x10
-    $x10 = COPY %2(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_1_s64
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_1_s64
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
-    ; CHECK-NEXT: [[PseudoReadVLENB1:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB1]]
-    ; CHECK-NEXT: $x11 = COPY [[COPY]]
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s32) = G_READ_VLENB
-    %1:gprb(s32) = G_CONSTANT i32 3
-    %2:gprb(s32) = G_LSHR %0, %1(s32)
-    %3:gprb(s32) = G_READ_VLENB
-    %4:gprb(s32) = G_CONSTANT i32 0
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %3(s32)
-    $x11 = COPY %4(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %2(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_2_s64
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_2_s64
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 2
-    ; CHECK-NEXT: [[PseudoReadVLENB1:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB1]]
-    ; CHECK-NEXT: $x11 = COPY [[COPY]]
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s32) = G_READ_VLENB
-    %1:gprb(s32) = G_CONSTANT i32 2
-    %2:gprb(s32) = G_LSHR %0, %1(s32)
-    %3:gprb(s32) = G_READ_VLENB
-    %4:gprb(s32) = G_CONSTANT i32 0
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %3(s32)
-    $x11 = COPY %4(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %2(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_3_s64
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_3_s64
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
-    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 3
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
-    ; CHECK-NEXT: $x11 = COPY [[ADDI]]
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
-    ; CHECK-NEXT: [[PseudoReadVLENB1:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB1]]
-    ; CHECK-NEXT: $x11 = COPY [[COPY1]]
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[COPY]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s32) = G_READ_VLENB
-    %1:gprb(s32) = G_CONSTANT i32 3
-    %2:gprb(s32) = G_LSHR %0, %1(s32)
-    %3:gprb(s32) = G_CONSTANT i32 3
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %2(s32)
-    $x11 = COPY %3(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    %4:gprb(s32) = COPY $x10
-    %5:gprb(s32) = G_READ_VLENB
-    %6:gprb(s32) = G_CONSTANT i32 0
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %5(s32)
-    $x11 = COPY %6(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %4(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_4_s64
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_4_s64
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 1
-    ; CHECK-NEXT: [[PseudoReadVLENB1:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB1]]
-    ; CHECK-NEXT: $x11 = COPY [[COPY]]
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s32) = G_READ_VLENB
-    %1:gprb(s32) = G_CONSTANT i32 1
-    %2:gprb(s32) = G_LSHR %0, %1(s32)
-    %3:gprb(s32) = G_READ_VLENB
-    %4:gprb(s32) = G_CONSTANT i32 0
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %3(s32)
-    $x11 = COPY %4(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %2(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_8_s64
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_8_s64
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[PseudoReadVLENB1:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB1]]
-    ; CHECK-NEXT: $x11 = COPY [[COPY]]
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s32) = G_READ_VLENB
-    %1:gprb(s32) = G_READ_VLENB
-    %2:gprb(s32) = G_CONSTANT i32 0
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %1(s32)
-    $x11 = COPY %2(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %0(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_16_s64
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_16_s64
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[SLLI:%[0-9]+]]:gpr = SLLI [[PseudoReadVLENB]], 1
-    ; CHECK-NEXT: [[PseudoReadVLENB1:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB1]]
-    ; CHECK-NEXT: $x11 = COPY [[COPY]]
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[SLLI]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s32) = G_READ_VLENB
-    %1:gprb(s32) = G_CONSTANT i32 1
-    %2:gprb(s32) = G_SHL %0, %1(s32)
-    %3:gprb(s32) = G_READ_VLENB
-    %4:gprb(s32) = G_CONSTANT i32 0
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %3(s32)
-    $x11 = COPY %4(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %2(s32)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_40_s64
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_40_s64
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 5
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB]]
-    ; CHECK-NEXT: $x11 = COPY [[ADDI]]
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
-    ; CHECK-NEXT: [[PseudoReadVLENB1:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x0
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB1]]
-    ; CHECK-NEXT: $x11 = COPY [[COPY1]]
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[COPY]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s32) = G_READ_VLENB
-    %1:gprb(s32) = G_CONSTANT i32 5
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %0(s32)
-    $x11 = COPY %1(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    %2:gprb(s32) = COPY $x10
-    %3:gprb(s32) = G_READ_VLENB
-    %4:gprb(s32) = G_CONSTANT i32 0
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %3(s32)
-    $x11 = COPY %4(s32)
-    PseudoCALL target-flags(riscv-call) &__mulsi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %2(s32)
-    PseudoRET implicit $x10
-
-...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale64.mir
deleted file mode 100644
index 3e480e5b8d73..000000000000
--- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale64.mir
+++ /dev/null
@@ -1,140 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=riscv64 -mattr=+v -run-pass=instruction-select \
-# RUN:   -simplify-mir -verify-machineinstrs %s -o - | FileCheck  %s
-
----
-name:            test_1
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_1
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 2
-    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s64) = G_READ_VLENB
-    %1:gprb(s64) = G_CONSTANT i64 2
-    %2:gprb(s64) = G_LSHR %0, %1(s64)
-    $x10 = COPY %2(s64)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_3
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_3
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
-    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 3
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
-    ; CHECK-NEXT: $x11 = COPY [[ADDI]]
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
-    ; CHECK-NEXT: $x10 = COPY [[COPY]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s64) = G_READ_VLENB
-    %1:gprb(s64) = G_CONSTANT i64 3
-    %2:gprb(s64) = G_LSHR %0, %1(s64)
-    %3:gprb(s64) = G_CONSTANT i64 3
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %2(s64)
-    $x11 = COPY %3(s64)
-    PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    %4:gprb(s64) = COPY $x10
-    $x10 = COPY %4(s64)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_4
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_4
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 1
-    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s64) = G_READ_VLENB
-    %1:gprb(s64) = G_CONSTANT i64 1
-    %2:gprb(s64) = G_LSHR %0, %1(s64)
-    $x10 = COPY %2(s64)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_8
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_8
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s64) = G_READ_VLENB
-    $x10 = COPY %0(s64)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_16
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_16
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[SLLI:%[0-9]+]]:gpr = SLLI [[PseudoReadVLENB]], 1
-    ; CHECK-NEXT: $x10 = COPY [[SLLI]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s64) = G_READ_VLENB
-    %1:gprb(s64) = G_CONSTANT i64 1
-    %2:gprb(s64) = G_SHL %0, %1(s64)
-    $x10 = COPY %2(s64)
-    PseudoRET implicit $x10
-
-...
----
-name:            test_40
-legalized:       true
-regBankSelected: true
-tracksRegLiveness: true
-body:             |
-  bb.0.entry:
-    ; CHECK-LABEL: name: test_40
-    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
-    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 5
-    ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB]]
-    ; CHECK-NEXT: $x11 = COPY [[ADDI]]
-    ; CHECK-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
-    ; CHECK-NEXT: $x10 = COPY [[COPY]]
-    ; CHECK-NEXT: PseudoRET implicit $x10
-    %0:gprb(s64) = G_READ_VLENB
-    %1:gprb(s64) = G_CONSTANT i64 5
-    ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
-    $x10 = COPY %0(s64)
-    $x11 = COPY %1(s64)
-    PseudoCALL target-flags(riscv-call) &__muldi3, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
-    ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2
-    %2:gprb(s64) = COPY $x10
-    $x10 = COPY %2(s64)
-    PseudoRET implicit $x10
-
-...
-- 
GitLab


From 4e6745cc4db309c0e1b5e41d4598f67763f4c096 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Mon, 25 Mar 2024 11:53:39 -0700
Subject: [PATCH 175/404] [flang][cuda] Lower simple host to device data
 transfer (#85960)

In CUDA Fortran data transfer can be done via assignment statements
between host and device variables.

This patch introduces a `fir.cuda_data_transfer` operation that
materialized the data transfer between two memory references.

Simple transfer not involving descriptors from host to device are also
lowered in this patch. When the rhs is an expression that required an
evaluation, a temporary is created. The evaluation is done on the host
and then the transfer is initiated.

Implicit transfer when device symbol are present on the rhs is not part
of this patch. Transfer from device to host is not part of this patch.
---
 .../flang/Optimizer/Dialect/FIRAttr.td        | 16 ++++++
 .../include/flang/Optimizer/Dialect/FIROps.td | 25 ++++++++
 flang/lib/Lower/Bridge.cpp                    | 40 +++++++++++--
 flang/lib/Optimizer/Dialect/FIRAttr.cpp       |  3 +-
 flang/test/Lower/CUDA/cuda-data-transfer.cuf  | 57 +++++++++++++++++++
 5 files changed, 134 insertions(+), 7 deletions(-)
 create mode 100644 flang/test/Lower/CUDA/cuda-data-transfer.cuf

diff --git a/flang/include/flang/Optimizer/Dialect/FIRAttr.td b/flang/include/flang/Optimizer/Dialect/FIRAttr.td
index 2ac4af9e66aa..f8b3fb861cc6 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRAttr.td
+++ b/flang/include/flang/Optimizer/Dialect/FIRAttr.td
@@ -137,4 +137,20 @@ def fir_CUDAClusterDimsAttr : fir_Attr<"CUDAClusterDims"> {
   let assemblyFormat = "`<` struct(params) `>`";
 }
 
+def fir_CUDADataTransferKind : I32EnumAttr<
+    "CUDADataTransferKind", "CUDA Fortran data transfer kind",
+    [
+      I32EnumAttrCase<"DeviceHost", 0, "device_host">,
+      I32EnumAttrCase<"HostDevice", 1, "host_device">,
+      I32EnumAttrCase<"DeviceDevice", 2, "device_device">,
+    ]> {
+  let genSpecializedAttr = 0;
+  let cppNamespace = "::fir";
+}
+
+def fir_CUDADataTransferKindAttr :
+    EnumAttr<FIROpsDialect, fir_CUDADataTransferKind, "cuda_transfer"> {
+  let assemblyFormat = [{ ```<` $value `>` }];
+}
+
 #endif // FIR_DIALECT_FIR_ATTRS
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index b991ec76fdd9..dff1cdb20cbf 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -3165,4 +3165,29 @@ def fir_CUDAKernelOp : fir_Op<"cuda_kernel", [AttrSizedOperandSegments,
   let hasVerifier = 1;
 }
 
+def fir_CUDADataTransferOp : fir_Op<"cuda_data_transfer", []> {
+  let summary = "Represent a data transfer between host and device memory";
+
+  let description = [{
+    CUDA Fortran allows data transfer to be done via intrinsic assignment
+    between a host and a device variable. This operation is used to materialized
+    the data transfer between the lhs and rhs memory references.
+    The kind of transfer is specified in the attribute. 
+
+    ```
+      adev = a ! transfer host to device
+      a = adev ! transfer device to host
+      bdev = adev ! transfer device to device
+    ```
+  }];
+
+  let arguments = (ins Arg<AnyReferenceLike, "", [MemWrite]>:$src,
+                       Arg<AnyReferenceLike, "", [MemRead]>:$dst,
+                       fir_CUDADataTransferKindAttr:$transfer_kind);
+
+  let assemblyFormat = [{
+    $src `to` $dst attr-dict `:` type(operands)
+  }];
+}
+
 #endif
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 0b54ee818e3c..48830dc55578 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -3706,15 +3706,39 @@ private:
     return false;
   }
 
+  static void genCUDADataTransfer(fir::FirOpBuilder &builder,
+                                  mlir::Location loc, bool lhsIsDevice,
+                                  hlfir::Entity &lhs, bool rhsIsDevice,
+                                  hlfir::Entity &rhs) {
+    if (rhs.isBoxAddressOrValue() || lhs.isBoxAddressOrValue())
+      TODO(loc, "CUDA data transfler with descriptors");
+    if (lhsIsDevice && !rhsIsDevice) {
+      auto transferKindAttr = fir::CUDADataTransferKindAttr::get(
+          builder.getContext(), fir::CUDADataTransferKind::HostDevice);
+      // device = host
+      if (!rhs.isVariable()) {
+        auto associate = hlfir::genAssociateExpr(
+            loc, builder, rhs, rhs.getType(), ".cuf_host_tmp");
+        builder.create<fir::CUDADataTransferOp>(loc, associate.getBase(), lhs,
+                                                transferKindAttr);
+        builder.create<hlfir::EndAssociateOp>(loc, associate);
+      } else {
+        builder.create<fir::CUDADataTransferOp>(loc, rhs, lhs,
+                                                transferKindAttr);
+      }
+      return;
+    }
+    TODO(loc, "Assignement with CUDA Fortran variables");
+  }
+
   void genDataAssignment(
       const Fortran::evaluate::Assignment &assign,
       const Fortran::evaluate::ProcedureRef *userDefinedAssignment) {
     mlir::Location loc = getCurrentLocation();
     fir::FirOpBuilder &builder = getFirOpBuilder();
 
-    if (Fortran::evaluate::HasCUDAAttrs(assign.lhs) ||
-        Fortran::evaluate::HasCUDAAttrs(assign.rhs))
-      TODO(loc, "Assignement with CUDA Fortran variables");
+    bool lhsIsDevice = Fortran::evaluate::HasCUDAAttrs(assign.lhs);
+    bool rhsIsDevice = Fortran::evaluate::HasCUDAAttrs(assign.rhs);
 
     // Gather some information about the assignment that will impact how it is
     // lowered.
@@ -3772,9 +3796,13 @@ private:
       Fortran::lower::StatementContext localStmtCtx;
       hlfir::Entity rhs = evaluateRhs(localStmtCtx);
       hlfir::Entity lhs = evaluateLhs(localStmtCtx);
-      builder.create<hlfir::AssignOp>(loc, rhs, lhs,
-                                      isWholeAllocatableAssignment,
-                                      keepLhsLengthInAllocatableAssignment);
+      if (lhsIsDevice || rhsIsDevice) {
+        genCUDADataTransfer(builder, loc, lhsIsDevice, lhs, rhsIsDevice, rhs);
+      } else {
+        builder.create<hlfir::AssignOp>(loc, rhs, lhs,
+                                        isWholeAllocatableAssignment,
+                                        keepLhsLengthInAllocatableAssignment);
+      }
       return;
     }
     // Assignments inside Forall, Where, or assignments to a vector subscripted
diff --git a/flang/lib/Optimizer/Dialect/FIRAttr.cpp b/flang/lib/Optimizer/Dialect/FIRAttr.cpp
index 0cf8dfb9f784..e43710f5627e 100644
--- a/flang/lib/Optimizer/Dialect/FIRAttr.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRAttr.cpp
@@ -299,5 +299,6 @@ void FIROpsDialect::registerAttributes() {
   addAttributes<ClosedIntervalAttr, ExactTypeAttr, FortranVariableFlagsAttr,
                 LowerBoundAttr, PointIntervalAttr, RealAttr, SubclassAttr,
                 UpperBoundAttr, CUDADataAttributeAttr, CUDAProcAttributeAttr,
-                CUDALaunchBoundsAttr, CUDAClusterDimsAttr>();
+                CUDALaunchBoundsAttr, CUDAClusterDimsAttr,
+                CUDADataTransferKindAttr>();
 }
diff --git a/flang/test/Lower/CUDA/cuda-data-transfer.cuf b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
new file mode 100644
index 000000000000..54226b8623e6
--- /dev/null
+++ b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
@@ -0,0 +1,57 @@
+! RUN: bbc -emit-hlfir -fcuda %s -o - | FileCheck %s
+
+! Test CUDA Fortran data transfer using assignment statements.
+
+subroutine sub1()
+  integer, device :: m
+  integer, device :: adev(10)
+  integer :: i, ahost(10), bhost(10)
+
+  m = 1 + i
+
+  m = 1
+
+  adev = ahost
+
+  adev = ahost + 1
+
+  adev(1:5) = ahost(1:5)
+
+  adev = ahost + bhost
+
+end
+
+! CHECK-LABEL: func.func @_QPsub1()
+
+! CHECK: %[[ADEV:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {cuda_attr = #fir.cuda<device>, uniq_name = "_QFsub1Eadev"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+! CHECK: %[[AHOST:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFsub1Eahost"} : (!fir.ref<!fir.array<10xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>)
+! CHECK: %[[I:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFsub1Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: %[[M:.*]]:2 = hlfir.declare %{{.*}} {cuda_attr = #fir.cuda<device>, uniq_name = "_QFsub1Em"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+
+! CHECK: %[[C1:.*]] = arith.constant 1 : i32
+! CHECK: %[[LOADED_I:.*]] = fir.load %[[I]]#0 : !fir.ref<i32>
+! CHECK: %[[ADD:.*]] = arith.addi %[[C1]], %[[LOADED_I]] : i32
+! CHECK: %[[ASSOC:.*]]:3 = hlfir.associate %[[ADD]] {uniq_name = ".cuf_host_tmp"} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
+! CHECK: fir.cuda_data_transfer %[[ASSOC]]#0 to %[[M]]#0 {transfer_kind = #fir.cuda_transfer<host_device>} : !fir.ref<i32>, !fir.ref<i32>
+! CHECK: hlfir.end_associate %[[ASSOC]]#1, %[[ASSOC]]#2 : !fir.ref<i32>, i1
+
+! CHECK: %[[C1:.*]] = arith.constant 1 : i32
+! CHECK: %[[ASSOC:.*]]:3 = hlfir.associate %[[C1]] {uniq_name = ".cuf_host_tmp"} : (i32) -> (!fir.ref<i32>, !fir.ref<i32>, i1)
+! CHECK: fir.cuda_data_transfer %[[ASSOC]]#0 to %[[M]]#0 {transfer_kind = #fir.cuda_transfer<host_device>} : !fir.ref<i32>, !fir.ref<i32>
+! CHECK: hlfir.end_associate %[[ASSOC]]#1, %[[ASSOC]]#2 : !fir.ref<i32>, i1
+
+! CHECK: fir.cuda_data_transfer %[[AHOST]]#0 to %[[ADEV]]#0 {transfer_kind = #fir.cuda_transfer<host_device>} : !fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>
+
+! CHECK: %[[ELEMENTAL:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr<10xi32> {
+! CHECK: %[[ASSOC:.*]]:3 = hlfir.associate %[[ELEMENTAL]](%{{.*}}) {uniq_name = ".cuf_host_tmp"} : (!hlfir.expr<10xi32>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>, i1)
+! CHECK: fir.cuda_data_transfer %[[ASSOC]]#0 to %[[ADEV]]#0 {transfer_kind = #fir.cuda_transfer<host_device>} : !fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>
+! CHECK: hlfir.end_associate %[[ASSOC]]#1, %[[ASSOC]]#2 : !fir.ref<!fir.array<10xi32>>, i1
+
+! CHECK: %[[DES_AHOST:.*]] = hlfir.designate %[[AHOST]]#0 (%c1{{.*}}:%c5{{.*}}:%c1{{.*}})  shape %{{.*}} : (!fir.ref<!fir.array<10xi32>>, index, index, index, !fir.shape<1>) -> !fir.ref<!fir.array<5xi32>>
+! CHECK: %[[DES_ADEV:.*]] = hlfir.designate %[[ADEV]]#0 (%c1{{.*}}:%c5{{.*}}:%c1{{.*}})  shape %{{.*}} : (!fir.ref<!fir.array<10xi32>>, index, index, index, !fir.shape<1>) -> !fir.ref<!fir.array<5xi32>>
+! CHECK: fir.cuda_data_transfer %[[DES_AHOST]] to %[[DES_ADEV]] {transfer_kind = #fir.cuda_transfer<host_device>} : !fir.ref<!fir.array<5xi32>>, !fir.ref<!fir.array<5xi32>>
+
+! CHECK: %[[ELEMENTAL:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<1>) -> !hlfir.expr<10xi32>
+! CHECK: %[[ASSOC:.*]]:3 = hlfir.associate %[[ELEMENTAL]](%{{.*}}) {uniq_name = ".cuf_host_tmp"} : (!hlfir.expr<10xi32>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>, i1)
+! CHECK: fir.cuda_data_transfer %[[ASSOC]]#0 to %[[ADEV]]#0 {transfer_kind = #fir.cuda_transfer<host_device>} : !fir.ref<!fir.array<10xi32>>, !fir.ref<!fir.array<10xi32>>
+! CHECK: hlfir.end_associate %[[ASSOC]]#1, %[[ASSOC]]#2 : !fir.ref<!fir.array<10xi32>>, i1
-- 
GitLab


From 3e3f0c3175a8b83a8c51bef5712dbc384385db8b Mon Sep 17 00:00:00 2001
From: Job Henandez Lara <hj93@protonmail.com>
Date: Mon, 25 Mar 2024 12:19:10 -0700
Subject: [PATCH 176/404] [libc][math][c23] add c23 floating point fmaximum and
 fminimum functions. (#86016)

Fixes https://github.com/llvm/llvm-project/issues/85496.

---------

Co-authored-by: Job Hernandez <h93@protonmail.com>
---
 libc/config/linux/aarch64/entrypoints.txt     |  32 ++
 libc/config/linux/arm/entrypoints.txt         |  24 +
 libc/config/linux/riscv/entrypoints.txt       |  32 ++
 libc/config/linux/x86_64/entrypoints.txt      |  32 ++
 libc/config/windows/entrypoints.txt           |  24 +
 libc/spec/stdc.td                             |  40 ++
 libc/src/__support/FPUtil/BasicOperations.h   | 105 +++++
 libc/src/math/CMakeLists.txt                  |  40 ++
 libc/src/math/fmaximum.h                      |  19 +
 libc/src/math/fmaximum_mag.h                  |  19 +
 libc/src/math/fmaximum_mag_num.h              |  18 +
 libc/src/math/fmaximum_mag_numf.h             |  19 +
 libc/src/math/fmaximum_mag_numf128.h          |  21 +
 libc/src/math/fmaximum_mag_numl.h             |  19 +
 libc/src/math/fmaximum_magf.h                 |  19 +
 libc/src/math/fmaximum_magf128.h              |  21 +
 libc/src/math/fmaximum_magl.h                 |  19 +
 libc/src/math/fmaximum_num.h                  |  19 +
 libc/src/math/fmaximum_numf.h                 |  19 +
 libc/src/math/fmaximum_numf128.h              |  21 +
 libc/src/math/fmaximum_numl.h                 |  19 +
 libc/src/math/fmaximumf.h                     |  19 +
 libc/src/math/fmaximumf128.h                  |  21 +
 libc/src/math/fmaximuml.h                     |  19 +
 libc/src/math/fminimum.h                      |  19 +
 libc/src/math/fminimum_mag.h                  |  19 +
 libc/src/math/fminimum_mag_num.h              |  19 +
 libc/src/math/fminimum_mag_numf.h             |  19 +
 libc/src/math/fminimum_mag_numf128.h          |  21 +
 libc/src/math/fminimum_mag_numl.h             |  19 +
 libc/src/math/fminimum_magf.h                 |  19 +
 libc/src/math/fminimum_magf128.h              |  21 +
 libc/src/math/fminimum_magl.h                 |  19 +
 libc/src/math/fminimum_num.h                  |  19 +
 libc/src/math/fminimum_numf.h                 |  19 +
 libc/src/math/fminimum_numf128.h              |  21 +
 libc/src/math/fminimum_numl.h                 |  19 +
 libc/src/math/fminimumf.h                     |  19 +
 libc/src/math/fminimumf128.h                  |  21 +
 libc/src/math/fminimuml.h                     |  19 +
 libc/src/math/generic/CMakeLists.txt          | 395 +++++++++++++++++
 libc/src/math/generic/fmaximum.cpp            |  19 +
 libc/src/math/generic/fmaximum_mag.cpp        |  19 +
 libc/src/math/generic/fmaximum_mag_num.cpp    |  19 +
 libc/src/math/generic/fmaximum_mag_numf.cpp   |  19 +
 .../src/math/generic/fmaximum_mag_numf128.cpp |  19 +
 libc/src/math/generic/fmaximum_mag_numl.cpp   |  20 +
 libc/src/math/generic/fmaximum_magf.cpp       |  19 +
 libc/src/math/generic/fmaximum_magf128.cpp    |  19 +
 libc/src/math/generic/fmaximum_magl.cpp       |  19 +
 libc/src/math/generic/fmaximum_num.cpp        |  19 +
 libc/src/math/generic/fmaximum_numf.cpp       |  19 +
 libc/src/math/generic/fmaximum_numf128.cpp    |  19 +
 libc/src/math/generic/fmaximum_numl.cpp       |  19 +
 libc/src/math/generic/fmaximumf.cpp           |  19 +
 libc/src/math/generic/fmaximumf128.cpp        |  19 +
 libc/src/math/generic/fmaximuml.cpp           |  19 +
 libc/src/math/generic/fminimum.cpp            |  19 +
 libc/src/math/generic/fminimum_mag.cpp        |  19 +
 libc/src/math/generic/fminimum_mag_num.cpp    |  19 +
 libc/src/math/generic/fminimum_mag_numf.cpp   |  19 +
 .../src/math/generic/fminimum_mag_numf128.cpp |  19 +
 libc/src/math/generic/fminimum_mag_numl.cpp   |  20 +
 libc/src/math/generic/fminimum_magf.cpp       |  19 +
 libc/src/math/generic/fminimum_magf128.cpp    |  19 +
 libc/src/math/generic/fminimum_magl.cpp       |  19 +
 libc/src/math/generic/fminimum_num.cpp        |  19 +
 libc/src/math/generic/fminimum_numf.cpp       |  19 +
 libc/src/math/generic/fminimum_numf128.cpp    |  19 +
 libc/src/math/generic/fminimum_numl.cpp       |  19 +
 libc/src/math/generic/fminimumf.cpp           |  19 +
 libc/src/math/generic/fminimumf128.cpp        |  19 +
 libc/src/math/generic/fminimuml.cpp           |  19 +
 libc/test/src/math/smoke/CMakeLists.txt       | 418 ++++++++++++++++++
 libc/test/src/math/smoke/FMaximumMagNumTest.h | 101 +++++
 libc/test/src/math/smoke/FMaximumMagTest.h    |  89 ++++
 libc/test/src/math/smoke/FMaximumNumTest.h    | 100 +++++
 libc/test/src/math/smoke/FMaximumTest.h       |  88 ++++
 libc/test/src/math/smoke/FMinimumMagNumTest.h | 101 +++++
 libc/test/src/math/smoke/FMinimumMagTest.h    |  89 ++++
 libc/test/src/math/smoke/FMinimumNumTest.h    | 100 +++++
 libc/test/src/math/smoke/FMinimumTest.h       |  88 ++++
 .../src/math/smoke/fmaximum_mag_num_test.cpp  |  13 +
 .../math/smoke/fmaximum_mag_numf128_test.cpp  |  13 +
 .../src/math/smoke/fmaximum_mag_numf_test.cpp |  13 +
 .../src/math/smoke/fmaximum_mag_numl_test.cpp |  13 +
 .../test/src/math/smoke/fmaximum_mag_test.cpp |  13 +
 .../src/math/smoke/fmaximum_magf128_test.cpp  |  13 +
 .../src/math/smoke/fmaximum_magf_test.cpp     |  13 +
 .../src/math/smoke/fmaximum_magl_test.cpp     |  13 +
 .../test/src/math/smoke/fmaximum_num_test.cpp |  13 +
 .../src/math/smoke/fmaximum_numf128_test.cpp  |  13 +
 .../src/math/smoke/fmaximum_numf_test.cpp     |  13 +
 .../src/math/smoke/fmaximum_numl_test.cpp     |  13 +
 libc/test/src/math/smoke/fmaximum_test.cpp    |  13 +
 .../test/src/math/smoke/fmaximumf128_test.cpp |  13 +
 libc/test/src/math/smoke/fmaximumf_test.cpp   |  13 +
 libc/test/src/math/smoke/fmaximuml_test.cpp   |  13 +
 .../src/math/smoke/fminimum_mag_num_test.cpp  |  13 +
 .../math/smoke/fminimum_mag_numf128_test.cpp  |  13 +
 .../src/math/smoke/fminimum_mag_numf_test.cpp |  13 +
 .../src/math/smoke/fminimum_mag_numl_test.cpp |  13 +
 .../test/src/math/smoke/fminimum_mag_test.cpp |  13 +
 .../src/math/smoke/fminimum_magf128_test.cpp  |  13 +
 .../src/math/smoke/fminimum_magf_test.cpp     |  13 +
 .../src/math/smoke/fminimum_magl_test.cpp     |  13 +
 .../test/src/math/smoke/fminimum_num_test.cpp |  13 +
 .../src/math/smoke/fminimum_numf128_test.cpp  |  13 +
 .../src/math/smoke/fminimum_numf_test.cpp     |  13 +
 .../src/math/smoke/fminimum_numl_test.cpp     |  13 +
 libc/test/src/math/smoke/fminimum_test.cpp    |  13 +
 .../test/src/math/smoke/fminimumf128_test.cpp |  13 +
 libc/test/src/math/smoke/fminimumf_test.cpp   |  13 +
 libc/test/src/math/smoke/fminimuml_test.cpp   |  13 +
 114 files changed, 3547 insertions(+)
 create mode 100644 libc/src/math/fmaximum.h
 create mode 100644 libc/src/math/fmaximum_mag.h
 create mode 100644 libc/src/math/fmaximum_mag_num.h
 create mode 100644 libc/src/math/fmaximum_mag_numf.h
 create mode 100644 libc/src/math/fmaximum_mag_numf128.h
 create mode 100644 libc/src/math/fmaximum_mag_numl.h
 create mode 100644 libc/src/math/fmaximum_magf.h
 create mode 100644 libc/src/math/fmaximum_magf128.h
 create mode 100644 libc/src/math/fmaximum_magl.h
 create mode 100644 libc/src/math/fmaximum_num.h
 create mode 100644 libc/src/math/fmaximum_numf.h
 create mode 100644 libc/src/math/fmaximum_numf128.h
 create mode 100644 libc/src/math/fmaximum_numl.h
 create mode 100644 libc/src/math/fmaximumf.h
 create mode 100644 libc/src/math/fmaximumf128.h
 create mode 100644 libc/src/math/fmaximuml.h
 create mode 100644 libc/src/math/fminimum.h
 create mode 100644 libc/src/math/fminimum_mag.h
 create mode 100644 libc/src/math/fminimum_mag_num.h
 create mode 100644 libc/src/math/fminimum_mag_numf.h
 create mode 100644 libc/src/math/fminimum_mag_numf128.h
 create mode 100644 libc/src/math/fminimum_mag_numl.h
 create mode 100644 libc/src/math/fminimum_magf.h
 create mode 100644 libc/src/math/fminimum_magf128.h
 create mode 100644 libc/src/math/fminimum_magl.h
 create mode 100644 libc/src/math/fminimum_num.h
 create mode 100644 libc/src/math/fminimum_numf.h
 create mode 100644 libc/src/math/fminimum_numf128.h
 create mode 100644 libc/src/math/fminimum_numl.h
 create mode 100644 libc/src/math/fminimumf.h
 create mode 100644 libc/src/math/fminimumf128.h
 create mode 100644 libc/src/math/fminimuml.h
 create mode 100644 libc/src/math/generic/fmaximum.cpp
 create mode 100644 libc/src/math/generic/fmaximum_mag.cpp
 create mode 100644 libc/src/math/generic/fmaximum_mag_num.cpp
 create mode 100644 libc/src/math/generic/fmaximum_mag_numf.cpp
 create mode 100644 libc/src/math/generic/fmaximum_mag_numf128.cpp
 create mode 100644 libc/src/math/generic/fmaximum_mag_numl.cpp
 create mode 100644 libc/src/math/generic/fmaximum_magf.cpp
 create mode 100644 libc/src/math/generic/fmaximum_magf128.cpp
 create mode 100644 libc/src/math/generic/fmaximum_magl.cpp
 create mode 100644 libc/src/math/generic/fmaximum_num.cpp
 create mode 100644 libc/src/math/generic/fmaximum_numf.cpp
 create mode 100644 libc/src/math/generic/fmaximum_numf128.cpp
 create mode 100644 libc/src/math/generic/fmaximum_numl.cpp
 create mode 100644 libc/src/math/generic/fmaximumf.cpp
 create mode 100644 libc/src/math/generic/fmaximumf128.cpp
 create mode 100644 libc/src/math/generic/fmaximuml.cpp
 create mode 100644 libc/src/math/generic/fminimum.cpp
 create mode 100644 libc/src/math/generic/fminimum_mag.cpp
 create mode 100644 libc/src/math/generic/fminimum_mag_num.cpp
 create mode 100644 libc/src/math/generic/fminimum_mag_numf.cpp
 create mode 100644 libc/src/math/generic/fminimum_mag_numf128.cpp
 create mode 100644 libc/src/math/generic/fminimum_mag_numl.cpp
 create mode 100644 libc/src/math/generic/fminimum_magf.cpp
 create mode 100644 libc/src/math/generic/fminimum_magf128.cpp
 create mode 100644 libc/src/math/generic/fminimum_magl.cpp
 create mode 100644 libc/src/math/generic/fminimum_num.cpp
 create mode 100644 libc/src/math/generic/fminimum_numf.cpp
 create mode 100644 libc/src/math/generic/fminimum_numf128.cpp
 create mode 100644 libc/src/math/generic/fminimum_numl.cpp
 create mode 100644 libc/src/math/generic/fminimumf.cpp
 create mode 100644 libc/src/math/generic/fminimumf128.cpp
 create mode 100644 libc/src/math/generic/fminimuml.cpp
 create mode 100644 libc/test/src/math/smoke/FMaximumMagNumTest.h
 create mode 100644 libc/test/src/math/smoke/FMaximumMagTest.h
 create mode 100644 libc/test/src/math/smoke/FMaximumNumTest.h
 create mode 100644 libc/test/src/math/smoke/FMaximumTest.h
 create mode 100644 libc/test/src/math/smoke/FMinimumMagNumTest.h
 create mode 100644 libc/test/src/math/smoke/FMinimumMagTest.h
 create mode 100644 libc/test/src/math/smoke/FMinimumNumTest.h
 create mode 100644 libc/test/src/math/smoke/FMinimumTest.h
 create mode 100644 libc/test/src/math/smoke/fmaximum_mag_num_test.cpp
 create mode 100644 libc/test/src/math/smoke/fmaximum_mag_numf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/fmaximum_mag_numf_test.cpp
 create mode 100644 libc/test/src/math/smoke/fmaximum_mag_numl_test.cpp
 create mode 100644 libc/test/src/math/smoke/fmaximum_mag_test.cpp
 create mode 100644 libc/test/src/math/smoke/fmaximum_magf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/fmaximum_magf_test.cpp
 create mode 100644 libc/test/src/math/smoke/fmaximum_magl_test.cpp
 create mode 100644 libc/test/src/math/smoke/fmaximum_num_test.cpp
 create mode 100644 libc/test/src/math/smoke/fmaximum_numf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/fmaximum_numf_test.cpp
 create mode 100644 libc/test/src/math/smoke/fmaximum_numl_test.cpp
 create mode 100644 libc/test/src/math/smoke/fmaximum_test.cpp
 create mode 100644 libc/test/src/math/smoke/fmaximumf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/fmaximumf_test.cpp
 create mode 100644 libc/test/src/math/smoke/fmaximuml_test.cpp
 create mode 100644 libc/test/src/math/smoke/fminimum_mag_num_test.cpp
 create mode 100644 libc/test/src/math/smoke/fminimum_mag_numf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/fminimum_mag_numf_test.cpp
 create mode 100644 libc/test/src/math/smoke/fminimum_mag_numl_test.cpp
 create mode 100644 libc/test/src/math/smoke/fminimum_mag_test.cpp
 create mode 100644 libc/test/src/math/smoke/fminimum_magf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/fminimum_magf_test.cpp
 create mode 100644 libc/test/src/math/smoke/fminimum_magl_test.cpp
 create mode 100644 libc/test/src/math/smoke/fminimum_num_test.cpp
 create mode 100644 libc/test/src/math/smoke/fminimum_numf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/fminimum_numf_test.cpp
 create mode 100644 libc/test/src/math/smoke/fminimum_numl_test.cpp
 create mode 100644 libc/test/src/math/smoke/fminimum_test.cpp
 create mode 100644 libc/test/src/math/smoke/fminimumf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/fminimumf_test.cpp
 create mode 100644 libc/test/src/math/smoke/fminimuml_test.cpp

diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 6abb35ab0ead..78da7f0b334b 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -366,6 +366,30 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fmin
     libc.src.math.fminf
     libc.src.math.fminl
+    libc.src.math.fmaximum
+    libc.src.math.fmaximumf
+    libc.src.math.fmaximuml
+    libc.src.math.fmaximum_num
+    libc.src.math.fmaximum_numf
+    libc.src.math.fmaximum_numl
+    libc.src.math.fmaximum_mag
+    libc.src.math.fmaximum_magf
+    libc.src.math.fmaximum_magl
+    libc.src.math.fmaximum_mag_num
+    libc.src.math.fmaximum_mag_numf
+    libc.src.math.fmaximum_mag_numl
+    libc.src.math.fminimum
+    libc.src.math.fminimumf
+    libc.src.math.fminimuml
+    libc.src.math.fminimum_num
+    libc.src.math.fminimum_numf
+    libc.src.math.fminimum_numl
+    libc.src.math.fminimum_mag
+    libc.src.math.fminimum_magf
+    libc.src.math.fminimum_magl
+    libc.src.math.fminimum_mag_num
+    libc.src.math.fminimum_mag_numf
+    libc.src.math.fminimum_mag_numl
     libc.src.math.fmod
     libc.src.math.fmodf
     libc.src.math.fmodl
@@ -466,6 +490,14 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.floorf128
     libc.src.math.fmaxf128
     libc.src.math.fminf128
+    libc.src.math.fmaximumf128
+    libc.src.math.fmaximum_numf128
+    libc.src.math.fmaximum_magf128
+    libc.src.math.fmaximum_mag_numf128
+    libc.src.math.fminimumf128
+    libc.src.math.fminimum_numf128
+    libc.src.math.fminimum_magf128
+    libc.src.math.fminimum_mag_numf128
     libc.src.math.fmodf128
     libc.src.math.frexpf128
     libc.src.math.ilogbf128
diff --git a/libc/config/linux/arm/entrypoints.txt b/libc/config/linux/arm/entrypoints.txt
index 3bc5d8efc9d2..6e63e270280e 100644
--- a/libc/config/linux/arm/entrypoints.txt
+++ b/libc/config/linux/arm/entrypoints.txt
@@ -234,6 +234,30 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fmin
     libc.src.math.fminf
     libc.src.math.fminl
+    libc.src.math.fmaximum
+    libc.src.math.fmaximumf
+    libc.src.math.fmaximuml
+    libc.src.math.fmaximum_num
+    libc.src.math.fmaximum_numf
+    libc.src.math.fmaximum_numl
+    libc.src.math.fmaximum_mag
+    libc.src.math.fmaximum_magf
+    libc.src.math.fmaximum_magl
+    libc.src.math.fmaximum_mag_num
+    libc.src.math.fmaximum_mag_numf
+    libc.src.math.fmaximum_mag_numl
+    libc.src.math.fminimum
+    libc.src.math.fminimumf
+    libc.src.math.fminimuml
+    libc.src.math.fminimum_num
+    libc.src.math.fminimum_numf
+    libc.src.math.fminimum_numl
+    libc.src.math.fminimum_mag
+    libc.src.math.fminimum_magf
+    libc.src.math.fminimum_magl
+    libc.src.math.fminimum_mag_num
+    libc.src.math.fminimum_mag_numf
+    libc.src.math.fminimum_mag_numl
     libc.src.math.fmod
     libc.src.math.fmodf
     libc.src.math.frexp
diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index e34c87ec6b5d..5aae4e246cfb 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -374,6 +374,30 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fmax
     libc.src.math.fmaxf
     libc.src.math.fmaxl
+    libc.src.math.fmaximum
+    libc.src.math.fmaximumf
+    libc.src.math.fmaximuml
+    libc.src.math.fmaximum_num
+    libc.src.math.fmaximum_numf
+    libc.src.math.fmaximum_numl
+    libc.src.math.fmaximum_mag
+    libc.src.math.fmaximum_magf
+    libc.src.math.fmaximum_magl
+    libc.src.math.fmaximum_mag_num
+    libc.src.math.fmaximum_mag_numf
+    libc.src.math.fmaximum_mag_numl
+    libc.src.math.fminimum
+    libc.src.math.fminimumf
+    libc.src.math.fminimuml
+    libc.src.math.fminimum_num
+    libc.src.math.fminimum_numf
+    libc.src.math.fminimum_numl
+    libc.src.math.fminimum_mag
+    libc.src.math.fminimum_magf
+    libc.src.math.fminimum_magl
+    libc.src.math.fminimum_mag_num
+    libc.src.math.fminimum_mag_numf
+    libc.src.math.fminimum_mag_numl
     libc.src.math.fmod
     libc.src.math.fmodf
     libc.src.math.fmodl
@@ -474,6 +498,14 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.floorf128
     libc.src.math.fmaxf128
     libc.src.math.fminf128
+    libc.src.math.fmaximumf128
+    libc.src.math.fmaximum_numf128
+    libc.src.math.fmaximum_magf128
+    libc.src.math.fmaximum_mag_numf128
+    libc.src.math.fminimumf128
+    libc.src.math.fminimum_numf128
+    libc.src.math.fminimum_magf128
+    libc.src.math.fminimum_mag_numf128
     libc.src.math.fmodf128
     libc.src.math.frexpf128
     libc.src.math.ilogbf128
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 2f5a3bffadc0..29080df54922 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -385,6 +385,30 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fmax
     libc.src.math.fmaxf
     libc.src.math.fmaxl
+    libc.src.math.fmaximum
+    libc.src.math.fmaximumf
+    libc.src.math.fmaximuml
+    libc.src.math.fmaximum_num
+    libc.src.math.fmaximum_numf
+    libc.src.math.fmaximum_numl
+    libc.src.math.fmaximum_mag
+    libc.src.math.fmaximum_magf
+    libc.src.math.fmaximum_magl
+    libc.src.math.fmaximum_mag_num
+    libc.src.math.fmaximum_mag_numf
+    libc.src.math.fmaximum_mag_numl
+    libc.src.math.fminimum
+    libc.src.math.fminimumf
+    libc.src.math.fminimuml
+    libc.src.math.fminimum_num
+    libc.src.math.fminimum_numf
+    libc.src.math.fminimum_numl
+    libc.src.math.fminimum_mag
+    libc.src.math.fminimum_magf
+    libc.src.math.fminimum_magl
+    libc.src.math.fminimum_mag_num
+    libc.src.math.fminimum_mag_numf
+    libc.src.math.fminimum_mag_numl
     libc.src.math.fmod
     libc.src.math.fmodf
     libc.src.math.fmodl
@@ -499,6 +523,14 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.floorf128
     libc.src.math.fmaxf128
     libc.src.math.fminf128
+    libc.src.math.fmaximumf128
+    libc.src.math.fmaximum_numf128
+    libc.src.math.fmaximum_magf128
+    libc.src.math.fmaximum_mag_numf128
+    libc.src.math.fminimumf128
+    libc.src.math.fminimum_numf128
+    libc.src.math.fminimum_magf128
+    libc.src.math.fminimum_mag_numf128
     libc.src.math.fmodf128
     libc.src.math.frexpf128
     libc.src.math.fromfpf128
diff --git a/libc/config/windows/entrypoints.txt b/libc/config/windows/entrypoints.txt
index d6227a427afe..f4456f561ec0 100644
--- a/libc/config/windows/entrypoints.txt
+++ b/libc/config/windows/entrypoints.txt
@@ -153,6 +153,30 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.fmax
     libc.src.math.fmaxf
     libc.src.math.fmaxl
+    libc.src.math.fmaximum
+    libc.src.math.fmaximumf
+    libc.src.math.fmaximuml
+    libc.src.math.fmaximum_num
+    libc.src.math.fmaximum_numf
+    libc.src.math.fmaximum_numl
+    libc.src.math.fmaximum_mag
+    libc.src.math.fmaximum_magf
+    libc.src.math.fmaximum_magl
+    libc.src.math.fmaximum_mag_num
+    libc.src.math.fmaximum_mag_numf
+    libc.src.math.fmaximum_mag_numl
+    libc.src.math.fminimum
+    libc.src.math.fminimumf
+    libc.src.math.fminimuml
+    libc.src.math.fminimum_num
+    libc.src.math.fminimum_numf
+    libc.src.math.fminimum_numl
+    libc.src.math.fminimum_mag
+    libc.src.math.fminimum_magf
+    libc.src.math.fminimum_magl
+    libc.src.math.fminimum_mag_num
+    libc.src.math.fminimum_mag_numf
+    libc.src.math.fminimum_mag_numl
     libc.src.math.fmod
     libc.src.math.fmodf
     libc.src.math.fmodl
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index 7407ead17711..89b43f06778d 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -406,6 +406,46 @@ def StdC : StandardSpec<"stdc"> {
           FunctionSpec<"fmaxf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
           FunctionSpec<"fmaxl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
           GuardedFunctionSpec<"fmaxf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
+	  
+	  FunctionSpec<"fmaximum", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
+          FunctionSpec<"fmaximumf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
+          FunctionSpec<"fmaximuml", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"fmaximumf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
+	  
+	  FunctionSpec<"fmaximum_num", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
+          FunctionSpec<"fmaximum_numf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
+          FunctionSpec<"fmaximum_numl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"fmaximum_numf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
+	  
+	  FunctionSpec<"fmaximum_mag", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
+          FunctionSpec<"fmaximum_magf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
+          FunctionSpec<"fmaximum_magl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"fmaximum_magf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
+	  
+	  FunctionSpec<"fmaximum_mag_num", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
+          FunctionSpec<"fmaximum_mag_numf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
+          FunctionSpec<"fmaximum_mag_numl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"fmaximum_mag_numf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
+	  
+	  FunctionSpec<"fminimum", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
+          FunctionSpec<"fminimumf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
+          FunctionSpec<"fminimuml", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"fminimumf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
+	  
+	  FunctionSpec<"fminimum_num", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
+          FunctionSpec<"fminimum_numf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
+          FunctionSpec<"fmaximum_numl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"fminimum_numf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
+	  
+	  FunctionSpec<"fminimum_mag", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
+          FunctionSpec<"fminimum_magf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
+          FunctionSpec<"fminimum_magl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"fminimum_magf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
+	  
+	  FunctionSpec<"fminimum_mag_num", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
+          FunctionSpec<"fminimum_mag_numf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
+          FunctionSpec<"fminimum_mag_numl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"fminimum_mag_numf128", RetValSpec<Float128Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"fma", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"fmaf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>, ArgSpec<FloatType>]>,
diff --git a/libc/src/__support/FPUtil/BasicOperations.h b/libc/src/__support/FPUtil/BasicOperations.h
index a19d6d0bef08..b8cff982d3a6 100644
--- a/libc/src/__support/FPUtil/BasicOperations.h
+++ b/libc/src/__support/FPUtil/BasicOperations.h
@@ -11,6 +11,7 @@
 
 #include "FPBits.h"
 
+#include "FEnvImpl.h"
 #include "src/__support/CPP/type_traits.h"
 #include "src/__support/common.h"
 
@@ -58,6 +59,110 @@ LIBC_INLINE T fmax(T x, T y) {
   }
 }
 
+template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0>
+LIBC_INLINE T fmaximum(T x, T y) {
+  FPBits<T> bitx(x), bity(y);
+
+  if (bitx.is_nan())
+    return x;
+  if (bity.is_nan())
+    return y;
+  if (bitx.sign() != bity.sign())
+    return (bitx.is_neg() ? y : x);
+  return x > y ? x : y;
+}
+
+template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0>
+LIBC_INLINE T fminimum(T x, T y) {
+  const FPBits<T> bitx(x), bity(y);
+
+  if (bitx.is_nan())
+    return x;
+  if (bity.is_nan())
+    return y;
+  if (bitx.sign() != bity.sign())
+    return (bitx.is_neg()) ? x : y;
+  return x < y ? x : y;
+}
+
+template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0>
+LIBC_INLINE T fmaximum_num(T x, T y) {
+  FPBits<T> bitx(x), bity(y);
+  if (bitx.is_signaling_nan() || bity.is_signaling_nan()) {
+    fputil::raise_except_if_required(FE_INVALID);
+    if (bitx.is_nan() && bity.is_nan())
+      return FPBits<T>::quiet_nan().get_val();
+  }
+  if (bitx.is_nan())
+    return y;
+  if (bity.is_nan())
+    return x;
+  if (bitx.sign() != bity.sign())
+    return (bitx.is_neg() ? y : x);
+  return x > y ? x : y;
+}
+
+template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0>
+LIBC_INLINE T fminimum_num(T x, T y) {
+  FPBits<T> bitx(x), bity(y);
+  if (bitx.is_signaling_nan() || bity.is_signaling_nan()) {
+    fputil::raise_except_if_required(FE_INVALID);
+    if (bitx.is_nan() && bity.is_nan())
+      return FPBits<T>::quiet_nan().get_val();
+  }
+  if (bitx.is_nan())
+    return y;
+  if (bity.is_nan())
+    return x;
+  if (bitx.sign() != bity.sign())
+    return (bitx.is_neg() ? x : y);
+  return x < y ? x : y;
+}
+
+template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0>
+LIBC_INLINE T fmaximum_mag(T x, T y) {
+  FPBits<T> bitx(x), bity(y);
+
+  if (abs(x) > abs(y))
+    return x;
+  if (abs(y) > abs(x))
+    return y;
+  return fmaximum(x, y);
+}
+
+template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0>
+LIBC_INLINE T fminimum_mag(T x, T y) {
+  FPBits<T> bitx(x), bity(y);
+
+  if (abs(x) < abs(y))
+    return x;
+  if (abs(y) < abs(x))
+    return y;
+  return fminimum(x, y);
+}
+
+template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0>
+LIBC_INLINE T fmaximum_mag_num(T x, T y) {
+  FPBits<T> bitx(x), bity(y);
+
+  if (abs(x) > abs(y))
+    return x;
+  if (abs(y) > abs(x))
+    return y;
+  return fmaximum_num(x, y);
+}
+
+template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0>
+LIBC_INLINE T fminimum_mag_num(T x, T y) {
+  FPBits<T> bitx(x), bity(y);
+
+  if (abs(x) < abs(y))
+    return x;
+  if (abs(y) < abs(x))
+    return y;
+  return fminimum_num(x, y);
+}
+
 template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0>
 LIBC_INLINE T fdim(T x, T y) {
   FPBits<T> bitx(x), bity(y);
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index 2f1d65780013..61baa498321c 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -117,6 +117,46 @@ add_math_entrypoint_object(fminf)
 add_math_entrypoint_object(fminl)
 add_math_entrypoint_object(fminf128)
 
+add_math_entrypoint_object(fmaximum)
+add_math_entrypoint_object(fmaximumf)
+add_math_entrypoint_object(fmaximuml)
+add_math_entrypoint_object(fmaximumf128)
+
+add_math_entrypoint_object(fmaximum_num)
+add_math_entrypoint_object(fmaximum_numf)
+add_math_entrypoint_object(fmaximum_numl)
+add_math_entrypoint_object(fmaximum_numf128)
+
+add_math_entrypoint_object(fmaximum_mag)
+add_math_entrypoint_object(fmaximum_magf)
+add_math_entrypoint_object(fmaximum_magl)
+add_math_entrypoint_object(fmaximum_magf128)
+
+add_math_entrypoint_object(fmaximum_mag_num)
+add_math_entrypoint_object(fmaximum_mag_numf)
+add_math_entrypoint_object(fmaximum_mag_numl)
+add_math_entrypoint_object(fmaximum_mag_numf128)
+
+add_math_entrypoint_object(fminimum)
+add_math_entrypoint_object(fminimumf)
+add_math_entrypoint_object(fminimuml)
+add_math_entrypoint_object(fminimumf128)
+
+add_math_entrypoint_object(fminimum_num)
+add_math_entrypoint_object(fminimum_numf)
+add_math_entrypoint_object(fminimum_numl)
+add_math_entrypoint_object(fminimum_numf128)
+
+add_math_entrypoint_object(fminimum_mag)
+add_math_entrypoint_object(fminimum_magf)
+add_math_entrypoint_object(fminimum_magl)
+add_math_entrypoint_object(fminimum_magf128)
+
+add_math_entrypoint_object(fminimum_mag_num)
+add_math_entrypoint_object(fminimum_mag_numf)
+add_math_entrypoint_object(fminimum_mag_numl)
+add_math_entrypoint_object(fminimum_mag_numf128)
+
 add_math_entrypoint_object(fmod)
 add_math_entrypoint_object(fmodf)
 add_math_entrypoint_object(fmodl)
diff --git a/libc/src/math/fmaximum.h b/libc/src/math/fmaximum.h
new file mode 100644
index 000000000000..8eac02b17b79
--- /dev/null
+++ b/libc/src/math/fmaximum.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fmaximum --------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUM_H
+
+namespace LIBC_NAMESPACE {
+
+double fmaximum(double x, double y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_H
diff --git a/libc/src/math/fmaximum_mag.h b/libc/src/math/fmaximum_mag.h
new file mode 100644
index 000000000000..31b7c0fcf7ee
--- /dev/null
+++ b/libc/src/math/fmaximum_mag.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fmaximum_mag------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_H
+
+namespace LIBC_NAMESPACE {
+
+double fmaximum_mag(double x, double y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_H
diff --git a/libc/src/math/fmaximum_mag_num.h b/libc/src/math/fmaximum_mag_num.h
new file mode 100644
index 000000000000..c4ff243846e0
--- /dev/null
+++ b/libc/src/math/fmaximum_mag_num.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for fmaximum_mag_num---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUM_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUM_H
+
+namespace LIBC_NAMESPACE {
+
+double fmaximum_mag_num(double x, double y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUM_H
diff --git a/libc/src/math/fmaximum_mag_numf.h b/libc/src/math/fmaximum_mag_numf.h
new file mode 100644
index 000000000000..702903ab8bcf
--- /dev/null
+++ b/libc/src/math/fmaximum_mag_numf.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fmaximum_mag_numf ----------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUMF_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUMF_H
+
+namespace LIBC_NAMESPACE {
+
+float fmaximum_mag_numf(float x, float y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUMF_H
diff --git a/libc/src/math/fmaximum_mag_numf128.h b/libc/src/math/fmaximum_mag_numf128.h
new file mode 100644
index 000000000000..2afae7fc37c4
--- /dev/null
+++ b/libc/src/math/fmaximum_mag_numf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fmaximum_mag_numf128 -------------------*- C++
+// -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUMF128_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUMF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float128 fmaximum_mag_numf128(float128 x, float128 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUMF128_H
diff --git a/libc/src/math/fmaximum_mag_numl.h b/libc/src/math/fmaximum_mag_numl.h
new file mode 100644
index 000000000000..32f9ae9708a6
--- /dev/null
+++ b/libc/src/math/fmaximum_mag_numl.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fmaximum_mag_numl ----------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUML_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUML_H
+
+namespace LIBC_NAMESPACE {
+
+long double fmaximum_mag_numl(long double x, long double y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUML_H
diff --git a/libc/src/math/fmaximum_magf.h b/libc/src/math/fmaximum_magf.h
new file mode 100644
index 000000000000..1bfcc795f8f8
--- /dev/null
+++ b/libc/src/math/fmaximum_magf.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fmaximum_magf -------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_MAGF_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUM_MAGF_H
+
+namespace LIBC_NAMESPACE {
+
+float fmaximum_magf(float x, float y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_MAGF_H
diff --git a/libc/src/math/fmaximum_magf128.h b/libc/src/math/fmaximum_magf128.h
new file mode 100644
index 000000000000..23c466b74cce
--- /dev/null
+++ b/libc/src/math/fmaximum_magf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fmaximum_magf128 ----------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_MAGF128_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUM_MAGF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float128 fmaximum_magf128(float128 x, float128 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_MAGF128_H
diff --git a/libc/src/math/fmaximum_magl.h b/libc/src/math/fmaximum_magl.h
new file mode 100644
index 000000000000..23b283cb5dbc
--- /dev/null
+++ b/libc/src/math/fmaximum_magl.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fmaximum_magl -------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_MAGL_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUM_MAGL_H
+
+namespace LIBC_NAMESPACE {
+
+long double fmaximum_magl(long double x, long double y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_MAGL_H
diff --git a/libc/src/math/fmaximum_num.h b/libc/src/math/fmaximum_num.h
new file mode 100644
index 000000000000..ce3ce12ca64b
--- /dev/null
+++ b/libc/src/math/fmaximum_num.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fmaximum_num--------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_NUM_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUM_NUM_H
+
+namespace LIBC_NAMESPACE {
+
+double fmaximum_num(double x, double y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_NUM_H
diff --git a/libc/src/math/fmaximum_numf.h b/libc/src/math/fmaximum_numf.h
new file mode 100644
index 000000000000..b3243ed16be9
--- /dev/null
+++ b/libc/src/math/fmaximum_numf.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fmaximum_numf -------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_NUMF_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUM_NUMF_H
+
+namespace LIBC_NAMESPACE {
+
+float fmaximum_numf(float x, float y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_NUMF_H
diff --git a/libc/src/math/fmaximum_numf128.h b/libc/src/math/fmaximum_numf128.h
new file mode 100644
index 000000000000..d55183ce88cc
--- /dev/null
+++ b/libc/src/math/fmaximum_numf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fmaximum_numf128 ----------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_NUMF128_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUM_NUMF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float128 fmaximum_numf128(float128 x, float128 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_NUMF128_H
diff --git a/libc/src/math/fmaximum_numl.h b/libc/src/math/fmaximum_numl.h
new file mode 100644
index 000000000000..f668cbdf7315
--- /dev/null
+++ b/libc/src/math/fmaximum_numl.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fmaximum_numl -------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_NUML_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUM_NUML_H
+
+namespace LIBC_NAMESPACE {
+
+long double fmaximum_numl(long double x, long double y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_NUML_H
diff --git a/libc/src/math/fmaximumf.h b/libc/src/math/fmaximumf.h
new file mode 100644
index 000000000000..4eee696ce152
--- /dev/null
+++ b/libc/src/math/fmaximumf.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fmaximumf -------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUMF_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUMF_H
+
+namespace LIBC_NAMESPACE {
+
+float fmaximumf(float x, float y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUMF_H
diff --git a/libc/src/math/fmaximumf128.h b/libc/src/math/fmaximumf128.h
new file mode 100644
index 000000000000..4a214ef096e1
--- /dev/null
+++ b/libc/src/math/fmaximumf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fmaximumf128 ----------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUMF128_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUMF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float128 fmaximumf128(float128 x, float128 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUMF128_H
diff --git a/libc/src/math/fmaximuml.h b/libc/src/math/fmaximuml.h
new file mode 100644
index 000000000000..ba8a8b12110c
--- /dev/null
+++ b/libc/src/math/fmaximuml.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fmaximuml -------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUML_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUML_H
+
+namespace LIBC_NAMESPACE {
+
+long double fmaximuml(long double x, long double y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUML_H
diff --git a/libc/src/math/fminimum.h b/libc/src/math/fminimum.h
new file mode 100644
index 000000000000..9d39b1bdd3db
--- /dev/null
+++ b/libc/src/math/fminimum.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fminimum -------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUM_H
+
+namespace LIBC_NAMESPACE {
+
+double fminimum(double x, double y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_H
diff --git a/libc/src/math/fminimum_mag.h b/libc/src/math/fminimum_mag.h
new file mode 100644
index 000000000000..10b242e6026e
--- /dev/null
+++ b/libc/src/math/fminimum_mag.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fminimum_mag--------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_H
+
+namespace LIBC_NAMESPACE {
+
+double fminimum_mag(double x, double y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_H
diff --git a/libc/src/math/fminimum_mag_num.h b/libc/src/math/fminimum_mag_num.h
new file mode 100644
index 000000000000..eb1823018851
--- /dev/null
+++ b/libc/src/math/fminimum_mag_num.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fminimum_mag_num------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUM_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUM_H
+
+namespace LIBC_NAMESPACE {
+
+double fminimum_mag_num(double x, double y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUMH
diff --git a/libc/src/math/fminimum_mag_numf.h b/libc/src/math/fminimum_mag_numf.h
new file mode 100644
index 000000000000..809199091139
--- /dev/null
+++ b/libc/src/math/fminimum_mag_numf.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fminimum_mag_numf ----------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUMF_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUMF_H
+
+namespace LIBC_NAMESPACE {
+
+float fminimum_mag_numf(float x, float y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUMF_H
diff --git a/libc/src/math/fminimum_mag_numf128.h b/libc/src/math/fminimum_mag_numf128.h
new file mode 100644
index 000000000000..803c5e641d17
--- /dev/null
+++ b/libc/src/math/fminimum_mag_numf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fminimum_mag_numf128 -------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUMF128_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUMF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float128 fminimum_mag_numf128(float128 x, float128 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUMF128_H
diff --git a/libc/src/math/fminimum_mag_numl.h b/libc/src/math/fminimum_mag_numl.h
new file mode 100644
index 000000000000..fdbb18328069
--- /dev/null
+++ b/libc/src/math/fminimum_mag_numl.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fminimum_mag_numl ----------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUML_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUML_H
+
+namespace LIBC_NAMESPACE {
+
+long double fminimum_mag_numl(long double x, long double y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUML_H
diff --git a/libc/src/math/fminimum_magf.h b/libc/src/math/fminimum_magf.h
new file mode 100644
index 000000000000..6209340074d2
--- /dev/null
+++ b/libc/src/math/fminimum_magf.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fminimum_magf -------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_MAGF_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUM_MAGF_H
+
+namespace LIBC_NAMESPACE {
+
+float fminimum_magf(float x, float y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_MAGF_H
diff --git a/libc/src/math/fminimum_magf128.h b/libc/src/math/fminimum_magf128.h
new file mode 100644
index 000000000000..05bd163be97c
--- /dev/null
+++ b/libc/src/math/fminimum_magf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fminimum_magf128 ----------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_MAGF128_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUM_MAGF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float128 fminimum_magf128(float128 x, float128 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_MAGF128_H
diff --git a/libc/src/math/fminimum_magl.h b/libc/src/math/fminimum_magl.h
new file mode 100644
index 000000000000..bcda35ce3bd2
--- /dev/null
+++ b/libc/src/math/fminimum_magl.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fminimum_magl -------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_MAGL_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUM_MAGL_H
+
+namespace LIBC_NAMESPACE {
+
+long double fminimum_magl(long double x, long double y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_MAGL_H
diff --git a/libc/src/math/fminimum_num.h b/libc/src/math/fminimum_num.h
new file mode 100644
index 000000000000..4c864cba8487
--- /dev/null
+++ b/libc/src/math/fminimum_num.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fminimum_num--------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_NUM_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUM_NUM_H
+
+namespace LIBC_NAMESPACE {
+
+double fminimum_num(double x, double y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_NUM_H
diff --git a/libc/src/math/fminimum_numf.h b/libc/src/math/fminimum_numf.h
new file mode 100644
index 000000000000..ac4b08b292be
--- /dev/null
+++ b/libc/src/math/fminimum_numf.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fminimum_numf -------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_NUMF_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUM_NUMF_H
+
+namespace LIBC_NAMESPACE {
+
+float fminimum_numf(float x, float y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_NUMF_H
diff --git a/libc/src/math/fminimum_numf128.h b/libc/src/math/fminimum_numf128.h
new file mode 100644
index 000000000000..00f8960ff127
--- /dev/null
+++ b/libc/src/math/fminimum_numf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fminimum_numf128 ----------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_NUMF128_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUM_NUMF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float128 fminimum_numf128(float128 x, float128 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_NUMF128_H
diff --git a/libc/src/math/fminimum_numl.h b/libc/src/math/fminimum_numl.h
new file mode 100644
index 000000000000..0da204e72c74
--- /dev/null
+++ b/libc/src/math/fminimum_numl.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fminimum_numl -------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_NUML_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUM_NUML_H
+
+namespace LIBC_NAMESPACE {
+
+long double fminimum_numl(long double x, long double y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_NUML_H
diff --git a/libc/src/math/fminimumf.h b/libc/src/math/fminimumf.h
new file mode 100644
index 000000000000..424309f3a531
--- /dev/null
+++ b/libc/src/math/fminimumf.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fminimumf -------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUMF_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUMF_H
+
+namespace LIBC_NAMESPACE {
+
+float fminimumf(float x, float y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUMF_H
diff --git a/libc/src/math/fminimumf128.h b/libc/src/math/fminimumf128.h
new file mode 100644
index 000000000000..7ff019072737
--- /dev/null
+++ b/libc/src/math/fminimumf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fminimumf128 ----------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUMF128_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUMF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float128 fminimumf128(float128 x, float128 y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUMF128_H
diff --git a/libc/src/math/fminimuml.h b/libc/src/math/fminimuml.h
new file mode 100644
index 000000000000..b9cc321354a2
--- /dev/null
+++ b/libc/src/math/fminimuml.h
@@ -0,0 +1,19 @@
+//===-- Implementation header for fminimuml -------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUML_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUML_H
+
+namespace LIBC_NAMESPACE {
+
+long double fminimuml(long double x, long double y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUML_H
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index 5198e9c58a98..60d329f9adc6 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -1545,6 +1545,400 @@ add_entrypoint_object(
     -O3
 )
 
+add_entrypoint_object(
+  fmaximum
+  SRCS
+    fmaximum.cpp
+  HDRS
+    ../fmaximum.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fmaximumf
+  SRCS
+    fmaximumf.cpp
+  HDRS
+    ../fmaximumf.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fmaximuml
+  SRCS
+    fmaximuml.cpp
+  HDRS
+    ../fmaximuml.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fmaximumf128
+  SRCS
+    fmaximumf128.cpp
+  HDRS
+    ../fmaximumf128.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  fmaximum_num
+  SRCS
+    fmaximum_num.cpp
+  HDRS
+    ../fmaximum_num.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fmaximum_numf
+  SRCS
+    fmaximum_numf.cpp
+  HDRS
+    ../fmaximum_numf.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fmaximum_numl
+  SRCS
+    fmaximum_numl.cpp
+  HDRS
+    ../fmaximum_numl.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fmaximum_numf128
+  SRCS
+    fmaximum_numf128.cpp
+  HDRS
+    ../fmaximum_numf128.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  fmaximum_mag
+  SRCS
+    fmaximum_mag.cpp
+  HDRS
+    ../fmaximum_mag.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fmaximum_magf
+  SRCS
+    fmaximum_magf.cpp
+  HDRS
+    ../fmaximum_magf.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fmaximum_magl
+  SRCS
+    fmaximum_magl.cpp
+  HDRS
+    ../fmaximum_magl.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fmaximum_magf128
+  SRCS
+    fmaximum_magf128.cpp
+  HDRS
+    ../fmaximum_magf128.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
+
+add_entrypoint_object(
+  fmaximum_mag_num
+  SRCS
+    fmaximum_mag_num.cpp
+  HDRS
+    ../fmaximum_mag_num.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fmaximum_mag_numf
+  SRCS
+    fmaximum_mag_numf.cpp
+  HDRS
+    ../fmaximum_mag_numf.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fmaximum_mag_numl
+  SRCS
+    fmaximum_mag_numl.cpp
+  HDRS
+    ../fmaximum_mag_numl.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fmaximum_mag_numf128
+  SRCS
+    fmaximum_mag_numf128.cpp
+  HDRS
+    ../fmaximum_mag_numf128.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  fminimum
+  SRCS
+    fminimum.cpp
+  HDRS
+    ../fminimum.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fminimumf
+  SRCS
+    fminimumf.cpp
+  HDRS
+    ../fminimumf.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fminimuml
+  SRCS
+    fminimuml.cpp
+  HDRS
+    ../fminimuml.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fminimumf128
+  SRCS
+    fminimumf128.cpp
+  HDRS
+    ../fminimumf128.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  fminimum_num
+  SRCS
+    fminimum_num.cpp
+  HDRS
+    ../fminimum_num.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fminimum_numf
+  SRCS
+    fminimum_numf.cpp
+  HDRS
+    ../fminimum_numf.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fminimum_numl
+  SRCS
+    fminimum_numl.cpp
+  HDRS
+    ../fminimum_numl.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fminimum_numf128
+  SRCS
+    fminimum_numf128.cpp
+  HDRS
+    ../fminimum_numf128.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
+add_entrypoint_object(
+  fminimum_mag
+  SRCS
+    fminimum_mag.cpp
+  HDRS
+    ../fminimum_mag.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fminimum_magf
+  SRCS
+    fminimum_magf.cpp
+  HDRS
+    ../fminimum_magf.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fminimum_magl
+  SRCS
+    fminimum_magl.cpp
+  HDRS
+    ../fminimum_magl.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fminimum_magf128
+  SRCS
+    fminimum_magf128.cpp
+  HDRS
+    ../fminimum_magf128.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
+
+add_entrypoint_object(
+  fminimum_mag_num
+  SRCS
+    fminimum_mag_num.cpp
+  HDRS
+    ../fminimum_mag_num.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fminimum_mag_numf
+  SRCS
+    fminimum_mag_numf.cpp
+  HDRS
+    ../fminimum_mag_numf.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fminimum_mag_numl
+  SRCS
+    fminimum_mag_numl.cpp
+  HDRS
+    ../fminimum_mag_numl.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fminimum_mag_numf128
+  SRCS
+    fminimum_mag_numf128.cpp
+  HDRS
+    ../fminimum_mag_numf128.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.basic_operations
+  COMPILE_OPTIONS
+    -O3
+)
+
 add_entrypoint_object(
   sqrt
   SRCS
@@ -1557,6 +1951,7 @@ add_entrypoint_object(
     -O3
 )
 
+
 add_entrypoint_object(
   sqrtf
   SRCS
diff --git a/libc/src/math/generic/fmaximum.cpp b/libc/src/math/generic/fmaximum.cpp
new file mode 100644
index 000000000000..ac9593b325d4
--- /dev/null
+++ b/libc/src/math/generic/fmaximum.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaximum function--------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximum.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, fmaximum, (double x, double y)) {
+  return fputil::fmaximum(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmaximum_mag.cpp b/libc/src/math/generic/fmaximum_mag.cpp
new file mode 100644
index 000000000000..0deb0c2835f9
--- /dev/null
+++ b/libc/src/math/generic/fmaximum_mag.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaximum_mag function----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximum_mag.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, fmaximum_mag, (double x, double y)) {
+  return fputil::fmaximum_mag(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmaximum_mag_num.cpp b/libc/src/math/generic/fmaximum_mag_num.cpp
new file mode 100644
index 000000000000..d0b1096b88bc
--- /dev/null
+++ b/libc/src/math/generic/fmaximum_mag_num.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaximum_mag_num function------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximum_mag_num.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, fmaximum_mag_num, (double x, double y)) {
+  return fputil::fmaximum_mag_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmaximum_mag_numf.cpp b/libc/src/math/generic/fmaximum_mag_numf.cpp
new file mode 100644
index 000000000000..672d3fd3b263
--- /dev/null
+++ b/libc/src/math/generic/fmaximum_mag_numf.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaximum_mag_numf function-----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximum_mag_numf.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, fmaximum_mag_numf, (float x, float y)) {
+  return fputil::fmaximum_mag_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmaximum_mag_numf128.cpp b/libc/src/math/generic/fmaximum_mag_numf128.cpp
new file mode 100644
index 000000000000..e7d13f13a098
--- /dev/null
+++ b/libc/src/math/generic/fmaximum_mag_numf128.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaximum_mag_numf128 function--------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximum_mag_numf128.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float128, fmaximum_mag_numf128, (float128 x, float128 y)) {
+  return fputil::fmaximum_mag_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmaximum_mag_numl.cpp b/libc/src/math/generic/fmaximum_mag_numl.cpp
new file mode 100644
index 000000000000..a8499ca473b3
--- /dev/null
+++ b/libc/src/math/generic/fmaximum_mag_numl.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of fmaximum_mag_numl function-----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximum_mag_numl.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long double, fmaximum_mag_numl,
+                   (long double x, long double y)) {
+  return fputil::fmaximum_mag_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmaximum_magf.cpp b/libc/src/math/generic/fmaximum_magf.cpp
new file mode 100644
index 000000000000..380aca05a525
--- /dev/null
+++ b/libc/src/math/generic/fmaximum_magf.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaximum_magf function---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximum_magf.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, fmaximum_magf, (float x, float y)) {
+  return fputil::fmaximum_mag(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmaximum_magf128.cpp b/libc/src/math/generic/fmaximum_magf128.cpp
new file mode 100644
index 000000000000..301938fb7ffd
--- /dev/null
+++ b/libc/src/math/generic/fmaximum_magf128.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaximum_magf128 function------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximum_magf128.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float128, fmaximum_magf128, (float128 x, float128 y)) {
+  return fputil::fmaximum_mag(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmaximum_magl.cpp b/libc/src/math/generic/fmaximum_magl.cpp
new file mode 100644
index 000000000000..283a11eda9aa
--- /dev/null
+++ b/libc/src/math/generic/fmaximum_magl.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaximum_magl function---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximum_magl.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long double, fmaximum_magl, (long double x, long double y)) {
+  return fputil::fmaximum_mag(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmaximum_num.cpp b/libc/src/math/generic/fmaximum_num.cpp
new file mode 100644
index 000000000000..23553dbcae7e
--- /dev/null
+++ b/libc/src/math/generic/fmaximum_num.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaximum_num function----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximum_num.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, fmaximum_num, (double x, double y)) {
+  return fputil::fmaximum_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmaximum_numf.cpp b/libc/src/math/generic/fmaximum_numf.cpp
new file mode 100644
index 000000000000..f946ff43f543
--- /dev/null
+++ b/libc/src/math/generic/fmaximum_numf.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaximum_numf function---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximum_numf.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, fmaximum_numf, (float x, float y)) {
+  return fputil::fmaximum_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmaximum_numf128.cpp b/libc/src/math/generic/fmaximum_numf128.cpp
new file mode 100644
index 000000000000..f33a5e195bf2
--- /dev/null
+++ b/libc/src/math/generic/fmaximum_numf128.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaximum_numf128 function------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximum_numf128.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float128, fmaximum_numf128, (float128 x, float128 y)) {
+  return fputil::fmaximum_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmaximum_numl.cpp b/libc/src/math/generic/fmaximum_numl.cpp
new file mode 100644
index 000000000000..503fc41409f6
--- /dev/null
+++ b/libc/src/math/generic/fmaximum_numl.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaximum_numl function---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximum_numl.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long double, fmaximum_numl, (long double x, long double y)) {
+  return fputil::fmaximum_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmaximumf.cpp b/libc/src/math/generic/fmaximumf.cpp
new file mode 100644
index 000000000000..3b2a60931bf6
--- /dev/null
+++ b/libc/src/math/generic/fmaximumf.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaximumf function-------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximumf.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, fmaximumf, (float x, float y)) {
+  return fputil::fmaximum(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmaximumf128.cpp b/libc/src/math/generic/fmaximumf128.cpp
new file mode 100644
index 000000000000..0099c913b052
--- /dev/null
+++ b/libc/src/math/generic/fmaximumf128.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaximumf128 function----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximumf128.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float128, fmaximumf128, (float128 x, float128 y)) {
+  return fputil::fmaximum(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fmaximuml.cpp b/libc/src/math/generic/fmaximuml.cpp
new file mode 100644
index 000000000000..ecd698300458
--- /dev/null
+++ b/libc/src/math/generic/fmaximuml.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fmaximuml function-------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximuml.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long double, fmaximuml, (long double x, long double y)) {
+  return fputil::fmaximum(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimum.cpp b/libc/src/math/generic/fminimum.cpp
new file mode 100644
index 000000000000..28b257d950f4
--- /dev/null
+++ b/libc/src/math/generic/fminimum.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminimum function--------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimum.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, fminimum, (double x, double y)) {
+  return fputil::fminimum(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimum_mag.cpp b/libc/src/math/generic/fminimum_mag.cpp
new file mode 100644
index 000000000000..6af99570e1ea
--- /dev/null
+++ b/libc/src/math/generic/fminimum_mag.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminimum_mag function----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimum_mag.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, fminimum_mag, (double x, double y)) {
+  return fputil::fminimum_mag(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimum_mag_num.cpp b/libc/src/math/generic/fminimum_mag_num.cpp
new file mode 100644
index 000000000000..fc5431ae2799
--- /dev/null
+++ b/libc/src/math/generic/fminimum_mag_num.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminimum_mag_num function------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimum_mag_num.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, fminimum_mag_num, (double x, double y)) {
+  return fputil::fminimum_mag_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimum_mag_numf.cpp b/libc/src/math/generic/fminimum_mag_numf.cpp
new file mode 100644
index 000000000000..71179a6f03c2
--- /dev/null
+++ b/libc/src/math/generic/fminimum_mag_numf.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminimum_mag_numf function-----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimum_mag_numf.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, fminimum_mag_numf, (float x, float y)) {
+  return fputil::fminimum_mag_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimum_mag_numf128.cpp b/libc/src/math/generic/fminimum_mag_numf128.cpp
new file mode 100644
index 000000000000..109ce7e4e011
--- /dev/null
+++ b/libc/src/math/generic/fminimum_mag_numf128.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminimum_mag_numf128 function--------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimum_mag_numf128.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float128, fminimum_mag_numf128, (float128 x, float128 y)) {
+  return fputil::fminimum_mag_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimum_mag_numl.cpp b/libc/src/math/generic/fminimum_mag_numl.cpp
new file mode 100644
index 000000000000..c97ce6ead042
--- /dev/null
+++ b/libc/src/math/generic/fminimum_mag_numl.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of fminimum_mag_numl function-----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimum_mag_numl.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long double, fminimum_mag_numl,
+                   (long double x, long double y)) {
+  return fputil::fminimum_mag_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimum_magf.cpp b/libc/src/math/generic/fminimum_magf.cpp
new file mode 100644
index 000000000000..834f6a4a9710
--- /dev/null
+++ b/libc/src/math/generic/fminimum_magf.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminimum_magf function---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimum_magf.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, fminimum_magf, (float x, float y)) {
+  return fputil::fminimum_mag(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimum_magf128.cpp b/libc/src/math/generic/fminimum_magf128.cpp
new file mode 100644
index 000000000000..2828e28bb0a4
--- /dev/null
+++ b/libc/src/math/generic/fminimum_magf128.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminimum_magf128 function------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimum_magf128.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float128, fminimum_magf128, (float128 x, float128 y)) {
+  return fputil::fminimum_mag(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimum_magl.cpp b/libc/src/math/generic/fminimum_magl.cpp
new file mode 100644
index 000000000000..50e328fd92d1
--- /dev/null
+++ b/libc/src/math/generic/fminimum_magl.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminimum_magl function---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimum_magl.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long double, fminimum_magl, (long double x, long double y)) {
+  return fputil::fminimum_mag(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimum_num.cpp b/libc/src/math/generic/fminimum_num.cpp
new file mode 100644
index 000000000000..e89c7f5acf9b
--- /dev/null
+++ b/libc/src/math/generic/fminimum_num.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminimum_num function----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimum_num.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, fminimum_num, (double x, double y)) {
+  return fputil::fminimum_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimum_numf.cpp b/libc/src/math/generic/fminimum_numf.cpp
new file mode 100644
index 000000000000..c37c8bd423a1
--- /dev/null
+++ b/libc/src/math/generic/fminimum_numf.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminimum_numf function---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimum_numf.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, fminimum_numf, (float x, float y)) {
+  return fputil::fminimum_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimum_numf128.cpp b/libc/src/math/generic/fminimum_numf128.cpp
new file mode 100644
index 000000000000..6b1f77bb447c
--- /dev/null
+++ b/libc/src/math/generic/fminimum_numf128.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminimum_numf128 function------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimum_numf128.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float128, fminimum_numf128, (float128 x, float128 y)) {
+  return fputil::fminimum_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimum_numl.cpp b/libc/src/math/generic/fminimum_numl.cpp
new file mode 100644
index 000000000000..22045f83f2a7
--- /dev/null
+++ b/libc/src/math/generic/fminimum_numl.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminimum_numl function---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimum_numl.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long double, fminimum_numl, (long double x, long double y)) {
+  return fputil::fminimum_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimumf.cpp b/libc/src/math/generic/fminimumf.cpp
new file mode 100644
index 000000000000..c937fb0ea01d
--- /dev/null
+++ b/libc/src/math/generic/fminimumf.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminimumf function-------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimumf.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, fminimumf, (float x, float y)) {
+  return fputil::fminimum(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimumf128.cpp b/libc/src/math/generic/fminimumf128.cpp
new file mode 100644
index 000000000000..24e02b8ff537
--- /dev/null
+++ b/libc/src/math/generic/fminimumf128.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminimumf128 function----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimumf128.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float128, fminimumf128, (float128 x, float128 y)) {
+  return fputil::fminimum(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/fminimuml.cpp b/libc/src/math/generic/fminimuml.cpp
new file mode 100644
index 000000000000..43319503661a
--- /dev/null
+++ b/libc/src/math/generic/fminimuml.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of fminimuml function-------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimuml.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long double, fminimuml, (long double x, long double y)) {
+  return fputil::fminimum(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index 56d864e78083..1345dca08a6a 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -1477,6 +1477,424 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  fmaximuml_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximuml_test.cpp
+  HDRS
+    FMaximumTest.h
+  DEPENDS
+    libc.src.math.fmaximuml
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaximumf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximumf128_test.cpp
+  HDRS
+    FMaximumTest.h
+  DEPENDS
+    libc.src.math.fmaximumf128
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaximum_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_test.cpp
+  HDRS
+    FMaximumTest.h
+  DEPENDS
+    libc.src.math.fmaximum
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaximumf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximumf_test.cpp
+  HDRS
+    FMaximumTest.h
+  DEPENDS
+    libc.src.math.fmaximumf
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaximum_numf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_numf_test.cpp
+  HDRS
+    FMaximumNumTest.h
+  DEPENDS
+    libc.src.math.fmaximum_numf
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaximum_num_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_num_test.cpp
+  HDRS
+    FMaximumNumTest.h
+  DEPENDS
+    libc.src.math.fmaximum_num
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaximum_numl_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_numl_test.cpp
+  HDRS
+    FMaximumNumTest.h
+  DEPENDS
+    libc.src.math.fmaximum_numl
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaximum_numf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_numf128_test.cpp
+  HDRS
+    FMaximumNumTest.h
+  DEPENDS
+    libc.src.math.fmaximum_numf128
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaximum_magf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_magf_test.cpp
+  HDRS
+    FMaximumMagTest.h
+  DEPENDS
+    libc.src.math.fmaximum_magf
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaximum_mag_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_mag_test.cpp
+  HDRS
+    FMaximumMagTest.h
+  DEPENDS
+    libc.src.math.fmaximum_mag
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaximum_magl_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_magl_test.cpp
+  HDRS
+    FMaximumMagTest.h
+  DEPENDS
+    libc.src.math.fmaximum_magl
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaximum_magf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_magf128_test.cpp
+  HDRS
+    FMaximumMagTest.h
+  DEPENDS
+    libc.src.math.fmaximum_magf128
+    libc.src.__support.FPUtil.fp_bits
+)
+
+
+add_fp_unittest(
+  fmaximum_mag_numf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_mag_numf_test.cpp
+  HDRS
+    FMaximumMagNumTest.h
+  DEPENDS
+    libc.src.math.fmaximum_mag_numf
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaximum_mag_num_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_mag_num_test.cpp
+  HDRS
+    FMaximumMagNumTest.h
+  DEPENDS
+    libc.src.math.fmaximum_mag_num
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaximum_mag_numl_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_mag_numl_test.cpp
+  HDRS
+    FMaximumMagNumTest.h
+  DEPENDS
+    libc.src.math.fmaximum_mag_numl
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaximum_mag_numf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_mag_numf128_test.cpp
+  HDRS
+    FMaximumMagNumTest.h
+  DEPENDS
+    libc.src.math.fmaximum_mag_numf128
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminimuml_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimuml_test.cpp
+  HDRS
+    FMinimumTest.h
+  DEPENDS
+    libc.src.math.fminimuml
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminimumf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimumf128_test.cpp
+  HDRS
+    FMinimumTest.h
+  DEPENDS
+    libc.src.math.fminimumf128
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminimum_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimum_test.cpp
+  HDRS
+    FMinimumTest.h
+  DEPENDS
+    libc.src.math.fminimum
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminimumf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimumf_test.cpp
+  HDRS
+    FMinimumTest.h
+  DEPENDS
+    libc.src.math.fminimumf
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminimum_numf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimum_numf_test.cpp
+  HDRS
+    FMinimumNumTest.h
+  DEPENDS
+    libc.src.math.fminimum_numf
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminimum_num_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimum_num_test.cpp
+  HDRS
+    FMinimumNumTest.h
+  DEPENDS
+    libc.src.math.fminimum_num
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminimum_numl_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimum_numl_test.cpp
+  HDRS
+    FMinimumNumTest.h
+  DEPENDS
+    libc.src.math.fminimum_numl
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminimum_numf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimum_numf128_test.cpp
+  HDRS
+    FMinimumNumTest.h
+  DEPENDS
+    libc.src.math.fminimum_numf128
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminimum_magf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimum_magf_test.cpp
+  HDRS
+    FMinimumMagTest.h
+  DEPENDS
+    libc.src.math.fminimum_magf
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminimum_mag_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimum_mag_test.cpp
+  HDRS
+    FMinimumMagTest.h
+  DEPENDS
+    libc.src.math.fminimum_mag
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminimum_magl_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimum_magl_test.cpp
+  HDRS
+    FMinimumMagTest.h
+  DEPENDS
+    libc.src.math.fminimum_magl
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminimum_magf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimum_magf128_test.cpp
+  HDRS
+    FMinimumMagTest.h
+  DEPENDS
+    libc.src.math.fminimum_magf128
+    libc.src.__support.FPUtil.fp_bits
+)
+
+
+add_fp_unittest(
+  fminimum_mag_numf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimum_mag_numf_test.cpp
+  HDRS
+    FMinimumMagNumTest.h
+  DEPENDS
+    libc.src.math.fminimum_mag_numf
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminimum_mag_num_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimum_mag_num_test.cpp
+  HDRS
+    FMinimumMagNumTest.h
+  DEPENDS
+    libc.src.math.fminimum_mag_num
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminimum_mag_numl_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimum_mag_numl_test.cpp
+  HDRS
+    FMinimumMagNumTest.h
+  DEPENDS
+    libc.src.math.fminimum_mag_numl
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fminimum_mag_numf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimum_mag_numf128_test.cpp
+  HDRS
+    FMinimumMagNumTest.h
+  DEPENDS
+    libc.src.math.fminimum_mag_numf128
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   sqrtf_test
   SUITE
diff --git a/libc/test/src/math/smoke/FMaximumMagNumTest.h b/libc/test/src/math/smoke/FMaximumMagNumTest.h
new file mode 100644
index 000000000000..715dd4ed913f
--- /dev/null
+++ b/libc/test/src/math/smoke/FMaximumMagNumTest.h
@@ -0,0 +1,101 @@
+//===-- Utility class to test fmaximum_mag_num[f|l] -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMMAG_NUMTEST_H
+#define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMMAG_NUMTEST_H
+
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+template <typename T>
+class FMaximumMagNumTest : public LIBC_NAMESPACE::testing::Test {
+
+  DECLARE_SPECIAL_CONSTANTS(T)
+
+public:
+  typedef T (*FMaximumMagNumFunc)(T, T);
+
+  void testNaN(FMaximumMagNumFunc func) {
+    EXPECT_FP_EQ(inf, func(aNaN, inf));
+    EXPECT_FP_EQ_WITH_EXCEPTION(inf, func(sNaN, inf), FE_INVALID);
+    EXPECT_FP_EQ(neg_inf, func(neg_inf, aNaN));
+    EXPECT_FP_EQ_WITH_EXCEPTION(neg_inf, func(neg_inf, sNaN), FE_INVALID);
+    EXPECT_EQ(FPBits(aNaN).uintval(), FPBits(func(aNaN, aNaN)).uintval());
+    EXPECT_FP_EQ(0.0, func(aNaN, 0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, aNaN));
+    EXPECT_FP_EQ_WITH_EXCEPTION(0.0, func(sNaN, 0.0), FE_INVALID);
+    EXPECT_FP_EQ_WITH_EXCEPTION(-0.0, func(-0.0, sNaN), FE_INVALID);
+    EXPECT_FP_EQ(T(-1.2345), func(aNaN, T(-1.2345)));
+    EXPECT_FP_EQ(T(1.2345), func(T(1.2345), aNaN));
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-1.2345), func(sNaN, T(-1.2345)), FE_INVALID);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.2345), func(T(1.2345), sNaN), FE_INVALID);
+    EXPECT_FP_IS_NAN_WITH_EXCEPTION(func(aNaN, sNaN), FE_INVALID);
+    EXPECT_FP_IS_NAN_WITH_EXCEPTION(func(sNaN, aNaN), FE_INVALID);
+    EXPECT_EQ(FPBits(aNaN).uintval(), FPBits(func(aNaN, sNaN)).uintval());
+    EXPECT_EQ(FPBits(aNaN).uintval(), FPBits(func(sNaN, aNaN)).uintval());
+    EXPECT_EQ(FPBits(aNaN).uintval(), FPBits(func(sNaN, sNaN)).uintval());
+  }
+
+  void testInfArg(FMaximumMagNumFunc func) {
+    EXPECT_FP_EQ(inf, func(neg_inf, inf));
+    EXPECT_FP_EQ(inf, func(inf, 0.0));
+    EXPECT_FP_EQ(inf, func(-0.0, inf));
+    EXPECT_FP_EQ(inf, func(inf, T(1.2345)));
+    EXPECT_FP_EQ(inf, func(T(-1.2345), inf));
+  }
+
+  void testNegInfArg(FMaximumMagNumFunc func) {
+    EXPECT_FP_EQ(inf, func(inf, neg_inf));
+    EXPECT_FP_EQ(neg_inf, func(neg_inf, 0.0));
+    EXPECT_FP_EQ(neg_inf, func(-0.0, neg_inf));
+    EXPECT_FP_EQ(neg_inf, func(neg_inf, T(-1.2345)));
+    EXPECT_FP_EQ(neg_inf, func(T(1.2345), neg_inf));
+  }
+
+  void testBothZero(FMaximumMagNumFunc func) {
+    EXPECT_FP_EQ(0.0, func(0.0, 0.0));
+    EXPECT_FP_EQ(0.0, func(-0.0, 0.0));
+    EXPECT_FP_EQ(0.0, func(0.0, -0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, -0.0));
+  }
+
+  void testRange(FMaximumMagNumFunc func) {
+    constexpr StorageType COUNT = 100'001;
+    constexpr StorageType STEP = STORAGE_MAX / COUNT;
+    for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT;
+         ++i, v += STEP, w -= STEP) {
+      FPBits xbits(v), ybits(w);
+      if (xbits.is_inf_or_nan())
+        continue;
+      if (ybits.is_inf_or_nan())
+        continue;
+      T x = xbits.get_val();
+      T y = ybits.get_val();
+      if ((x == 0) && (y == 0))
+        continue;
+
+      if (LIBC_NAMESPACE::fputil::abs(x) > LIBC_NAMESPACE::fputil::abs(y)) {
+        EXPECT_FP_EQ(x, func(x, y));
+      } else {
+        EXPECT_FP_EQ(y, func(x, y));
+      }
+    }
+  }
+};
+
+#define LIST_FMAXIMUM_MAG_NUM_TESTS(T, func)                                   \
+  using LlvmLibcFMaximumMagNumTest = FMaximumMagNumTest<T>;                    \
+  TEST_F(LlvmLibcFMaximumMagNumTest, NaN) { testNaN(&func); }                  \
+  TEST_F(LlvmLibcFMaximumMagNumTest, InfArg) { testInfArg(&func); }            \
+  TEST_F(LlvmLibcFMaximumMagNumTest, NegInfArg) { testNegInfArg(&func); }      \
+  TEST_F(LlvmLibcFMaximumMagNumTest, BothZero) { testBothZero(&func); }        \
+  TEST_F(LlvmLibcFMaximumMagNumTest, Range) { testRange(&func); }
+
+#endif // LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMMAG_NUMTEST_H
diff --git a/libc/test/src/math/smoke/FMaximumMagTest.h b/libc/test/src/math/smoke/FMaximumMagTest.h
new file mode 100644
index 000000000000..38276e0fe2fd
--- /dev/null
+++ b/libc/test/src/math/smoke/FMaximumMagTest.h
@@ -0,0 +1,89 @@
+//===-- Utility class to test fmaximum_mag[f|l] -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUM_MAGTEST_H
+#define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUM_MAGTEST_H
+
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+template <typename T>
+class FMaximumMagTest : public LIBC_NAMESPACE::testing::Test {
+
+  DECLARE_SPECIAL_CONSTANTS(T)
+
+public:
+  typedef T (*FMaximumMagFunc)(T, T);
+
+  void testNaN(FMaximumMagFunc func) {
+    EXPECT_FP_EQ(aNaN, func(aNaN, inf));
+    EXPECT_FP_EQ(aNaN, func(neg_inf, aNaN));
+    EXPECT_FP_EQ(aNaN, func(aNaN, 0.0));
+    EXPECT_FP_EQ(aNaN, func(-0.0, aNaN));
+    EXPECT_FP_EQ(aNaN, func(aNaN, T(-1.2345)));
+    EXPECT_FP_EQ(aNaN, func(T(1.2345), aNaN));
+    EXPECT_FP_EQ(aNaN, func(aNaN, aNaN));
+  }
+
+  void testInfArg(FMaximumMagFunc func) {
+    EXPECT_FP_EQ(inf, func(neg_inf, inf));
+    EXPECT_FP_EQ(inf, func(inf, 0.0));
+    EXPECT_FP_EQ(inf, func(-0.0, inf));
+    EXPECT_FP_EQ(inf, func(inf, T(1.2345)));
+    EXPECT_FP_EQ(inf, func(T(-1.2345), inf));
+  }
+
+  void testNegInfArg(FMaximumMagFunc func) {
+    EXPECT_FP_EQ(inf, func(inf, neg_inf));
+    EXPECT_FP_EQ(neg_inf, func(neg_inf, 0.0));
+    EXPECT_FP_EQ(neg_inf, func(-0.0, neg_inf));
+    EXPECT_FP_EQ(neg_inf, func(neg_inf, T(-1.2345)));
+    EXPECT_FP_EQ(neg_inf, func(T(1.2345), neg_inf));
+  }
+
+  void testBothZero(FMaximumMagFunc func) {
+    EXPECT_FP_EQ(0.0, func(0.0, 0.0));
+    EXPECT_FP_EQ(0.0, func(-0.0, 0.0));
+    EXPECT_FP_EQ(0.0, func(0.0, -0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, -0.0));
+  }
+
+  void testRange(FMaximumMagFunc func) {
+    constexpr StorageType COUNT = 100'001;
+    constexpr StorageType STEP = STORAGE_MAX / COUNT;
+    for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT;
+         ++i, v += STEP, w -= STEP) {
+      FPBits xbits(v), ybits(w);
+      if (xbits.is_inf_or_nan())
+        continue;
+      if (ybits.is_inf_or_nan())
+        continue;
+      T x = xbits.get_val();
+      T y = ybits.get_val();
+      if ((x == 0) && (y == 0))
+        continue;
+
+      if (LIBC_NAMESPACE::fputil::abs(x) > LIBC_NAMESPACE::fputil::abs(y)) {
+        EXPECT_FP_EQ(x, func(x, y));
+      } else {
+        EXPECT_FP_EQ(y, func(x, y));
+      }
+    }
+  }
+};
+
+#define LIST_FMAXIMUM_MAG_TESTS(T, func)                                       \
+  using LlvmLibcFMaximumMagTest = FMaximumMagTest<T>;                          \
+  TEST_F(LlvmLibcFMaximumMagTest, NaN) { testNaN(&func); }                     \
+  TEST_F(LlvmLibcFMaximumMagTest, InfArg) { testInfArg(&func); }               \
+  TEST_F(LlvmLibcFMaximumMagTest, NegInfArg) { testNegInfArg(&func); }         \
+  TEST_F(LlvmLibcFMaximumMagTest, BothZero) { testBothZero(&func); }           \
+  TEST_F(LlvmLibcFMaximumMagTest, Range) { testRange(&func); }
+
+#endif // LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUM_MAGTEST_H
diff --git a/libc/test/src/math/smoke/FMaximumNumTest.h b/libc/test/src/math/smoke/FMaximumNumTest.h
new file mode 100644
index 000000000000..57096f6b614a
--- /dev/null
+++ b/libc/test/src/math/smoke/FMaximumNumTest.h
@@ -0,0 +1,100 @@
+//===-- Utility class to test fmaximum_num[f|l] -----------------*- C++ -*-===//
+//
+// Part Of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMNUMTEST_H
+#define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMNUMTEST_H
+
+#include "src/__support/FPUtil/FPBits.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+template <typename T>
+class FMaximumNumTest : public LIBC_NAMESPACE::testing::Test {
+
+  DECLARE_SPECIAL_CONSTANTS(T)
+
+public:
+  typedef T (*FMaximumNumFunc)(T, T);
+
+  void testNaN(FMaximumNumFunc func) {
+    EXPECT_FP_EQ(inf, func(aNaN, inf));
+    EXPECT_FP_EQ_WITH_EXCEPTION(inf, func(sNaN, inf), FE_INVALID);
+    EXPECT_FP_EQ(neg_inf, func(neg_inf, aNaN));
+    EXPECT_FP_EQ_WITH_EXCEPTION(neg_inf, func(neg_inf, sNaN), FE_INVALID);
+    EXPECT_EQ(FPBits(aNaN).uintval(), FPBits(func(aNaN, aNaN)).uintval());
+    EXPECT_FP_EQ(0.0, func(aNaN, 0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, aNaN));
+    EXPECT_FP_EQ_WITH_EXCEPTION(0.0, func(sNaN, 0.0), FE_INVALID);
+    EXPECT_FP_EQ_WITH_EXCEPTION(-0.0, func(-0.0, sNaN), FE_INVALID);
+    EXPECT_FP_EQ(T(-1.2345), func(aNaN, T(-1.2345)));
+    EXPECT_FP_EQ(T(1.2345), func(T(1.2345), aNaN));
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-1.2345), func(sNaN, T(-1.2345)), FE_INVALID);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.2345), func(T(1.2345), sNaN), FE_INVALID);
+    EXPECT_FP_IS_NAN_WITH_EXCEPTION(func(aNaN, sNaN), FE_INVALID);
+    EXPECT_FP_IS_NAN_WITH_EXCEPTION(func(sNaN, aNaN), FE_INVALID);
+    EXPECT_EQ(FPBits(aNaN).uintval(), FPBits(func(aNaN, sNaN)).uintval());
+    EXPECT_EQ(FPBits(aNaN).uintval(), FPBits(func(sNaN, aNaN)).uintval());
+    EXPECT_EQ(FPBits(aNaN).uintval(), FPBits(func(sNaN, sNaN)).uintval());
+  }
+
+  void testInfArg(FMaximumNumFunc func) {
+    EXPECT_FP_EQ(inf, func(neg_inf, inf));
+    EXPECT_FP_EQ(inf, func(inf, 0.0));
+    EXPECT_FP_EQ(inf, func(-0.0, inf));
+    EXPECT_FP_EQ(inf, func(inf, T(1.2345)));
+    EXPECT_FP_EQ(inf, func(T(-1.2345), inf));
+  }
+
+  void testNegInfArg(FMaximumNumFunc func) {
+    EXPECT_FP_EQ(inf, func(inf, neg_inf));
+    EXPECT_FP_EQ(0.0, func(neg_inf, 0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, neg_inf));
+    EXPECT_FP_EQ(T(-1.2345), func(neg_inf, T(-1.2345)));
+    EXPECT_FP_EQ(T(1.2345), func(T(1.2345), neg_inf));
+  }
+
+  void testBothZero(FMaximumNumFunc func) {
+    EXPECT_FP_EQ(0.0, func(0.0, 0.0));
+    EXPECT_FP_EQ(0.0, func(-0.0, 0.0));
+    EXPECT_FP_EQ(0.0, func(0.0, -0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, -0.0));
+  }
+
+  void testRange(FMaximumNumFunc func) {
+    constexpr StorageType COUNT = 100'001;
+    constexpr StorageType STEP = STORAGE_MAX / COUNT;
+    for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT;
+         ++i, v += STEP, w -= STEP) {
+      FPBits xbits(v), ybits(w);
+      if (xbits.is_inf_or_nan())
+        continue;
+      if (ybits.is_inf_or_nan())
+        continue;
+      T x = xbits.get_val();
+      T y = ybits.get_val();
+      if ((x == 0) && (y == 0))
+        continue;
+
+      if (x > y) {
+        EXPECT_FP_EQ(x, func(x, y));
+      } else {
+        EXPECT_FP_EQ(y, func(x, y));
+      }
+    }
+  }
+};
+
+#define LIST_FMAXIMUM_NUM_TESTS(T, func)                                       \
+  using LlvmLibcFMaximumNumTest = FMaximumNumTest<T>;                          \
+  TEST_F(LlvmLibcFMaximumNumTest, NaN) { testNaN(&func); }                     \
+  TEST_F(LlvmLibcFMaximumNumTest, InfArg) { testInfArg(&func); }               \
+  TEST_F(LlvmLibcFMaximumNumTest, NegInfArg) { testNegInfArg(&func); }         \
+  TEST_F(LlvmLibcFMaximumNumTest, BothZero) { testBothZero(&func); }           \
+  TEST_F(LlvmLibcFMaximumNumTest, Range) { testRange(&func); }
+
+#endif // LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMNUMTEST_H
diff --git a/libc/test/src/math/smoke/FMaximumTest.h b/libc/test/src/math/smoke/FMaximumTest.h
new file mode 100644
index 000000000000..4db8bb93baae
--- /dev/null
+++ b/libc/test/src/math/smoke/FMaximumTest.h
@@ -0,0 +1,88 @@
+//===-- Utility class to test fmaximum[f|l] ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMTEST_H
+#define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMTEST_H
+
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+template <typename T>
+class FMaximumTest : public LIBC_NAMESPACE::testing::Test {
+
+  DECLARE_SPECIAL_CONSTANTS(T)
+
+public:
+  typedef T (*FMaximumFunc)(T, T);
+
+  void testNaN(FMaximumFunc func) {
+    EXPECT_FP_EQ(aNaN, func(aNaN, inf));
+    EXPECT_FP_EQ(aNaN, func(neg_inf, aNaN));
+    EXPECT_FP_EQ(aNaN, func(aNaN, 0.0));
+    EXPECT_FP_EQ(aNaN, func(-0.0, aNaN));
+    EXPECT_FP_EQ(aNaN, func(aNaN, T(-1.2345)));
+    EXPECT_FP_EQ(aNaN, func(T(1.2345), aNaN));
+    EXPECT_FP_EQ(aNaN, func(aNaN, aNaN));
+  }
+
+  void testInfArg(FMaximumFunc func) {
+    EXPECT_FP_EQ(inf, func(neg_inf, inf));
+    EXPECT_FP_EQ(inf, func(inf, 0.0));
+    EXPECT_FP_EQ(inf, func(-0.0, inf));
+    EXPECT_FP_EQ(inf, func(inf, T(1.2345)));
+    EXPECT_FP_EQ(inf, func(T(-1.2345), inf));
+  }
+
+  void testNegInfArg(FMaximumFunc func) {
+    EXPECT_FP_EQ(inf, func(inf, neg_inf));
+    EXPECT_FP_EQ(0.0, func(neg_inf, 0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, neg_inf));
+    EXPECT_FP_EQ(T(-1.2345), func(neg_inf, T(-1.2345)));
+    EXPECT_FP_EQ(T(1.2345), func(T(1.2345), neg_inf));
+  }
+
+  void testBothZero(FMaximumFunc func) {
+    EXPECT_FP_EQ(0.0, func(0.0, 0.0));
+    EXPECT_FP_EQ(0.0, func(-0.0, 0.0));
+    EXPECT_FP_EQ(0.0, func(0.0, -0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, -0.0));
+  }
+
+  void testRange(FMaximumFunc func) {
+    constexpr StorageType COUNT = 100'001;
+    constexpr StorageType STEP = STORAGE_MAX / COUNT;
+    for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT;
+         ++i, v += STEP, w -= STEP) {
+      FPBits xbits(v), ybits(w);
+      if (xbits.is_inf_or_nan())
+        continue;
+      if (ybits.is_inf_or_nan())
+        continue;
+      T x = xbits.get_val();
+      T y = ybits.get_val();
+      if ((x == 0) && (y == 0))
+        continue;
+
+      if (x > y) {
+        EXPECT_FP_EQ(x, func(x, y));
+      } else {
+        EXPECT_FP_EQ(y, func(x, y));
+      }
+    }
+  }
+};
+
+#define LIST_FMAXIMUM_TESTS(T, func)                                           \
+  using LlvmLibcFMaximumTest = FMaximumTest<T>;                                \
+  TEST_F(LlvmLibcFMaximumTest, NaN) { testNaN(&func); }                        \
+  TEST_F(LlvmLibcFMaximumTest, InfArg) { testInfArg(&func); }                  \
+  TEST_F(LlvmLibcFMaximumTest, NegInfArg) { testNegInfArg(&func); }            \
+  TEST_F(LlvmLibcFMaximumTest, BothZero) { testBothZero(&func); }              \
+  TEST_F(LlvmLibcFMaximumTest, Range) { testRange(&func); }
+
+#endif // LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMAXIMUMTEST_H
diff --git a/libc/test/src/math/smoke/FMinimumMagNumTest.h b/libc/test/src/math/smoke/FMinimumMagNumTest.h
new file mode 100644
index 000000000000..dec8b70740ca
--- /dev/null
+++ b/libc/test/src/math/smoke/FMinimumMagNumTest.h
@@ -0,0 +1,101 @@
+//===-- Utility class to test fminimum_mag_num[f|l] -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMMAG_NUMTEST_H
+#define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMMAG_NUMTEST_H
+
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+template <typename T>
+class FMinimumMagNumTest : public LIBC_NAMESPACE::testing::Test {
+
+  DECLARE_SPECIAL_CONSTANTS(T)
+
+public:
+  typedef T (*FMinimumMagNumFunc)(T, T);
+
+  void testNaN(FMinimumMagNumFunc func) {
+    EXPECT_FP_EQ(inf, func(aNaN, inf));
+    EXPECT_FP_EQ_WITH_EXCEPTION(inf, func(sNaN, inf), FE_INVALID);
+    EXPECT_FP_EQ(neg_inf, func(neg_inf, aNaN));
+    EXPECT_FP_EQ_WITH_EXCEPTION(neg_inf, func(neg_inf, sNaN), FE_INVALID);
+    EXPECT_EQ(FPBits(aNaN).uintval(), FPBits(func(aNaN, aNaN)).uintval());
+    EXPECT_FP_EQ(0.0, func(aNaN, 0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, aNaN));
+    EXPECT_FP_EQ_WITH_EXCEPTION(0.0, func(sNaN, 0.0), FE_INVALID);
+    EXPECT_FP_EQ_WITH_EXCEPTION(-0.0, func(-0.0, sNaN), FE_INVALID);
+    EXPECT_FP_EQ(T(-1.2345), func(aNaN, T(-1.2345)));
+    EXPECT_FP_EQ(T(1.2345), func(T(1.2345), aNaN));
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-1.2345), func(sNaN, T(-1.2345)), FE_INVALID);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.2345), func(T(1.2345), sNaN), FE_INVALID);
+    EXPECT_FP_IS_NAN_WITH_EXCEPTION(func(aNaN, sNaN), FE_INVALID);
+    EXPECT_FP_IS_NAN_WITH_EXCEPTION(func(sNaN, aNaN), FE_INVALID);
+    EXPECT_EQ(FPBits(aNaN).uintval(), FPBits(func(aNaN, sNaN)).uintval());
+    EXPECT_EQ(FPBits(aNaN).uintval(), FPBits(func(sNaN, aNaN)).uintval());
+    EXPECT_EQ(FPBits(aNaN).uintval(), FPBits(func(sNaN, sNaN)).uintval());
+  }
+
+  void testInfArg(FMinimumMagNumFunc func) {
+    EXPECT_FP_EQ(neg_inf, func(neg_inf, inf));
+    EXPECT_FP_EQ(0.0, func(inf, 0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, inf));
+    EXPECT_FP_EQ(T(1.2345), func(inf, T(1.2345)));
+    EXPECT_FP_EQ(T(-1.2345), func(T(-1.2345), inf));
+  }
+
+  void testNegInfArg(FMinimumMagNumFunc func) {
+    EXPECT_FP_EQ(neg_inf, func(inf, neg_inf));
+    EXPECT_FP_EQ(0.0, func(neg_inf, 0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, neg_inf));
+    EXPECT_FP_EQ(T(-1.2345), func(neg_inf, T(-1.2345)));
+    EXPECT_FP_EQ(T(1.2345), func(T(1.2345), neg_inf));
+  }
+
+  void testBothZero(FMinimumMagNumFunc func) {
+    EXPECT_FP_EQ(0.0, func(0.0, 0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, 0.0));
+    EXPECT_FP_EQ(-0.0, func(0.0, -0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, -0.0));
+  }
+
+  void testRange(FMinimumMagNumFunc func) {
+    constexpr StorageType COUNT = 100'001;
+    constexpr StorageType STEP = STORAGE_MAX / COUNT;
+    for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT;
+         ++i, v += STEP, w -= STEP) {
+      FPBits xbits(v), ybits(w);
+      if (xbits.is_inf_or_nan())
+        continue;
+      if (ybits.is_inf_or_nan())
+        continue;
+      T x = xbits.get_val();
+      T y = ybits.get_val();
+      if ((x == 0) && (y == 0))
+        continue;
+
+      if (LIBC_NAMESPACE::fputil::abs(x) > LIBC_NAMESPACE::fputil::abs(y)) {
+        EXPECT_FP_EQ(y, func(x, y));
+      } else {
+        EXPECT_FP_EQ(x, func(x, y));
+      }
+    }
+  }
+};
+
+#define LIST_FMINIMUM_MAG_NUM_TESTS(T, func)                                   \
+  using LlvmLibcFMinimumMagNumTest = FMinimumMagNumTest<T>;                    \
+  TEST_F(LlvmLibcFMinimumMagNumTest, NaN) { testNaN(&func); }                  \
+  TEST_F(LlvmLibcFMinimumMagNumTest, InfArg) { testInfArg(&func); }            \
+  TEST_F(LlvmLibcFMinimumMagNumTest, NegInfArg) { testNegInfArg(&func); }      \
+  TEST_F(LlvmLibcFMinimumMagNumTest, BothZero) { testBothZero(&func); }        \
+  TEST_F(LlvmLibcFMinimumMagNumTest, Range) { testRange(&func); }
+
+#endif // LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMMAG_NUMTEST_H
diff --git a/libc/test/src/math/smoke/FMinimumMagTest.h b/libc/test/src/math/smoke/FMinimumMagTest.h
new file mode 100644
index 000000000000..b11092e5379b
--- /dev/null
+++ b/libc/test/src/math/smoke/FMinimumMagTest.h
@@ -0,0 +1,89 @@
+//===-- Utility class to test fminimum_mag[f|l] -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUM_MAGTEST_H
+#define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUM_MAGTEST_H
+
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+template <typename T>
+class FMinimumMagTest : public LIBC_NAMESPACE::testing::Test {
+
+  DECLARE_SPECIAL_CONSTANTS(T)
+
+public:
+  typedef T (*FMinimumMagFunc)(T, T);
+
+  void testNaN(FMinimumMagFunc func) {
+    EXPECT_FP_EQ(aNaN, func(aNaN, inf));
+    EXPECT_FP_EQ(aNaN, func(neg_inf, aNaN));
+    EXPECT_FP_EQ(aNaN, func(aNaN, 0.0));
+    EXPECT_FP_EQ(aNaN, func(-0.0, aNaN));
+    EXPECT_FP_EQ(aNaN, func(aNaN, T(-1.2345)));
+    EXPECT_FP_EQ(aNaN, func(T(1.2345), aNaN));
+    EXPECT_FP_EQ(aNaN, func(aNaN, aNaN));
+  }
+
+  void testInfArg(FMinimumMagFunc func) {
+    EXPECT_FP_EQ(neg_inf, func(neg_inf, inf));
+    EXPECT_FP_EQ(0.0, func(inf, 0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, inf));
+    EXPECT_FP_EQ(T(1.2345), func(inf, T(1.2345)));
+    EXPECT_FP_EQ(T(-1.2345), func(T(-1.2345), inf));
+  }
+
+  void testNegInfArg(FMinimumMagFunc func) {
+    EXPECT_FP_EQ(neg_inf, func(inf, neg_inf));
+    EXPECT_FP_EQ(0.0, func(neg_inf, 0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, neg_inf));
+    EXPECT_FP_EQ(T(-1.2345), func(neg_inf, T(-1.2345)));
+    EXPECT_FP_EQ(T(1.2345), func(T(1.2345), neg_inf));
+  }
+
+  void testBothZero(FMinimumMagFunc func) {
+    EXPECT_FP_EQ(0.0, func(0.0, 0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, 0.0));
+    EXPECT_FP_EQ(-0.0, func(0.0, -0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, -0.0));
+  }
+
+  void testRange(FMinimumMagFunc func) {
+    constexpr StorageType COUNT = 100'001;
+    constexpr StorageType STEP = STORAGE_MAX / COUNT;
+    for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT;
+         ++i, v += STEP, w -= STEP) {
+      FPBits xbits(v), ybits(w);
+      if (xbits.is_inf_or_nan())
+        continue;
+      if (ybits.is_inf_or_nan())
+        continue;
+      T x = xbits.get_val();
+      T y = ybits.get_val();
+      if ((x == 0) && (y == 0))
+        continue;
+
+      if (LIBC_NAMESPACE::fputil::abs(x) < LIBC_NAMESPACE::fputil::abs(y)) {
+        EXPECT_FP_EQ(x, func(x, y));
+      } else {
+        EXPECT_FP_EQ(y, func(x, y));
+      }
+    }
+  }
+};
+
+#define LIST_FMINIMUM_MAG_TESTS(T, func)                                       \
+  using LlvmLibcFMinimumMagTest = FMinimumMagTest<T>;                          \
+  TEST_F(LlvmLibcFMinimumMagTest, NaN) { testNaN(&func); }                     \
+  TEST_F(LlvmLibcFMinimumMagTest, InfArg) { testInfArg(&func); }               \
+  TEST_F(LlvmLibcFMinimumMagTest, NegInfArg) { testNegInfArg(&func); }         \
+  TEST_F(LlvmLibcFMinimumMagTest, BothZero) { testBothZero(&func); }           \
+  TEST_F(LlvmLibcFMinimumMagTest, Range) { testRange(&func); }
+
+#endif // LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUM_MAGTEST_H
diff --git a/libc/test/src/math/smoke/FMinimumNumTest.h b/libc/test/src/math/smoke/FMinimumNumTest.h
new file mode 100644
index 000000000000..7fcc291b4c00
--- /dev/null
+++ b/libc/test/src/math/smoke/FMinimumNumTest.h
@@ -0,0 +1,100 @@
+//===-- Utility class to test fminimum_num[f|l] -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMNUMTEST_H
+#define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMNUMTEST_H
+
+#include "src/__support/FPUtil/FPBits.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+template <typename T>
+class FMinimumNumTest : public LIBC_NAMESPACE::testing::Test {
+
+  DECLARE_SPECIAL_CONSTANTS(T)
+
+public:
+  typedef T (*FMinimumNumFunc)(T, T);
+
+  void testNaN(FMinimumNumFunc func) {
+    EXPECT_FP_EQ(inf, func(aNaN, inf));
+    EXPECT_FP_EQ_WITH_EXCEPTION(inf, func(sNaN, inf), FE_INVALID);
+    EXPECT_FP_EQ(neg_inf, func(neg_inf, aNaN));
+    EXPECT_FP_EQ_WITH_EXCEPTION(neg_inf, func(neg_inf, sNaN), FE_INVALID);
+    EXPECT_EQ(FPBits(aNaN).uintval(), FPBits(func(aNaN, aNaN)).uintval());
+    EXPECT_FP_EQ(0.0, func(aNaN, 0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, aNaN));
+    EXPECT_FP_EQ_WITH_EXCEPTION(0.0, func(sNaN, 0.0), FE_INVALID);
+    EXPECT_FP_EQ_WITH_EXCEPTION(-0.0, func(-0.0, sNaN), FE_INVALID);
+    EXPECT_FP_EQ(T(-1.2345), func(aNaN, T(-1.2345)));
+    EXPECT_FP_EQ(T(1.2345), func(T(1.2345), aNaN));
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-1.2345), func(sNaN, T(-1.2345)), FE_INVALID);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(1.2345), func(T(1.2345), sNaN), FE_INVALID);
+    EXPECT_FP_IS_NAN_WITH_EXCEPTION(func(aNaN, sNaN), FE_INVALID);
+    EXPECT_FP_IS_NAN_WITH_EXCEPTION(func(sNaN, aNaN), FE_INVALID);
+    EXPECT_EQ(FPBits(aNaN).uintval(), FPBits(func(aNaN, sNaN)).uintval());
+    EXPECT_EQ(FPBits(aNaN).uintval(), FPBits(func(sNaN, aNaN)).uintval());
+    EXPECT_EQ(FPBits(aNaN).uintval(), FPBits(func(sNaN, sNaN)).uintval());
+  }
+
+  void testInfArg(FMinimumNumFunc func) {
+    EXPECT_FP_EQ(neg_inf, func(neg_inf, inf));
+    EXPECT_FP_EQ(0.0, func(inf, 0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, inf));
+    EXPECT_FP_EQ(T(1.2345), func(inf, T(1.2345)));
+    EXPECT_FP_EQ(T(-1.2345), func(T(-1.2345), inf));
+  }
+
+  void testNegInfArg(FMinimumNumFunc func) {
+    EXPECT_FP_EQ(neg_inf, func(inf, neg_inf));
+    EXPECT_FP_EQ(neg_inf, func(neg_inf, 0.0));
+    EXPECT_FP_EQ(neg_inf, func(-0.0, neg_inf));
+    EXPECT_FP_EQ(neg_inf, func(neg_inf, T(-1.2345)));
+    EXPECT_FP_EQ(neg_inf, func(T(1.2345), neg_inf));
+  }
+
+  void testBothZero(FMinimumNumFunc func) {
+    EXPECT_FP_EQ(0.0, func(0.0, 0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, 0.0));
+    EXPECT_FP_EQ(-0.0, func(0.0, -0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, -0.0));
+  }
+
+  void testRange(FMinimumNumFunc func) {
+    constexpr StorageType COUNT = 100'001;
+    constexpr StorageType STEP = STORAGE_MAX / COUNT;
+    for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT;
+         ++i, v += STEP, w -= STEP) {
+      FPBits xbits(v), ybits(w);
+      if (xbits.is_inf_or_nan())
+        continue;
+      if (ybits.is_inf_or_nan())
+        continue;
+      T x = xbits.get_val();
+      T y = ybits.get_val();
+      if ((x == 0) && (y == 0))
+        continue;
+
+      if (x > y) {
+        EXPECT_FP_EQ(y, func(x, y));
+      } else {
+        EXPECT_FP_EQ(x, func(x, y));
+      }
+    }
+  }
+};
+
+#define LIST_FMINIMUM_NUM_TESTS(T, func)                                       \
+  using LlvmLibcFMinimumNumTest = FMinimumNumTest<T>;                          \
+  TEST_F(LlvmLibcFMinimumNumTest, NaN) { testNaN(&func); }                     \
+  TEST_F(LlvmLibcFMinimumNumTest, InfArg) { testInfArg(&func); }               \
+  TEST_F(LlvmLibcFMinimumNumTest, NegInfArg) { testNegInfArg(&func); }         \
+  TEST_F(LlvmLibcFMinimumNumTest, BothZero) { testBothZero(&func); }           \
+  TEST_F(LlvmLibcFMinimumNumTest, Range) { testRange(&func); }
+
+#endif // LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMNUMTEST_H
diff --git a/libc/test/src/math/smoke/FMinimumTest.h b/libc/test/src/math/smoke/FMinimumTest.h
new file mode 100644
index 000000000000..bc04a6d99356
--- /dev/null
+++ b/libc/test/src/math/smoke/FMinimumTest.h
@@ -0,0 +1,88 @@
+//===-- Utility class to test fminimum[f|l] ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMTEST_H
+#define LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMTEST_H
+
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+template <typename T>
+class FMinimumTest : public LIBC_NAMESPACE::testing::Test {
+
+  DECLARE_SPECIAL_CONSTANTS(T)
+
+public:
+  typedef T (*FMinimumFunc)(T, T);
+
+  void testNaN(FMinimumFunc func) {
+    EXPECT_FP_EQ(aNaN, func(aNaN, inf));
+    EXPECT_FP_EQ(aNaN, func(neg_inf, aNaN));
+    EXPECT_FP_EQ(aNaN, func(aNaN, 0.0));
+    EXPECT_FP_EQ(aNaN, func(-0.0, aNaN));
+    EXPECT_FP_EQ(aNaN, func(aNaN, T(-1.2345)));
+    EXPECT_FP_EQ(aNaN, func(T(1.2345), aNaN));
+    EXPECT_FP_EQ(aNaN, func(aNaN, aNaN));
+  }
+
+  void testInfArg(FMinimumFunc func) {
+    EXPECT_FP_EQ(neg_inf, func(neg_inf, inf));
+    EXPECT_FP_EQ(0.0, func(inf, 0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, inf));
+    EXPECT_FP_EQ(T(1.2345), func(inf, T(1.2345)));
+    EXPECT_FP_EQ(T(1.2345), func(T(1.2345), inf));
+  }
+
+  void testNegInfArg(FMinimumFunc func) {
+    EXPECT_FP_EQ(neg_inf, func(inf, neg_inf));
+    EXPECT_FP_EQ(neg_inf, func(neg_inf, 0.0));
+    EXPECT_FP_EQ(neg_inf, func(-0.0, neg_inf));
+    EXPECT_FP_EQ(neg_inf, func(neg_inf, T(-1.2345)));
+    EXPECT_FP_EQ(neg_inf, func(T(1.2345), neg_inf));
+  }
+
+  void testBothZero(FMinimumFunc func) {
+    EXPECT_FP_EQ(0.0, func(0.0, 0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, 0.0));
+    EXPECT_FP_EQ(-0.0, func(0.0, -0.0));
+    EXPECT_FP_EQ(-0.0, func(-0.0, -0.0));
+  }
+
+  void testRange(FMinimumFunc func) {
+    constexpr StorageType COUNT = 100'001;
+    constexpr StorageType STEP = STORAGE_MAX / COUNT;
+    for (StorageType i = 0, v = 0, w = STORAGE_MAX; i <= COUNT;
+         ++i, v += STEP, w -= STEP) {
+      FPBits xbits(v), ybits(w);
+      if (xbits.is_inf_or_nan())
+        continue;
+      if (ybits.is_inf_or_nan())
+        continue;
+      T x = xbits.get_val();
+      T y = ybits.get_val();
+      if ((x == 0) && (y == 0))
+        continue;
+
+      if (x > y) {
+        EXPECT_FP_EQ(y, func(x, y));
+      } else {
+        EXPECT_FP_EQ(x, func(x, y));
+      }
+    }
+  }
+};
+
+#define LIST_FMINIMUM_TESTS(T, func)                                           \
+  using LlvmLibcFMinimumTest = FMinimumTest<T>;                                \
+  TEST_F(LlvmLibcFMinimumTest, NaN) { testNaN(&func); }                        \
+  TEST_F(LlvmLibcFMinimumTest, InfArg) { testInfArg(&func); }                  \
+  TEST_F(LlvmLibcFMinimumTest, NegInfArg) { testNegInfArg(&func); }            \
+  TEST_F(LlvmLibcFMinimumTest, BothZero) { testBothZero(&func); }              \
+  TEST_F(LlvmLibcFMinimumTest, Range) { testRange(&func); }
+
+#endif // LLVM_LIBC_TEST_SRC_MATH_SMOKE_FMINIMUMTEST_H
diff --git a/libc/test/src/math/smoke/fmaximum_mag_num_test.cpp b/libc/test/src/math/smoke/fmaximum_mag_num_test.cpp
new file mode 100644
index 000000000000..16ec1b0ea343
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximum_mag_num_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximum_mag_num-------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#include "FMaximumMagNumTest.h"
+
+#include "src/math/fmaximum_mag_num.h"
+
+LIST_FMAXIMUM_MAG_NUM_TESTS(double, LIBC_NAMESPACE::fmaximum_mag_num)
diff --git a/libc/test/src/math/smoke/fmaximum_mag_numf128_test.cpp b/libc/test/src/math/smoke/fmaximum_mag_numf128_test.cpp
new file mode 100644
index 000000000000..17f2a24f7fd0
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximum_mag_numf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximum_mag_numf128---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumMagNumTest.h"
+
+#include "src/math/fmaximum_mag_numf128.h"
+
+LIST_FMAXIMUM_MAG_NUM_TESTS(float128, LIBC_NAMESPACE::fmaximum_mag_numf128)
diff --git a/libc/test/src/math/smoke/fmaximum_mag_numf_test.cpp b/libc/test/src/math/smoke/fmaximum_mag_numf_test.cpp
new file mode 100644
index 000000000000..a8a46f96970e
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximum_mag_numf_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximum_mag_numf------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumMagNumTest.h"
+
+#include "src/math/fmaximum_mag_numf.h"
+
+LIST_FMAXIMUM_MAG_NUM_TESTS(float, LIBC_NAMESPACE::fmaximum_mag_numf)
diff --git a/libc/test/src/math/smoke/fmaximum_mag_numl_test.cpp b/libc/test/src/math/smoke/fmaximum_mag_numl_test.cpp
new file mode 100644
index 000000000000..c03fa20bd367
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximum_mag_numl_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximum_mag_numl------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumMagNumTest.h"
+
+#include "src/math/fmaximum_mag_numl.h"
+
+LIST_FMAXIMUM_MAG_NUM_TESTS(long double, LIBC_NAMESPACE::fmaximum_mag_numl)
diff --git a/libc/test/src/math/smoke/fmaximum_mag_test.cpp b/libc/test/src/math/smoke/fmaximum_mag_test.cpp
new file mode 100644
index 000000000000..e70602f044ec
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximum_mag_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximum_mag-----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#include "FMaximumMagTest.h"
+
+#include "src/math/fmaximum_mag.h"
+
+LIST_FMAXIMUM_MAG_TESTS(double, LIBC_NAMESPACE::fmaximum_mag)
diff --git a/libc/test/src/math/smoke/fmaximum_magf128_test.cpp b/libc/test/src/math/smoke/fmaximum_magf128_test.cpp
new file mode 100644
index 000000000000..d7ae8ec8d7a0
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximum_magf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximum_magf128-------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumMagTest.h"
+
+#include "src/math/fmaximum_magf128.h"
+
+LIST_FMAXIMUM_MAG_TESTS(float128, LIBC_NAMESPACE::fmaximum_magf128)
diff --git a/libc/test/src/math/smoke/fmaximum_magf_test.cpp b/libc/test/src/math/smoke/fmaximum_magf_test.cpp
new file mode 100644
index 000000000000..efca320b2b37
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximum_magf_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximum_magf----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumMagTest.h"
+
+#include "src/math/fmaximum_magf.h"
+
+LIST_FMAXIMUM_MAG_TESTS(float, LIBC_NAMESPACE::fmaximum_magf)
diff --git a/libc/test/src/math/smoke/fmaximum_magl_test.cpp b/libc/test/src/math/smoke/fmaximum_magl_test.cpp
new file mode 100644
index 000000000000..16b420b8714b
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximum_magl_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximum_magl----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumMagTest.h"
+
+#include "src/math/fmaximum_magl.h"
+
+LIST_FMAXIMUM_MAG_TESTS(long double, LIBC_NAMESPACE::fmaximum_magl)
diff --git a/libc/test/src/math/smoke/fmaximum_num_test.cpp b/libc/test/src/math/smoke/fmaximum_num_test.cpp
new file mode 100644
index 000000000000..cb9afdf78b83
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximum_num_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximum_num-----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#include "FMaximumNumTest.h"
+
+#include "src/math/fmaximum_num.h"
+
+LIST_FMAXIMUM_NUM_TESTS(double, LIBC_NAMESPACE::fmaximum_num)
diff --git a/libc/test/src/math/smoke/fmaximum_numf128_test.cpp b/libc/test/src/math/smoke/fmaximum_numf128_test.cpp
new file mode 100644
index 000000000000..6855ea3b39a9
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximum_numf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximum_numf128-------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumNumTest.h"
+
+#include "src/math/fmaximum_numf128.h"
+
+LIST_FMAXIMUM_NUM_TESTS(float128, LIBC_NAMESPACE::fmaximum_numf128)
diff --git a/libc/test/src/math/smoke/fmaximum_numf_test.cpp b/libc/test/src/math/smoke/fmaximum_numf_test.cpp
new file mode 100644
index 000000000000..053d18803e07
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximum_numf_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximum_numf----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumNumTest.h"
+
+#include "src/math/fmaximum_numf.h"
+
+LIST_FMAXIMUM_NUM_TESTS(float, LIBC_NAMESPACE::fmaximum_numf)
diff --git a/libc/test/src/math/smoke/fmaximum_numl_test.cpp b/libc/test/src/math/smoke/fmaximum_numl_test.cpp
new file mode 100644
index 000000000000..bf9612c4111d
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximum_numl_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximum_numl----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumNumTest.h"
+
+#include "src/math/fmaximum_numl.h"
+
+LIST_FMAXIMUM_NUM_TESTS(long double, LIBC_NAMESPACE::fmaximum_numl)
diff --git a/libc/test/src/math/smoke/fmaximum_test.cpp b/libc/test/src/math/smoke/fmaximum_test.cpp
new file mode 100644
index 000000000000..990e0177d868
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximum_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximum---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#include "FMaximumTest.h"
+
+#include "src/math/fmaximum.h"
+
+LIST_FMAXIMUM_TESTS(double, LIBC_NAMESPACE::fmaximum)
diff --git a/libc/test/src/math/smoke/fmaximumf128_test.cpp b/libc/test/src/math/smoke/fmaximumf128_test.cpp
new file mode 100644
index 000000000000..7e0b97bcdd4d
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximumf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximumf128-----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumTest.h"
+
+#include "src/math/fmaximumf128.h"
+
+LIST_FMAXIMUM_TESTS(float128, LIBC_NAMESPACE::fmaximumf128)
diff --git a/libc/test/src/math/smoke/fmaximumf_test.cpp b/libc/test/src/math/smoke/fmaximumf_test.cpp
new file mode 100644
index 000000000000..a92bbd1446e7
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximumf_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximumf--------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumTest.h"
+
+#include "src/math/fmaximumf.h"
+
+LIST_FMAXIMUM_TESTS(float, LIBC_NAMESPACE::fmaximumf)
diff --git a/libc/test/src/math/smoke/fmaximuml_test.cpp b/libc/test/src/math/smoke/fmaximuml_test.cpp
new file mode 100644
index 000000000000..080847b9f4de
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximuml_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fmaximuml--------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumTest.h"
+
+#include "src/math/fmaximuml.h"
+
+LIST_FMAXIMUM_TESTS(long double, LIBC_NAMESPACE::fmaximuml)
diff --git a/libc/test/src/math/smoke/fminimum_mag_num_test.cpp b/libc/test/src/math/smoke/fminimum_mag_num_test.cpp
new file mode 100644
index 000000000000..471f2ceb3c2c
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimum_mag_num_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimum_mag_num-------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#include "FMinimumMagNumTest.h"
+
+#include "src/math/fminimum_mag_num.h"
+
+LIST_FMINIMUM_MAG_NUM_TESTS(double, LIBC_NAMESPACE::fminimum_mag_num)
diff --git a/libc/test/src/math/smoke/fminimum_mag_numf128_test.cpp b/libc/test/src/math/smoke/fminimum_mag_numf128_test.cpp
new file mode 100644
index 000000000000..f1db025f4895
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimum_mag_numf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimum_mag_numf128---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumMagNumTest.h"
+
+#include "src/math/fminimum_mag_numf128.h"
+
+LIST_FMINIMUM_MAG_NUM_TESTS(float128, LIBC_NAMESPACE::fminimum_mag_numf128)
diff --git a/libc/test/src/math/smoke/fminimum_mag_numf_test.cpp b/libc/test/src/math/smoke/fminimum_mag_numf_test.cpp
new file mode 100644
index 000000000000..773ba806b99e
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimum_mag_numf_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimum_mag_numf------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumMagNumTest.h"
+
+#include "src/math/fminimum_mag_numf.h"
+
+LIST_FMINIMUM_MAG_NUM_TESTS(float, LIBC_NAMESPACE::fminimum_mag_numf)
diff --git a/libc/test/src/math/smoke/fminimum_mag_numl_test.cpp b/libc/test/src/math/smoke/fminimum_mag_numl_test.cpp
new file mode 100644
index 000000000000..b4152779db73
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimum_mag_numl_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimum_mag_numl------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumMagNumTest.h"
+
+#include "src/math/fminimum_mag_numl.h"
+
+LIST_FMINIMUM_MAG_NUM_TESTS(long double, LIBC_NAMESPACE::fminimum_mag_numl)
diff --git a/libc/test/src/math/smoke/fminimum_mag_test.cpp b/libc/test/src/math/smoke/fminimum_mag_test.cpp
new file mode 100644
index 000000000000..f4138f3353d1
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimum_mag_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimum_mag-----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#include "FMinimumMagTest.h"
+
+#include "src/math/fminimum_mag.h"
+
+LIST_FMINIMUM_MAG_TESTS(double, LIBC_NAMESPACE::fminimum_mag)
diff --git a/libc/test/src/math/smoke/fminimum_magf128_test.cpp b/libc/test/src/math/smoke/fminimum_magf128_test.cpp
new file mode 100644
index 000000000000..010ee6e73bee
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimum_magf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimum_magf128-------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumMagTest.h"
+
+#include "src/math/fminimum_magf128.h"
+
+LIST_FMINIMUM_MAG_TESTS(float128, LIBC_NAMESPACE::fminimum_magf128)
diff --git a/libc/test/src/math/smoke/fminimum_magf_test.cpp b/libc/test/src/math/smoke/fminimum_magf_test.cpp
new file mode 100644
index 000000000000..aa2743148930
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimum_magf_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimum_magf----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumMagTest.h"
+
+#include "src/math/fminimum_magf.h"
+
+LIST_FMINIMUM_MAG_TESTS(float, LIBC_NAMESPACE::fminimum_magf)
diff --git a/libc/test/src/math/smoke/fminimum_magl_test.cpp b/libc/test/src/math/smoke/fminimum_magl_test.cpp
new file mode 100644
index 000000000000..c6fbf7df0e70
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimum_magl_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimum_magl----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumMagTest.h"
+
+#include "src/math/fminimum_magl.h"
+
+LIST_FMINIMUM_MAG_TESTS(long double, LIBC_NAMESPACE::fminimum_magl)
diff --git a/libc/test/src/math/smoke/fminimum_num_test.cpp b/libc/test/src/math/smoke/fminimum_num_test.cpp
new file mode 100644
index 000000000000..1be7ebb5fe8e
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimum_num_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimum_num-----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#include "FMinimumNumTest.h"
+
+#include "src/math/fminimum_num.h"
+
+LIST_FMINIMUM_NUM_TESTS(double, LIBC_NAMESPACE::fminimum_num)
diff --git a/libc/test/src/math/smoke/fminimum_numf128_test.cpp b/libc/test/src/math/smoke/fminimum_numf128_test.cpp
new file mode 100644
index 000000000000..d773d8973286
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimum_numf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimum_numf128-------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumNumTest.h"
+
+#include "src/math/fminimum_numf128.h"
+
+LIST_FMINIMUM_NUM_TESTS(float128, LIBC_NAMESPACE::fminimum_numf128)
diff --git a/libc/test/src/math/smoke/fminimum_numf_test.cpp b/libc/test/src/math/smoke/fminimum_numf_test.cpp
new file mode 100644
index 000000000000..9b60f39d55a8
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimum_numf_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimum_numf----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumNumTest.h"
+
+#include "src/math/fminimum_numf.h"
+
+LIST_FMINIMUM_NUM_TESTS(float, LIBC_NAMESPACE::fminimum_numf)
diff --git a/libc/test/src/math/smoke/fminimum_numl_test.cpp b/libc/test/src/math/smoke/fminimum_numl_test.cpp
new file mode 100644
index 000000000000..8a72cd8adc5b
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimum_numl_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimum_numl----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumNumTest.h"
+
+#include "src/math/fminimum_numl.h"
+
+LIST_FMINIMUM_NUM_TESTS(long double, LIBC_NAMESPACE::fminimum_numl)
diff --git a/libc/test/src/math/smoke/fminimum_test.cpp b/libc/test/src/math/smoke/fminimum_test.cpp
new file mode 100644
index 000000000000..7778f1c2311c
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimum_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimum---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#include "FMinimumTest.h"
+
+#include "src/math/fminimum.h"
+
+LIST_FMINIMUM_TESTS(double, LIBC_NAMESPACE::fminimum)
diff --git a/libc/test/src/math/smoke/fminimumf128_test.cpp b/libc/test/src/math/smoke/fminimumf128_test.cpp
new file mode 100644
index 000000000000..163090b8a9ec
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimumf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimumf128-----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumTest.h"
+
+#include "src/math/fminimumf128.h"
+
+LIST_FMINIMUM_TESTS(float128, LIBC_NAMESPACE::fminimumf128)
diff --git a/libc/test/src/math/smoke/fminimumf_test.cpp b/libc/test/src/math/smoke/fminimumf_test.cpp
new file mode 100644
index 000000000000..2ca0f2f35a7f
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimumf_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimumf--------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumTest.h"
+
+#include "src/math/fminimumf.h"
+
+LIST_FMINIMUM_TESTS(float, LIBC_NAMESPACE::fminimumf)
diff --git a/libc/test/src/math/smoke/fminimuml_test.cpp b/libc/test/src/math/smoke/fminimuml_test.cpp
new file mode 100644
index 000000000000..3c067ae23f45
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimuml_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for fminimuml--------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumTest.h"
+
+#include "src/math/fminimuml.h"
+
+LIST_FMINIMUM_TESTS(long double, LIBC_NAMESPACE::fminimuml)
-- 
GitLab


From 2d40f179124f874aca4cf1145fdbc42fb8fb17f3 Mon Sep 17 00:00:00 2001
From: Cyndy Ishida <cyndy_ishida@apple.com>
Date: Mon, 25 Mar 2024 12:38:27 -0700
Subject: [PATCH 177/404] [InstallAPI] Add --extra* and --exclude* cli options
 for header input (#86522)

InstallAPI takes a json list of headers that is typically generated from
a build system like Xcode based on a project's attributes. Sometimes,
maintainers may want to alter this for tapi input. Using e.g.
`--extra-public-headers`, users can manipulate what headers will be used
for TBD file generation.
---
 .../clang/Basic/DiagnosticInstallAPIKinds.td  |    3 +
 clang/include/clang/InstallAPI/HeaderFile.h   |   54 +-
 clang/include/clang/InstallAPI/MachO.h        |    1 +
 clang/lib/InstallAPI/Frontend.cpp             |    2 +
 clang/lib/InstallAPI/HeaderFile.cpp           |   51 +
 .../Inputs/Simple/Extra/SimpleExtraAPI1.h     |    1 +
 .../Inputs/Simple/Extra/SimpleExtraAPI2.h     |    1 +
 .../Simple/Simple.framework/Headers/Basic.h   |  103 +
 .../Simple.framework/Headers/External.h       |   19 +
 .../Simple/Simple.framework/Headers/Simple.h  |   45 +
 .../Simple.framework/Headers/SimpleAPI.h      |    1 +
 .../PrivateHeaders/SimplePrivate.h            |    5 +
 .../PrivateHeaders/SimplePrivateSPI.h         |    2 +
 .../test/InstallAPI/Inputs/Simple/Simple.yaml | 3196 +++++++++++++++++
 .../Inputs/Simple/SimpleInternalAPI.h         |    3 +
 .../Inputs/Simple/SimpleInternalAPI2.h        |    7 +
 .../Inputs/Simple/SimpleInternalSPI.h         |    5 +
 .../InstallAPI/extra-exclude-headers.test     |  207 ++
 .../tools/clang-installapi/InstallAPIOpts.td  |   32 +
 clang/tools/clang-installapi/Options.cpp      |  121 +
 clang/tools/clang-installapi/Options.h        |   21 +
 llvm/include/llvm/TextAPI/Utils.h             |    6 +
 llvm/lib/TextAPI/Utils.cpp                    |   46 +
 23 files changed, 3930 insertions(+), 2 deletions(-)
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI1.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI2.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Basic.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/External.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Simple.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/SimpleAPI.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivate.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivateSPI.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.yaml
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI2.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/SimpleInternalSPI.h
 create mode 100644 clang/test/InstallAPI/extra-exclude-headers.test

diff --git a/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td b/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td
index a4c6e630ac5f..27df731fa286 100644
--- a/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td
+++ b/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td
@@ -15,6 +15,9 @@ let CategoryName = "Command line" in {
 def err_cannot_write_file : Error<"cannot write file '%0': %1">;
 def err_no_install_name : Error<"no install name specified: add -install_name <path>">;
 def err_no_output_file: Error<"no output file specified">;
+def err_no_such_header_file : Error<"no such %select{public|private|project}1 header file: '%0'">;
+def warn_no_such_excluded_header_file : Warning<"no such excluded %select{public|private}0 header file: '%1'">, InGroup<InstallAPIViolation>;
+def warn_glob_did_not_match: Warning<"glob '%0' did not match any header file">, InGroup<InstallAPIViolation>;
 } // end of command line category.
 
 let CategoryName = "Verification" in {
diff --git a/clang/include/clang/InstallAPI/HeaderFile.h b/clang/include/clang/InstallAPI/HeaderFile.h
index 70e83bbb3e76..235b4da3add8 100644
--- a/clang/include/clang/InstallAPI/HeaderFile.h
+++ b/clang/include/clang/InstallAPI/HeaderFile.h
@@ -13,7 +13,9 @@
 #ifndef LLVM_CLANG_INSTALLAPI_HEADERFILE_H
 #define LLVM_CLANG_INSTALLAPI_HEADERFILE_H
 
+#include "clang/Basic/FileManager.h"
 #include "clang/Basic/LangStandard.h"
+#include "clang/InstallAPI/MachO.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Regex.h"
@@ -56,6 +58,10 @@ class HeaderFile {
   std::string IncludeName;
   /// Supported language mode for header.
   std::optional<clang::Language> Language;
+  /// Exclude header file from processing.
+  bool Excluded{false};
+  /// Add header file to processing.
+  bool Extra{false};
 
 public:
   HeaderFile() = delete;
@@ -71,17 +77,48 @@ public:
   StringRef getIncludeName() const { return IncludeName; }
   StringRef getPath() const { return FullPath; }
 
+  void setExtra(bool V = true) { Extra = V; }
+  void setExcluded(bool V = true) { Excluded = V; }
+  bool isExtra() const { return Extra; }
+  bool isExcluded() const { return Excluded; }
+
   bool useIncludeName() const {
     return Type != HeaderType::Project && !IncludeName.empty();
   }
 
   bool operator==(const HeaderFile &Other) const {
-    return std::tie(Type, FullPath, IncludeName, Language) ==
+    return std::tie(Type, FullPath, IncludeName, Language, Excluded, Extra) ==
            std::tie(Other.Type, Other.FullPath, Other.IncludeName,
-                    Other.Language);
+                    Other.Language, Other.Excluded, Other.Extra);
   }
 };
 
+/// Glob that represents a pattern of header files to retreive.
+class HeaderGlob {
+private:
+  std::string GlobString;
+  llvm::Regex Rule;
+  HeaderType Type;
+  bool FoundMatch{false};
+
+public:
+  HeaderGlob(StringRef GlobString, llvm::Regex &&, HeaderType Type);
+
+  /// Create a header glob from string for the header access level.
+  static llvm::Expected<std::unique_ptr<HeaderGlob>>
+  create(StringRef GlobString, HeaderType Type);
+
+  /// Query if provided header matches glob.
+  bool match(const HeaderFile &Header);
+
+  /// Query if a header was matched in the glob, used primarily for error
+  /// reporting.
+  bool didMatch() { return FoundMatch; }
+
+  /// Provide back input glob string.
+  StringRef str() { return GlobString; }
+};
+
 /// Assemble expected way header will be included by clients.
 /// As in what maps inside the brackets of `#include <IncludeName.h>`
 /// For example,
@@ -93,6 +130,19 @@ public:
 std::optional<std::string> createIncludeHeaderName(const StringRef FullPath);
 using HeaderSeq = std::vector<HeaderFile>;
 
+/// Determine if Path is a header file.
+/// It does not touch the file system.
+///
+/// \param  Path File path to file.
+bool isHeaderFile(StringRef Path);
+
+/// Given input directory, collect all header files.
+///
+/// \param FM FileManager for finding input files.
+/// \param Directory Path to directory file.
+llvm::Expected<PathSeq> enumerateFiles(clang::FileManager &FM,
+                                       StringRef Directory);
+
 } // namespace clang::installapi
 
 #endif // LLVM_CLANG_INSTALLAPI_HEADERFILE_H
diff --git a/clang/include/clang/InstallAPI/MachO.h b/clang/include/clang/InstallAPI/MachO.h
index f0dea8bbd24c..4961c596fd68 100644
--- a/clang/include/clang/InstallAPI/MachO.h
+++ b/clang/include/clang/InstallAPI/MachO.h
@@ -40,6 +40,7 @@ using SymbolSet = llvm::MachO::SymbolSet;
 using SimpleSymbol = llvm::MachO::SimpleSymbol;
 using FileType = llvm::MachO::FileType;
 using PackedVersion = llvm::MachO::PackedVersion;
+using PathSeq = llvm::MachO::PathSeq;
 using Target = llvm::MachO::Target;
 using TargetList = llvm::MachO::TargetList;
 
diff --git a/clang/lib/InstallAPI/Frontend.cpp b/clang/lib/InstallAPI/Frontend.cpp
index 12cd5fcbc22b..e07ccb14e0b8 100644
--- a/clang/lib/InstallAPI/Frontend.cpp
+++ b/clang/lib/InstallAPI/Frontend.cpp
@@ -138,6 +138,8 @@ std::unique_ptr<MemoryBuffer> createInputBuffer(InstallAPIContext &Ctx) {
   SmallString<4096> Contents;
   raw_svector_ostream OS(Contents);
   for (const HeaderFile &H : Ctx.InputHeaders) {
+    if (H.isExcluded())
+      continue;
     if (H.getType() != Ctx.Type)
       continue;
     if (Ctx.LangMode == Language::C || Ctx.LangMode == Language::CXX)
diff --git a/clang/lib/InstallAPI/HeaderFile.cpp b/clang/lib/InstallAPI/HeaderFile.cpp
index c2d8372741ee..0b7041ec8147 100644
--- a/clang/lib/InstallAPI/HeaderFile.cpp
+++ b/clang/lib/InstallAPI/HeaderFile.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/InstallAPI/HeaderFile.h"
+#include "llvm/TextAPI/Utils.h"
 
 using namespace llvm;
 namespace clang::installapi {
@@ -34,4 +35,54 @@ std::optional<std::string> createIncludeHeaderName(const StringRef FullPath) {
   return Matches[1].drop_front(Matches[1].rfind('/') + 1).str() + "/" +
          Matches[3].str();
 }
+
+bool isHeaderFile(StringRef Path) {
+  return StringSwitch<bool>(sys::path::extension(Path))
+      .Cases(".h", ".H", ".hh", ".hpp", ".hxx", true)
+      .Default(false);
+}
+
+llvm::Expected<PathSeq> enumerateFiles(FileManager &FM, StringRef Directory) {
+  PathSeq Files;
+  std::error_code EC;
+  auto &FS = FM.getVirtualFileSystem();
+  for (llvm::vfs::recursive_directory_iterator i(FS, Directory, EC), ie;
+       i != ie; i.increment(EC)) {
+    if (EC)
+      return errorCodeToError(EC);
+
+    // Skip files that do not exist. This usually happens for broken symlinks.
+    if (FS.status(i->path()) == std::errc::no_such_file_or_directory)
+      continue;
+
+    StringRef Path = i->path();
+    if (isHeaderFile(Path))
+      Files.emplace_back(Path);
+  }
+
+  return Files;
+}
+
+HeaderGlob::HeaderGlob(StringRef GlobString, Regex &&Rule, HeaderType Type)
+    : GlobString(GlobString), Rule(std::move(Rule)), Type(Type) {}
+
+bool HeaderGlob::match(const HeaderFile &Header) {
+  if (Header.getType() != Type)
+    return false;
+
+  bool Match = Rule.match(Header.getPath());
+  if (Match)
+    FoundMatch = true;
+  return Match;
+}
+
+Expected<std::unique_ptr<HeaderGlob>> HeaderGlob::create(StringRef GlobString,
+                                                         HeaderType Type) {
+  auto Rule = MachO::createRegexFromGlob(GlobString);
+  if (!Rule)
+    return Rule.takeError();
+
+  return std::make_unique<HeaderGlob>(GlobString, std::move(*Rule), Type);
+}
+
 } // namespace clang::installapi
diff --git a/clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI1.h b/clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI1.h
new file mode 100644
index 000000000000..83a5b9507de3
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI1.h
@@ -0,0 +1 @@
+extern int extraGlobalAPI1;
diff --git a/clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI2.h b/clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI2.h
new file mode 100644
index 000000000000..34fe3364bba8
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI2.h
@@ -0,0 +1 @@
+extern int extraGlobalAPI2;
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Basic.h b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Basic.h
new file mode 100644
index 000000000000..08412bb2de28
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Basic.h
@@ -0,0 +1,103 @@
+#import <Foundation/Foundation.h>
+
+// Basic class with no super class
+@interface Basic1
+@end
+
+@interface Basic2 : NSObject
+@end
+
+@interface Basic3 : NSObject
+@property BOOL property1;
+@property(readonly) BOOL property2;
+@property(getter=isProperty3) BOOL property3;
+@property BOOL dynamicProp;
+@end
+
+@interface Basic4 : NSObject {
+@public
+  BOOL ivar1;
+@protected
+  BOOL ivar2;
+@package
+  BOOL ivar3;
+@private
+  BOOL ivar4;
+}
+@end
+
+__attribute__((visibility("hidden"))) @interface Basic4_1 : NSObject {
+@public
+  BOOL ivar1;
+@protected
+  BOOL ivar2;
+@package
+  BOOL ivar3;
+@private
+  BOOL ivar4;
+}
+@end
+
+@interface Basic4_2 : NSObject {
+@private
+  BOOL ivar4;
+@package
+  BOOL ivar3;
+@protected
+  BOOL ivar2;
+@public
+  BOOL ivar1;
+}
+@end
+
+@interface Basic5 : NSObject
++ (void)aClassMethod;
+- (void)anInstanceMethod;
+@end
+
+@interface Basic6 : NSObject
+@end
+
+@interface Basic6 () {
+@public
+  BOOL ivar1;
+}
+@property BOOL property1;
+- (void)anInstanceMethodFromAnExtension;
+@end
+
+@interface Basic6 (Foo)
+@property BOOL property2;
+- (void)anInstanceMethodFromACategory;
+@end
+
+__attribute__((visibility("hidden")))
+@interface Basic7 : NSObject
+@end
+
+@interface Basic7 ()
+- (void) anInstanceMethodFromAnHiddenExtension;
+@end
+
+@interface Basic8 : NSObject
++ (void)useSameName;
+@end
+
+// Classes and protocols can have the same name. For now they would only clash
+// in the selector map if the protocl starts with '_'.
+@protocol _A
+- (void)aMethod;
+@end
+
+@interface A : NSObject
+- (void)aMethod NS_AVAILABLE(10_11, 9_0);
+- (void)bMethod NS_UNAVAILABLE;
+@end
+
+@interface Basic9 : NSObject
+@property(readonly) BOOL aProperty NS_AVAILABLE(10_10, 8_0);
+@end
+
+@interface Basic9 (deprecated)
+@property(readwrite) BOOL aProperty NS_DEPRECATED_MAC(10_8, 10_10);
+@end
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/External.h b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/External.h
new file mode 100644
index 000000000000..5dc3c92f34c2
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/External.h
@@ -0,0 +1,19 @@
+#import <Foundation/Foundation.h>
+
+// Sub-class an external defined ObjC Class.
+@interface ExternalManagedObject : NSManagedObject
+- (void)foo;
+@end
+
+// Add category to external defined ObjC Class.
+@interface NSManagedObject (Simple)
+- (int)supportsSimple;
+@end
+
+// CoreData Accessors are dynamically generated and have no implementation.
+@interface ExternalManagedObject (CoreDataGeneratedAccessors)
+- (void)addChildObject:(ExternalManagedObject *)value;
+- (void)removeChildObject:(ExternalManagedObject *)value;
+- (void)addChild:(NSSet *)values;
+- (void)removeChild:(NSSet *)values;
+@end
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Simple.h b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Simple.h
new file mode 100644
index 000000000000..12c77098a8d9
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Simple.h
@@ -0,0 +1,45 @@
+#import <Foundation/Foundation.h>
+
+// Useless forward declaration. This is used for testing.
+@class FooBar;
+@protocol FooProtocol;
+
+@protocol ForwardProcotol;
+
+// Test public global.
+extern int publicGlobalVariable;
+
+// Test weak public global.
+extern int weakPublicGlobalVariable __attribute__((weak));
+
+// Test public ObjC class
+@interface Simple : NSObject
+@end
+
+__attribute__((objc_exception))
+@interface Base : NSObject
+@end
+
+@interface SubClass : Base
+@end
+
+@protocol BaseProtocol
+- (void) baseMethod;
+@end
+
+NS_AVAILABLE(10_11, 9_0)
+@protocol FooProtocol <BaseProtocol>
+- (void) protocolMethod;
+@end
+
+@protocol BarProtocol
+- (void) barMethod;
+@end
+
+@interface FooClass <FooProtocol, BarProtocol>
+@end
+
+// Create an empty category conforms to a forward declared protocol.
+// <rdar://problem/35605892>
+@interface FooClass (Test) <ForwardProcotol>
+@end
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/SimpleAPI.h b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/SimpleAPI.h
new file mode 100644
index 000000000000..d953fac966da
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/SimpleAPI.h
@@ -0,0 +1 @@
+extern int otherFrameworkAPI;
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivate.h b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivate.h
new file mode 100644
index 000000000000..5a28cda3928e
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivate.h
@@ -0,0 +1,5 @@
+// Test private global variable.
+extern int privateGlobalVariable;
+
+// Test weak private global.
+extern int weakPrivateGlobalVariable __attribute__((weak));
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivateSPI.h b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivateSPI.h
new file mode 100644
index 000000000000..c9aca30fa82f
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivateSPI.h
@@ -0,0 +1,2 @@
+// Test private global variable.
+extern int otherFrameworkSPI;
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.yaml b/clang/test/InstallAPI/Inputs/Simple/Simple.yaml
new file mode 100644
index 000000000000..998e51f1a67d
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/Simple.yaml
@@ -0,0 +1,3196 @@
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x1000007
+  cpusubtype:      0x3
+  filetype:        0x6
+  ncmds:           15
+  sizeofcmds:      1952
+  flags:           0x118085
+  reserved:        0x0
+LoadCommands:
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         472
+    segname:         __TEXT
+    vmaddr:          0
+    vmsize:          12288
+    fileoff:         0
+    filesize:        12288
+    maxprot:         5
+    initprot:        5
+    nsects:          5
+    flags:           0
+    Sections:
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x1BC0
+        size:            180
+        offset:          0x1BC0
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         554889E50FBE47085DC3554889E58857085DC3554889E50FBE47095DC3554889E50FBE470A5DC3554889E588570A5DC3554889E55DC3554889E55DC3554889E55DC3554889E50FBE47095DC3554889E58857095DC3554889E5B8010000005DC3554889E55DC3554889E55DC3554889E55DC3554889E55DC3554889E5B0015DC3554889E55DC3554889E55DC3554889E55DC3554889E50FBE47085DC3554889E55DC3554889E55DC3554889E55DC3554889E55DC3
+      - sectname:        __cstring
+        segname:         __TEXT
+        addr:            0x1C74
+        size:            296
+        offset:          0x1C74
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         53696D706C65004261736500537562436C6173730053696D706C65496E7465726E616C4150490053696D706C65496E7465726E616C53504900426173696331004261736963320042617369633300426173696334004261736963345F31004261736963345F32004261736963350042617369633600466F6F004261736963370045787465726E616C4D616E616765644F626A6563740048696464656E436C61737300426173696338004100426173696339006465707265636174656400466F6F436C61737300466F6F50726F746F636F6C004261736550726F746F636F6C0042617250726F746F636F6C0050726976617465005072697661746550726F746F636F6C0063313640303A380076323040303A3863313600630076313640303A380042313640303A3800
+      - sectname:        __objc_methname
+        segname:         __TEXT
+        addr:            0x1D9C
+        size:            450
+        offset:          0x1D9C
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         70726F7065727479310073657450726F7065727479313A0070726F70657274793200697350726F7065727479330073657450726F7065727479333A0070726F7065727479330054632C5670726F7065727479310054632C522C5670726F7065727479320054632C47697350726F7065727479332C5670726F7065727479330064796E616D696350726F700054632C440069766172310069766172320069766172330069766172340061436C6173734D6574686F6400616E496E7374616E63654D6574686F6400616E496E7374616E63654D6574686F6446726F6D416E457874656E73696F6E0073657450726F7065727479323A00616E496E7374616E63654D6574686F6446726F6D4143617465676F727900546300616E496E7374616E63654D6574686F6446726F6D416E48696464656E457874656E73696F6E00666F6F00737570706F72747353696D706C650075736553616D654E616D6500614D6574686F64006150726F7065727479005F6150726F70657274790054632C522C565F6150726F706572747900626173654D6574686F640070726F746F636F6C4D6574686F64006261724D6574686F64007072697661746550726F636F746F6C4D6574686F6400
+      - sectname:        __unwind_info
+        segname:         __TEXT
+        addr:            0x1F60
+        size:            4152
+        offset:          0x1F60
+        align:           2
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         010000001C000000010000002000000000000000200000000200000000000001C01B00003800000038000000741C00000000000038000000030000000C0001001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+      - sectname:        __eh_frame
+        segname:         __TEXT
+        addr:            0x2F98
+        size:            24
+        offset:          0x2F98
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x6000000B
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         1400000000000000017A520001781001100C070890010000
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         792
+    segname:         __DATA
+    vmaddr:          12288
+    vmsize:          8192
+    fileoff:         12288
+    filesize:        8192
+    maxprot:         3
+    initprot:        3
+    nsects:          9
+    flags:           0
+    Sections:
+      - sectname:        __objc_const
+        segname:         __DATA
+        addr:            0x3000
+        size:            4952
+        offset:          0x3000
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         010000002800000028000000000000000000000000000000741C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000008000000000000000000000000000000741C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000028000000280000000000000000000000000000007B1C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000008000000080000000000000000000000000000007B1C0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000007B1C000000000000D043000000000000010000002800000028000000000000000000000000000000801C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000800000008000000000000000000000000000000801C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000801C0000000000002044000000000000010000002800000028000000000000000000000000000000891C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000800000008000000000000000000000000000000891C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000891C00000000000070440000000000000100000028000000280000000000000000000000000000009B1C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000008000000080000000000000000000000000000009B1C0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000009B1C000000000000C044000000000000030000002800000028000000000000000000000000000000AD1C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000AD1C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000002800000028000000000000000000000000000000B41C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000008000000000000000000000000000000B41C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000002800000028000000000000000000000000000000BB1C0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000018000000050000009C1D000000000000771D000000000000C01B000000000000A61D0000000000007F1D000000000000CA1B000000000000B41D000000000000771D000000000000D31B000000000000BE1D000000000000771D000000000000DD1B000000000000CA1D0000000000007F1D000000000000E71B000000000000200000000300000098490000000000009C1D0000000000008A1D0000000000000000000001000000A049000000000000B41D0000000000008A1D0000000000000000000001000000A849000000000000D81D0000000000008A1D000000000000000000000100000010000000040000009C1D000000000000E21D000000000000B41D000000000000F01D000000000000D81D000000000000001E0000000000001B1E000000000000271E00000000000000000000080000000B000000000000000000000000000000BB1C00000000000098340000000000000000000000000000183500000000000000000000000000008035000000000000010000002800000028000000000000000000000000000000C21C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000004000000B0490000000000002C1E0000000000008A1D0000000000000000000001000000B849000000000000321E0000000000008A1D0000000000000000000001000000C049000000000000381E0000000000008A1D0000000000000000000001000000C8490000000000003E1E0000000000008A1D000000000000000000000100000000000000080000000C000000000000000000000000000000C21C00000000000000000000000000000000000000000000583600000000000000000000000000000000000000000000110000002800000028000000000000000000000000000000C91C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000004000000D0490000000000002C1E0000000000008A1D0000000000000000000001000000D849000000000000321E0000000000008A1D0000000000000000000001000000E049000000000000381E0000000000008A1D0000000000000000000001000000E8490000000000003E1E0000000000008A1D000000000000000000000100000010000000080000000C000000000000000000000000000000C91C00000000000000000000000000000000000000000000703700000000000000000000000000000000000000000000010000002800000028000000000000000000000000000000D21C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000004000000F0490000000000003E1E0000000000008A1D0000000000000000000001000000F849000000000000381E0000000000008A1D0000000000000000000001000000004A000000000000321E0000000000008A1D0000000000000000000001000000084A0000000000002C1E0000000000008A1D000000000000000000000100000000000000080000000C000000000000000000000000000000D21C000000000000000000000000000000000000000000008838000000000000000000000000000000000000000000001800000001000000441E0000000000008C1D000000000000F01B000000000000010000002800000028000000000000000000000000000000DB1C000000000000583900000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000511E0000000000008C1D000000000000F61B000000000000000000000800000008000000000000000000000000000000DB1C000000000000C0390000000000000000000000000000000000000000000000000000000000000000000000000000010000002800000028000000000000000000000000000000E21C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001800000003000000621E0000000000008C1D000000000000FC1B0000000000009C1D000000000000771D000000000000021C000000000000A61D0000000000007F1D0000000000000C1C0000000000002000000002000000104A0000000000002C1E0000000000008A1D0000000000000000000001000000184A0000000000009C1D0000000000008A1D000000000000000000000100000010000000010000009C1D000000000000E21D00000000000000000000080000000A000000000000000000000000000000E21C000000000000703A0000000000000000000000000000C03A0000000000000000000000000000083B0000000000001800000003000000B41D000000000000771D000000000000151C000000000000821E0000000000007F1D000000000000201C000000000000901E0000000000008C1D000000000000261C0000000000001000000001000000B41D000000000000AE1E000000000000E91C0000000000004047000000000000683B00000000000000000000000000000000000000000000B83B00000000000000000000000000004000000000000000110000002800000028000000000000000000000000000000ED1C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000B11E0000000000008C1D0000000000002C1C000000000000100000000800000008000000000000000000000000000000ED1C000000000000583C0000000000000000000000000000000000000000000000000000000000000000000000000000010000002800000028000000000000000000000000000000F41C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000D71E0000000000008C1D000000000000321C000000000000000000000800000008000000000000000000000000000000F41C000000000000083D00000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000DB1E000000000000941D000000000000381C000000000000741C0000000000000000000000000000703D000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000001100000028000000280000000000000000000000000000000A1D000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000008000000080000000000000000000000000000000A1D000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000EA1E0000000000008C1D000000000000401C000000000000010000002800000028000000000000000000000000000000161D000000000000603E00000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000EA1E0000000000008C1D000000000000461C000000000000000000000800000008000000000000000000000000000000161D000000000000C83E00000000000000000000000000000000000000000000000000000000000000000000000000000100000028000000280000000000000000000000000000001D1D000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000F61E0000000000008C1D0000000000004C1C0000000000000000000008000000080000000000000000000000000000001D1D000000000000783F00000000000000000000000000000000000000000000000000000000000000000000000000000100000028000000280000000000000000000000000000001F1D000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000FE1E000000000000771D000000000000521C0000000000002000000001000000204A000000000000081F0000000000008A1D00000000000000000000010000001000000001000000FE1E000000000000131F0000000000000000000008000000090000000000000000000000000000001F1D000000000000284000000000000000000000000000004840000000000000000000000000000070400000000000001000000001000000FE1E000000000000AE1E000000000000261D0000000000002049000000000000000000000000000000000000000000000000000000000000D040000000000000000000000000000040000000000000001800000001000000241F0000000000008C1D00000000000000000000000000008C1D0000000000000100000000000000284A000000000000000000000000000018000000010000002F1F0000000000008C1D00000000000000000000000000008C1D00000000000018000000010000003E1F0000000000008C1D00000000000000000000000000008C1D0000000000000200000000000000884A000000000000E84A0000000000000000000000000000030000002800000028000000000000000000000000000000311D0000000000000000000000000000B8410000000000000000000000000000000000000000000000000000000000001800000003000000241F0000000000008C1D0000000000005C1C0000000000002F1F0000000000008C1D000000000000621C0000000000003E1F0000000000008C1D000000000000681C000000000000020000000000000000000000000000000000000000000000311D0000000000002042000000000000B8410000000000000000000000000000000000000000000000000000000000001800000001000000481F0000000000008C1D0000000000006E1C0000000000001800000001000000481F0000000000008C1D00000000000000000000000000008C1D0000000000000100000000000000484B00000000000000000000000000005F1D0000000000004849000000000000B84200000000000000000000000000000043000000000000000000000000000000000000000000004000000000000000
+      - sectname:        __objc_data
+        segname:         __DATA
+        addr:            0x4358
+        size:            1600
+        offset:          0x4358
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         000000000000000000000000000000000000000000000000000000000000000000300000000000005843000000000000000000000000000000000000000000000000000000000000483000000000000000000000000000000000000000000000000000000000000000000000000000009030000000000000A843000000000000000000000000000000000000000000000000000000000000D8300000000000000000000000000000A843000000000000000000000000000000000000000000003831000000000000F843000000000000D0430000000000000000000000000000000000000000000080310000000000000000000000000000000000000000000000000000000000000000000000000000E03100000000000048440000000000000000000000000000000000000000000000000000000000002832000000000000000000000000000000000000000000000000000000000000000000000000000088320000000000009844000000000000000000000000000000000000000000000000000000000000D032000000000000104500000000000000000000000000000000000000000000000000000000000078330000000000001045000000000000E8440000000000000000000000000000000000000000000030330000000000000000000000000000000000000000000000000000000000000000000000000000C03300000000000038450000000000000000000000000000000000000000000000000000000000000834000000000000000000000000000000000000000000000000000000000000000000000000000050340000000000008845000000000000000000000000000000000000000000000000000000000000C83500000000000000000000000000000000000000000000000000000000000000000000000000001036000000000000D845000000000000000000000000000000000000000000000000000000000000E036000000000000000000000000000000000000000000000000000000000000000000000000000028370000000000002846000000000000000000000000000000000000000000000000000000000000F837000000000000000000000000000000000000000000000000000000000000000000000000000040380000000000007846000000000000000000000000000000000000000000000000000000000000103900000000000000000000000000000000000000000000000000000000000000000000000000007839000000000000C846000000000000000000000000000000000000000000000000000000000000E0390000000000000000000000000000000000000000000000000000000000000000000000000000283A0000000000001847000000000000000000000000000000000000000000000000000000000000203B0000000000000000000000000000000000000000000000000000000000000000000000000000103C0000000000006847000000000000000000000000000000000000000000000000000000000000783C0000000000000000000000000000000000000000000000000000000000000000000000000000C03C000000000000B847000000000000000000000000000000000000000000000000000000000000283D0000000000000000000000000000000000000000000000000000000000000000000000000000D03D0000000000000848000000000000000000000000000000000000000000000000000000000000183E0000000000000000000000000000000000000000000000000000000000000000000000000000803E0000000000005848000000000000000000000000000000000000000000000000000000000000E83E0000000000000000000000000000000000000000000000000000000000000000000000000000303F000000000000A848000000000000000000000000000000000000000000000000000000000000983F0000000000000000000000000000000000000000000000000000000000000000000000000000E03F000000000000F8480000000000000000000000000000000000000000000000000000000000008840000000000000704900000000000000000000000000000000000000000000000000000000000070420000000000007049000000000000484900000000000000000000000000000000000000000000D841000000000000
+      - sectname:        __objc_ivar
+        segname:         __DATA
+        addr:            0x4998
+        size:            144
+        offset:          0x4998
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         080000000000000009000000000000000A00000000000000080000000000000009000000000000000A000000000000000B00000000000000080000000000000009000000000000000A000000000000000B00000000000000080000000000000009000000000000000A000000000000000B00000000000000080000000000000009000000000000000800000000000000
+      - sectname:        __data
+        segname:         __DATA
+        addr:            0x4A28
+        size:            392
+        offset:          0x4A28
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         0000000000000000461D000000000000000000000000000028410000000000000000000000000000000000000000000000000000000000000000000000000000600000000000000048410000000000000000000000000000000000000000000000000000000000003A1D00000000000050410000000000006841000000000000000000000000000000000000000000000000000000000000000000000000000060000000000000008841000000000000000000000000000000000000000000000000000000000000531D0000000000000000000000000000904100000000000000000000000000000000000000000000000000000000000000000000000000006000000000000000B041000000000000000000000000000000000000000000000000000000000000671D0000000000000000000000000000D84200000000000000000000000000000000000000000000000000000000000000000000000000006000000000000000F842000000000000000000000000000000000000000000000000000000000000
+      - sectname:        __objc_protolist
+        segname:         __DATA
+        addr:            0x4BB0
+        size:            32
+        offset:          0x4BB0
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x1000000B
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         284A000000000000884A000000000000E84A000000000000484B000000000000
+      - sectname:        __objc_classlist
+        segname:         __DATA
+        addr:            0x4BD0
+        size:            160
+        offset:          0x4BD0
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x10000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         8043000000000000D04300000000000020440000000000007044000000000000C044000000000000E8440000000000006045000000000000B04500000000000000460000000000005046000000000000A046000000000000F04600000000000040470000000000009047000000000000E04700000000000030480000000000008048000000000000D04800000000000020490000000000004849000000000000
+      - sectname:        __objc_catlist
+        segname:         __DATA
+        addr:            0x4C70
+        size:            32
+        offset:          0x4C70
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x10000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         D03B000000000000903D000000000000E8400000000000001843000000000000
+      - sectname:        __objc_imageinfo
+        segname:         __DATA
+        addr:            0x4C90
+        size:            8
+        offset:          0x4C90
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         '0000000040000000'
+      - sectname:        __common
+        segname:         __DATA
+        addr:            0x4C98
+        size:            16
+        offset:          0x0
+        align:           2
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x1
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __LINKEDIT
+    vmaddr:          20480
+    vmsize:          10272
+    fileoff:         20480
+    filesize:        10272
+    maxprot:         1
+    initprot:        1
+    nsects:          0
+    flags:           0
+  - cmd:             LC_DYLD_INFO_ONLY
+    cmdsize:         48
+    rebase_off:      20480
+    rebase_size:     320
+    bind_off:        20800
+    bind_size:       480
+    weak_bind_off:   0
+    weak_bind_size:  0
+    lazy_bind_off:   0
+    lazy_bind_size:  0
+    export_off:      21280
+    export_size:     896
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          22208
+    nsyms:           187
+    stroff:          25200
+    strsize:         5552
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       131
+    iextdefsym:      131
+    nextdefsym:      49
+    iundefsym:       180
+    nundefsym:       7
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+  - cmd:             LC_ID_DYLIB
+    cmdsize:         88
+    dylib:
+      name:            24
+      timestamp:       0
+      current_version: 66051
+      compatibility_version: 65536
+    Content:         '/System/Library/Frameworks/Simple.framework/Versions/A/Simple'
+    ZeroPadBytes:    3
+  - cmd:             LC_UUID
+    cmdsize:         24
+    uuid:            4C4C441D-5555-3144-A104-DD1AF4EF8FE7
+  - cmd:             LC_VERSION_MIN_MACOSX
+    cmdsize:         16
+    version:         658432
+    sdk:             983040
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         96
+    dylib:
+      name:            24
+      timestamp:       0
+      current_version: 197722368
+      compatibility_version: 19660800
+    Content:         '/System/Library/Frameworks/Foundation.framework/Versions/C/Foundation'
+    ZeroPadBytes:    3
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         56
+    dylib:
+      name:            24
+      timestamp:       0
+      current_version: 14942208
+      compatibility_version: 65536
+    Content:         '/usr/lib/libobjc.A.dylib'
+    ZeroPadBytes:    8
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         96
+    dylib:
+      name:            24
+      timestamp:       0
+      current_version: 91750400
+      compatibility_version: 65536
+    Content:         '/System/Library/Frameworks/CoreData.framework/Versions/A/CoreData'
+    ZeroPadBytes:    7
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         56
+    dylib:
+      name:            24
+      timestamp:       0
+      current_version: 88539136
+      compatibility_version: 65536
+    Content:         '/usr/lib/libSystem.B.dylib'
+    ZeroPadBytes:    6
+  - cmd:             LC_FUNCTION_STARTS
+    cmdsize:         16
+    dataoff:         22176
+    datasize:        32
+  - cmd:             LC_DATA_IN_CODE
+    cmdsize:         16
+    dataoff:         22208
+    datasize:        0
+LinkEditData:
+  RebaseOpcodes:
+    - Opcode:          REBASE_OPCODE_SET_TYPE_IMM
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB
+      Imm:             1
+      ExtraData:       [ 0x18 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x3, 0x40 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x40 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x40 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x40 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x4, 0x40 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             15
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             8
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x10 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x10 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x10 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             5
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             7
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             9
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             9
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             5
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x28 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             7
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             5
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x40 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             5
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             7
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             7
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x18 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x8 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x8 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             9
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x38 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x98 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x18 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             5
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x18 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x18 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x18 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES
+      Imm:             0
+      ExtraData:       [ 0x1C ]
+    - Opcode:          REBASE_OPCODE_DONE
+      Imm:             0
+  BindOpcodes:
+    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
+      Imm:             0
+      Symbol:          _objc_ehtype_vtable
+    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
+      Imm:             1
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_DYLIB_ORDINAL_IMM
+      Imm:             2
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB
+      Imm:             1
+      ULEBExtraData:   [ 0x120 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_ADDEND_SLEB
+      Imm:             0
+      SLEBExtraData:   [ 16 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0xA0 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0xA0 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0xA0 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
+      Imm:             0
+      Symbol:          '_OBJC_CLASS_$_NSManagedObject'
+    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
+      Imm:             1
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_DYLIB_ORDINAL_IMM
+      Imm:             3
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0xA78 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_ADDEND_SLEB
+      Imm:             0
+      SLEBExtraData:   [ 0 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0xA48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
+      Imm:             0
+      Symbol:          '_OBJC_METACLASS_$_NSObject'
+    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
+      Imm:             1
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_DYLIB_ORDINAL_IMM
+      Imm:             2
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0xFFFFFFFFFFFFFB68 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x90 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
+      Imm:             0
+      Symbol:          __objc_empty_cache
+    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
+      Imm:             1
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0xFFFFFFFFFFFFFA60 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
+      Imm:             0
+      Symbol:          '_OBJC_CLASS_$_NSObject'
+    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
+      Imm:             1
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0xFFFFFFFFFFFFFA00 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x98 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x98 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x98 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
+      Imm:             0
+      Symbol:          '_OBJC_METACLASS_$_NSManagedObject'
+    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
+      Imm:             1
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_DYLIB_ORDINAL_IMM
+      Imm:             3
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0xFFFFFFFFFFFFFE90 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DONE
+      Imm:             0
+      Symbol:          ''
+  ExportTrie:
+    TerminalSize:    0
+    NodeOffset:      0
+    Name:            ''
+    Flags:           0x0
+    Address:         0x0
+    Other:           0x0
+    ImportName:      ''
+    Children:
+      - TerminalSize:    0
+        NodeOffset:      5
+        Name:            _
+        Flags:           0x0
+        Address:         0x0
+        Other:           0x0
+        ImportName:      ''
+        Children:
+          - TerminalSize:    0
+            NodeOffset:      41
+            Name:            p
+            Flags:           0x0
+            Address:         0x0
+            Other:           0x0
+            ImportName:      ''
+            Children:
+              - TerminalSize:    4
+                NodeOffset:      86
+                Name:            rivateGlobalVariable
+                Flags:           0x0
+                Address:         0x4CA0
+                Other:           0x0
+                ImportName:      ''
+              - TerminalSize:    4
+                NodeOffset:      92
+                Name:            ublicGlobalVariable
+                Flags:           0x0
+                Address:         0x4CA4
+                Other:           0x0
+                ImportName:      ''
+          - TerminalSize:    0
+            NodeOffset:      98
+            Name:            extraGlobalAPI
+            Flags:           0x0
+            Address:         0x0
+            Other:           0x0
+            ImportName:      ''
+            Children:
+              - TerminalSize:    4
+                NodeOffset:      106
+                Name:            '1'
+                Flags:           0x0
+                Address:         0x4C98
+                Other:           0x0
+                ImportName:      ''
+              - TerminalSize:    4
+                NodeOffset:      112
+                Name:            '2'
+                Flags:           0x0
+                Address:         0x4C9C
+                Other:           0x0
+                ImportName:      ''
+          - TerminalSize:    0
+            NodeOffset:      118
+            Name:            weakP
+            Flags:           0x0
+            Address:         0x0
+            Other:           0x0
+            ImportName:      ''
+            Children:
+              - TerminalSize:    4
+                NodeOffset:      165
+                Name:            rivateGlobalVariable
+                Flags:           0x4
+                Address:         0x4BAC
+                Other:           0x0
+                ImportName:      ''
+              - TerminalSize:    4
+                NodeOffset:      171
+                Name:            ublicGlobalVariable
+                Flags:           0x4
+                Address:         0x4BA8
+                Other:           0x0
+                ImportName:      ''
+          - TerminalSize:    0
+            NodeOffset:      177
+            Name:            OBJC_
+            Flags:           0x0
+            Address:         0x0
+            Other:           0x0
+            ImportName:      ''
+            Children:
+              - TerminalSize:    0
+                NodeOffset:      232
+                Name:            'IVAR_$_Basic'
+                Flags:           0x0
+                Address:         0x0
+                Other:           0x0
+                ImportName:      ''
+                Children:
+                  - TerminalSize:    4
+                    NodeOffset:      248
+                    Name:            6.ivar1
+                    Flags:           0x0
+                    Address:         0x4A10
+                    Other:           0x0
+                    ImportName:      ''
+                  - TerminalSize:    0
+                    NodeOffset:      254
+                    Name:            '4'
+                    Flags:           0x0
+                    Address:         0x0
+                    Other:           0x0
+                    ImportName:      ''
+                    Children:
+                      - TerminalSize:    0
+                        NodeOffset:      274
+                        Name:            .ivar
+                        Flags:           0x0
+                        Address:         0x0
+                        Other:           0x0
+                        ImportName:      ''
+                        Children:
+                          - TerminalSize:    4
+                            NodeOffset:      284
+                            Name:            '2'
+                            Flags:           0x0
+                            Address:         0x49B8
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      290
+                            Name:            '1'
+                            Flags:           0x0
+                            Address:         0x49B0
+                            Other:           0x0
+                            ImportName:      ''
+                      - TerminalSize:    0
+                        NodeOffset:      296
+                        Name:            _2.ivar
+                        Flags:           0x0
+                        Address:         0x0
+                        Other:           0x0
+                        ImportName:      ''
+                        Children:
+                          - TerminalSize:    4
+                            NodeOffset:      306
+                            Name:            '1'
+                            Flags:           0x0
+                            Address:         0x4A08
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      312
+                            Name:            '2'
+                            Flags:           0x0
+                            Address:         0x4A00
+                            Other:           0x0
+                            ImportName:      ''
+              - TerminalSize:    0
+                NodeOffset:      318
+                Name:            'METACLASS_$_'
+                Flags:           0x0
+                Address:         0x0
+                Other:           0x0
+                ImportName:      ''
+                Children:
+                  - TerminalSize:    4
+                    NodeOffset:      369
+                    Name:            FooClass
+                    Flags:           0x0
+                    Address:         0x4970
+                    Other:           0x0
+                    ImportName:      ''
+                  - TerminalSize:    4
+                    NodeOffset:      375
+                    Name:            ExternalManagedObject
+                    Flags:           0x0
+                    Address:         0x47B8
+                    Other:           0x0
+                    ImportName:      ''
+                  - TerminalSize:    0
+                    NodeOffset:      381
+                    Name:            S
+                    Flags:           0x0
+                    Address:         0x0
+                    Other:           0x0
+                    ImportName:      ''
+                    Children:
+                      - TerminalSize:    4
+                        NodeOffset:      401
+                        Name:            imple
+                        Flags:           0x0
+                        Address:         0x4358
+                        Other:           0x0
+                        ImportName:      ''
+                        Children:
+                          - TerminalSize:    0
+                            NodeOffset:      418
+                            Name:            Internal
+                            Flags:           0x0
+                            Address:         0x0
+                            Other:           0x0
+                            ImportName:      ''
+                            Children:
+                              - TerminalSize:    4
+                                NodeOffset:      432
+                                Name:            SPI
+                                Flags:           0x0
+                                Address:         0x4498
+                                Other:           0x0
+                                ImportName:      ''
+                              - TerminalSize:    4
+                                NodeOffset:      438
+                                Name:            API
+                                Flags:           0x0
+                                Address:         0x4448
+                                Other:           0x0
+                                ImportName:      ''
+                      - TerminalSize:    4
+                        NodeOffset:      444
+                        Name:            ubClass
+                        Flags:           0x0
+                        Address:         0x43F8
+                        Other:           0x0
+                        ImportName:      ''
+                  - TerminalSize:    0
+                    NodeOffset:      450
+                    Name:            Bas
+                    Flags:           0x0
+                    Address:         0x0
+                    Other:           0x0
+                    ImportName:      ''
+                    Children:
+                      - TerminalSize:    4
+                        NodeOffset:      461
+                        Name:            e
+                        Flags:           0x0
+                        Address:         0x43A8
+                        Other:           0x0
+                        ImportName:      ''
+                      - TerminalSize:    0
+                        NodeOffset:      467
+                        Name:            ic
+                        Flags:           0x0
+                        Address:         0x0
+                        Other:           0x0
+                        ImportName:      ''
+                        Children:
+                          - TerminalSize:    4
+                            NodeOffset:      501
+                            Name:            '2'
+                            Flags:           0x0
+                            Address:         0x4538
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      507
+                            Name:            '3'
+                            Flags:           0x0
+                            Address:         0x4588
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      513
+                            Name:            '5'
+                            Flags:           0x0
+                            Address:         0x46C8
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      519
+                            Name:            '4'
+                            Flags:           0x0
+                            Address:         0x45D8
+                            Other:           0x0
+                            ImportName:      ''
+                            Children:
+                              - TerminalSize:    4
+                                NodeOffset:      530
+                                Name:            _2
+                                Flags:           0x0
+                                Address:         0x4678
+                                Other:           0x0
+                                ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      536
+                            Name:            '9'
+                            Flags:           0x0
+                            Address:         0x48F8
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      542
+                            Name:            '8'
+                            Flags:           0x0
+                            Address:         0x4858
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      548
+                            Name:            '6'
+                            Flags:           0x0
+                            Address:         0x4718
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      554
+                            Name:            '1'
+                            Flags:           0x0
+                            Address:         0x4510
+                            Other:           0x0
+                            ImportName:      ''
+                  - TerminalSize:    4
+                    NodeOffset:      560
+                    Name:            A
+                    Flags:           0x0
+                    Address:         0x48A8
+                    Other:           0x0
+                    ImportName:      ''
+              - TerminalSize:    0
+                NodeOffset:      566
+                Name:            'EHTYPE_$_'
+                Flags:           0x0
+                Address:         0x0
+                Other:           0x0
+                ImportName:      ''
+                Children:
+                  - TerminalSize:    3
+                    NodeOffset:      579
+                    Name:            Base
+                    Flags:           0x0
+                    Address:         0x3120
+                    Other:           0x0
+                    ImportName:      ''
+                  - TerminalSize:    0
+                    NodeOffset:      584
+                    Name:            S
+                    Flags:           0x0
+                    Address:         0x0
+                    Other:           0x0
+                    ImportName:      ''
+                    Children:
+                      - TerminalSize:    3
+                        NodeOffset:      612
+                        Name:            ubClass
+                        Flags:           0x0
+                        Address:         0x31C8
+                        Other:           0x0
+                        ImportName:      ''
+                      - TerminalSize:    0
+                        NodeOffset:      617
+                        Name:            impleInternal
+                        Flags:           0x0
+                        Address:         0x0
+                        Other:           0x0
+                        ImportName:      ''
+                        Children:
+                          - TerminalSize:    3
+                            NodeOffset:      631
+                            Name:            SPI
+                            Flags:           0x0
+                            Address:         0x3318
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    3
+                            NodeOffset:      636
+                            Name:            API
+                            Flags:           0x0
+                            Address:         0x3270
+                            Other:           0x0
+                            ImportName:      ''
+              - TerminalSize:    0
+                NodeOffset:      641
+                Name:            'CLASS_$_'
+                Flags:           0x0
+                Address:         0x0
+                Other:           0x0
+                ImportName:      ''
+                Children:
+                  - TerminalSize:    4
+                    NodeOffset:      692
+                    Name:            A
+                    Flags:           0x0
+                    Address:         0x48D0
+                    Other:           0x0
+                    ImportName:      ''
+                  - TerminalSize:    4
+                    NodeOffset:      698
+                    Name:            ExternalManagedObject
+                    Flags:           0x0
+                    Address:         0x47E0
+                    Other:           0x0
+                    ImportName:      ''
+                  - TerminalSize:    4
+                    NodeOffset:      704
+                    Name:            FooClass
+                    Flags:           0x0
+                    Address:         0x4948
+                    Other:           0x0
+                    ImportName:      ''
+                  - TerminalSize:    0
+                    NodeOffset:      710
+                    Name:            S
+                    Flags:           0x0
+                    Address:         0x0
+                    Other:           0x0
+                    ImportName:      ''
+                    Children:
+                      - TerminalSize:    4
+                        NodeOffset:      730
+                        Name:            ubClass
+                        Flags:           0x0
+                        Address:         0x4420
+                        Other:           0x0
+                        ImportName:      ''
+                      - TerminalSize:    4
+                        NodeOffset:      736
+                        Name:            imple
+                        Flags:           0x0
+                        Address:         0x4380
+                        Other:           0x0
+                        ImportName:      ''
+                        Children:
+                          - TerminalSize:    0
+                            NodeOffset:      753
+                            Name:            Internal
+                            Flags:           0x0
+                            Address:         0x0
+                            Other:           0x0
+                            ImportName:      ''
+                            Children:
+                              - TerminalSize:    4
+                                NodeOffset:      767
+                                Name:            API
+                                Flags:           0x0
+                                Address:         0x4470
+                                Other:           0x0
+                                ImportName:      ''
+                              - TerminalSize:    4
+                                NodeOffset:      773
+                                Name:            SPI
+                                Flags:           0x0
+                                Address:         0x44C0
+                                Other:           0x0
+                                ImportName:      ''
+                  - TerminalSize:    0
+                    NodeOffset:      779
+                    Name:            Bas
+                    Flags:           0x0
+                    Address:         0x0
+                    Other:           0x0
+                    ImportName:      ''
+                    Children:
+                      - TerminalSize:    4
+                        NodeOffset:      790
+                        Name:            e
+                        Flags:           0x0
+                        Address:         0x43D0
+                        Other:           0x0
+                        ImportName:      ''
+                      - TerminalSize:    0
+                        NodeOffset:      796
+                        Name:            ic
+                        Flags:           0x0
+                        Address:         0x0
+                        Other:           0x0
+                        ImportName:      ''
+                        Children:
+                          - TerminalSize:    4
+                            NodeOffset:      830
+                            Name:            '1'
+                            Flags:           0x0
+                            Address:         0x44E8
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      836
+                            Name:            '3'
+                            Flags:           0x0
+                            Address:         0x45B0
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      842
+                            Name:            '4'
+                            Flags:           0x0
+                            Address:         0x4600
+                            Other:           0x0
+                            ImportName:      ''
+                            Children:
+                              - TerminalSize:    4
+                                NodeOffset:      853
+                                Name:            _2
+                                Flags:           0x0
+                                Address:         0x46A0
+                                Other:           0x0
+                                ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      859
+                            Name:            '2'
+                            Flags:           0x0
+                            Address:         0x4560
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      865
+                            Name:            '8'
+                            Flags:           0x0
+                            Address:         0x4880
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      871
+                            Name:            '9'
+                            Flags:           0x0
+                            Address:         0x4920
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      877
+                            Name:            '6'
+                            Flags:           0x0
+                            Address:         0x4740
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      883
+                            Name:            '5'
+                            Flags:           0x0
+                            Address:         0x46F0
+                            Other:           0x0
+                            ImportName:      ''
+  NameList:
+    - n_strx:          2
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7104
+    - n_strx:          22
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7114
+    - n_strx:          46
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7123
+    - n_strx:          66
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7133
+    - n_strx:          88
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7143
+    - n_strx:          112
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7152
+    - n_strx:          135
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7158
+    - n_strx:          162
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7164
+    - n_strx:          204
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7170
+    - n_strx:          224
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7180
+    - n_strx:          248
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7189
+    - n_strx:          273
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7200
+    - n_strx:          302
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7206
+    - n_strx:          347
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7212
+    - n_strx:          395
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7218
+    - n_strx:          424
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7224
+    - n_strx:          466
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7232
+    - n_strx:          488
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7238
+    - n_strx:          510
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7244
+    - n_strx:          523
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7250
+    - n_strx:          543
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7260
+    - n_strx:          566
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7266
+    - n_strx:          593
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7272
+    - n_strx:          615
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7278
+    - n_strx:          658
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         12288
+    - n_strx:          687
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         12360
+    - n_strx:          712
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         12432
+    - n_strx:          739
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         12504
+    - n_strx:          762
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         12600
+    - n_strx:          793
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         12672
+    - n_strx:          820
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         12768
+    - n_strx:          860
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         12840
+    - n_strx:          896
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         12936
+    - n_strx:          936
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13008
+    - n_strx:          972
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13176
+    - n_strx:          997
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13104
+    - n_strx:          1026
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13248
+    - n_strx:          1055
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13320
+    - n_strx:          1080
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13392
+    - n_strx:          1109
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13464
+    - n_strx:          1142
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13592
+    - n_strx:          1177
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13696
+    - n_strx:          1203
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13768
+    - n_strx:          1228
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13840
+    - n_strx:          1257
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13912
+    - n_strx:          1292
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14048
+    - n_strx:          1317
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14120
+    - n_strx:          1348
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14192
+    - n_strx:          1385
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14328
+    - n_strx:          1412
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14400
+    - n_strx:          1443
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14472
+    - n_strx:          1480
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14608
+    - n_strx:          1507
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14680
+    - n_strx:          1537
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14712
+    - n_strx:          1566
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14784
+    - n_strx:          1599
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14816
+    - n_strx:          1624
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14888
+    - n_strx:          1653
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14960
+    - n_strx:          1686
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15040
+    - n_strx:          1721
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15112
+    - n_strx:          1747
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15136
+    - n_strx:          1772
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15208
+    - n_strx:          1820
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15288
+    - n_strx:          1852
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15312
+    - n_strx:          1883
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15376
+    - n_strx:          1912
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15448
+    - n_strx:          1945
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15480
+    - n_strx:          1970
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15552
+    - n_strx:          2014
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15624
+    - n_strx:          2062
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15656
+    - n_strx:          2102
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15728
+    - n_strx:          2162
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15760
+    - n_strx:          2205
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15824
+    - n_strx:          2239
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15896
+    - n_strx:          2269
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15968
+    - n_strx:          2299
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16000
+    - n_strx:          2328
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16072
+    - n_strx:          2361
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16104
+    - n_strx:          2386
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16176
+    - n_strx:          2410
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16248
+    - n_strx:          2438
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16280
+    - n_strx:          2458
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16352
+    - n_strx:          2487
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16424
+    - n_strx:          2520
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16456
+    - n_strx:          2555
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16496
+    - n_strx:          2581
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16520
+    - n_strx:          2606
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16592
+    - n_strx:          2645
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16616
+    - n_strx:          2683
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         17008
+    - n_strx:          2710
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16856
+    - n_strx:          2741
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16680
+    - n_strx:          2789
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16712
+    - n_strx:          2833
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16720
+    - n_strx:          2868
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16744
+    - n_strx:          2915
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16776
+    - n_strx:          2958
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16784
+    - n_strx:          3005
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16816
+    - n_strx:          3048
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16824
+    - n_strx:          3082
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16928
+    - n_strx:          3117
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         17080
+    - n_strx:          3171
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         17112
+    - n_strx:          3222
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         17144
+    - n_strx:          3269
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         17152
+    - n_strx:          3316
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         17176
+    - n_strx:          4000
+      n_type:          0x1E
+      n_sect:          7
+      n_desc:          0
+      n_value:         17960
+    - n_strx:          4027
+      n_type:          0x1E
+      n_sect:          7
+      n_desc:          0
+      n_value:         18000
+    - n_strx:          4192
+      n_type:          0x1E
+      n_sect:          7
+      n_desc:          0
+      n_value:         18280
+    - n_strx:          4217
+      n_type:          0x1E
+      n_sect:          7
+      n_desc:          0
+      n_value:         18320
+    - n_strx:          4314
+      n_type:          0x1E
+      n_sect:          7
+      n_desc:          0
+      n_value:         18440
+    - n_strx:          4344
+      n_type:          0x1E
+      n_sect:          7
+      n_desc:          0
+      n_value:         18480
+    - n_strx:          4548
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18840
+    - n_strx:          4578
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18848
+    - n_strx:          4608
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18856
+    - n_strx:          4690
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18880
+    - n_strx:          4716
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18888
+    - n_strx:          4742
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18896
+    - n_strx:          4770
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18904
+    - n_strx:          4798
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18912
+    - n_strx:          4826
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18920
+    - n_strx:          4854
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18928
+    - n_strx:          4882
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18936
+    - n_strx:          4992
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18968
+    - n_strx:          5022
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18976
+    - n_strx:          5053
+      n_type:          0x1E
+      n_sect:          9
+      n_desc:          0
+      n_value:         18984
+    - n_strx:          5084
+      n_type:          0x1E
+      n_sect:          9
+      n_desc:          0
+      n_value:         19080
+    - n_strx:          5114
+      n_type:          0x1E
+      n_sect:          9
+      n_desc:          0
+      n_value:         19176
+    - n_strx:          5144
+      n_type:          0x1E
+      n_sect:          9
+      n_desc:          0
+      n_value:         19272
+    - n_strx:          5231
+      n_type:          0x1E
+      n_sect:          10
+      n_desc:          0
+      n_value:         19376
+    - n_strx:          5268
+      n_type:          0x1E
+      n_sect:          10
+      n_desc:          0
+      n_value:         19384
+    - n_strx:          5304
+      n_type:          0x1E
+      n_sect:          10
+      n_desc:          0
+      n_value:         19392
+    - n_strx:          5340
+      n_type:          0x1E
+      n_sect:          10
+      n_desc:          0
+      n_value:         19400
+    - n_strx:          3353
+      n_type:          0xF
+      n_sect:          14
+      n_desc:          0
+      n_value:         19608
+    - n_strx:          3370
+      n_type:          0xF
+      n_sect:          14
+      n_desc:          0
+      n_value:         19612
+    - n_strx:          3387
+      n_type:          0xF
+      n_sect:          14
+      n_desc:          0
+      n_value:         19616
+    - n_strx:          3410
+      n_type:          0xF
+      n_sect:          14
+      n_desc:          0
+      n_value:         19620
+    - n_strx:          3432
+      n_type:          0xF
+      n_sect:          6
+      n_desc:          0
+      n_value:         12576
+    - n_strx:          3452
+      n_type:          0xF
+      n_sect:          6
+      n_desc:          0
+      n_value:         12744
+    - n_strx:          3476
+      n_type:          0xF
+      n_sect:          6
+      n_desc:          0
+      n_value:         12912
+    - n_strx:          3509
+      n_type:          0xF
+      n_sect:          6
+      n_desc:          0
+      n_value:         13080
+    - n_strx:          3542
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17240
+    - n_strx:          3567
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17280
+    - n_strx:          3588
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17320
+    - n_strx:          3611
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17360
+    - n_strx:          3630
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17400
+    - n_strx:          3657
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17440
+    - n_strx:          3680
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17480
+    - n_strx:          3716
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17520
+    - n_strx:          3748
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17560
+    - n_strx:          3784
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17600
+    - n_strx:          3816
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17640
+    - n_strx:          3837
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17680
+    - n_strx:          3862
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17720
+    - n_strx:          3887
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17760
+    - n_strx:          3908
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17800
+    - n_strx:          3933
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17840
+    - n_strx:          3954
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17880
+    - n_strx:          3979
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17920
+    - n_strx:          4050
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18040
+    - n_strx:          4077
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18080
+    - n_strx:          4100
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18120
+    - n_strx:          4125
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18160
+    - n_strx:          4146
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18200
+    - n_strx:          4171
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18240
+    - n_strx:          4238
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18360
+    - n_strx:          4278
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18400
+    - n_strx:          4370
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18520
+    - n_strx:          4395
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18560
+    - n_strx:          4416
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18600
+    - n_strx:          4436
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18640
+    - n_strx:          4452
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18680
+    - n_strx:          4477
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18720
+    - n_strx:          4498
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18760
+    - n_strx:          4521
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18800
+    - n_strx:          4638
+      n_type:          0xF
+      n_sect:          8
+      n_desc:          0
+      n_value:         18864
+    - n_strx:          4664
+      n_type:          0xF
+      n_sect:          8
+      n_desc:          0
+      n_value:         18872
+    - n_strx:          4910
+      n_type:          0xF
+      n_sect:          8
+      n_desc:          0
+      n_value:         18944
+    - n_strx:          4938
+      n_type:          0xF
+      n_sect:          8
+      n_desc:          0
+      n_value:         18952
+    - n_strx:          4966
+      n_type:          0xF
+      n_sect:          8
+      n_desc:          0
+      n_value:         18960
+    - n_strx:          5178
+      n_type:          0xF
+      n_sect:          9
+      n_desc:          128
+      n_value:         19368
+    - n_strx:          5204
+      n_type:          0xF
+      n_sect:          9
+      n_desc:          128
+      n_value:         19372
+    - n_strx:          5380
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          768
+      n_value:         0
+    - n_strx:          5410
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          512
+      n_value:         0
+    - n_strx:          5433
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          768
+      n_value:         0
+    - n_strx:          5467
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          512
+      n_value:         0
+    - n_strx:          5494
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          512
+      n_value:         0
+    - n_strx:          5513
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          512
+      n_value:         0
+    - n_strx:          5533
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          1024
+      n_value:         0
+  StringTable:
+    - ' '
+    - '-[Basic3 property1]'
+    - '-[Basic3 setProperty1:]'
+    - '-[Basic3 property2]'
+    - '-[Basic3 isProperty3]'
+    - '-[Basic3 setProperty3:]'
+    - '+[Basic5 aClassMethod]'
+    - '-[Basic5 anInstanceMethod]'
+    - '-[Basic6 anInstanceMethodFromAnExtension]'
+    - '-[Basic6 property1]'
+    - '-[Basic6 setProperty1:]'
+    - '-[Basic6(Foo) property2]'
+    - '-[Basic6(Foo) setProperty2:]'
+    - '-[Basic6(Foo) anInstanceMethodFromACategory]'
+    - '-[Basic7 anInstanceMethodFromAnHiddenExtension]'
+    - '-[ExternalManagedObject foo]'
+    - '-[NSManagedObject(Simple) supportsSimple]'
+    - '+[Basic8 useSameName]'
+    - '-[Basic8 useSameName]'
+    - '-[A aMethod]'
+    - '-[Basic9 aProperty]'
+    - '-[FooClass baseMethod]'
+    - '-[FooClass protocolMethod]'
+    - '-[FooClass barMethod]'
+    - '-[FooClass(Private) privateProcotolMethod]'
+    - '__OBJC_METACLASS_RO_$_Simple'
+    - '__OBJC_CLASS_RO_$_Simple'
+    - '__OBJC_METACLASS_RO_$_Base'
+    - '__OBJC_CLASS_RO_$_Base'
+    - '__OBJC_METACLASS_RO_$_SubClass'
+    - '__OBJC_CLASS_RO_$_SubClass'
+    - '__OBJC_METACLASS_RO_$_SimpleInternalAPI'
+    - '__OBJC_CLASS_RO_$_SimpleInternalAPI'
+    - '__OBJC_METACLASS_RO_$_SimpleInternalSPI'
+    - '__OBJC_CLASS_RO_$_SimpleInternalSPI'
+    - '__OBJC_CLASS_RO_$_Basic1'
+    - '__OBJC_METACLASS_RO_$_Basic1'
+    - '__OBJC_METACLASS_RO_$_Basic2'
+    - '__OBJC_CLASS_RO_$_Basic2'
+    - '__OBJC_METACLASS_RO_$_Basic3'
+    - '__OBJC_$_INSTANCE_METHODS_Basic3'
+    - '__OBJC_$_INSTANCE_VARIABLES_Basic3'
+    - '__OBJC_$_PROP_LIST_Basic3'
+    - '__OBJC_CLASS_RO_$_Basic3'
+    - '__OBJC_METACLASS_RO_$_Basic4'
+    - '__OBJC_$_INSTANCE_VARIABLES_Basic4'
+    - '__OBJC_CLASS_RO_$_Basic4'
+    - '__OBJC_METACLASS_RO_$_Basic4_1'
+    - '__OBJC_$_INSTANCE_VARIABLES_Basic4_1'
+    - '__OBJC_CLASS_RO_$_Basic4_1'
+    - '__OBJC_METACLASS_RO_$_Basic4_2'
+    - '__OBJC_$_INSTANCE_VARIABLES_Basic4_2'
+    - '__OBJC_CLASS_RO_$_Basic4_2'
+    - '__OBJC_$_CLASS_METHODS_Basic5'
+    - '__OBJC_METACLASS_RO_$_Basic5'
+    - '__OBJC_$_INSTANCE_METHODS_Basic5'
+    - '__OBJC_CLASS_RO_$_Basic5'
+    - '__OBJC_METACLASS_RO_$_Basic6'
+    - '__OBJC_$_INSTANCE_METHODS_Basic6'
+    - '__OBJC_$_INSTANCE_VARIABLES_Basic6'
+    - '__OBJC_$_PROP_LIST_Basic6'
+    - '__OBJC_CLASS_RO_$_Basic6'
+    - '__OBJC_$_CATEGORY_INSTANCE_METHODS_Basic6_$_Foo'
+    - '__OBJC_$_PROP_LIST_Basic6_$_Foo'
+    - '__OBJC_$_CATEGORY_Basic6_$_Foo'
+    - '__OBJC_METACLASS_RO_$_Basic7'
+    - '__OBJC_$_INSTANCE_METHODS_Basic7'
+    - '__OBJC_CLASS_RO_$_Basic7'
+    - '__OBJC_METACLASS_RO_$_ExternalManagedObject'
+    - '__OBJC_$_INSTANCE_METHODS_ExternalManagedObject'
+    - '__OBJC_CLASS_RO_$_ExternalManagedObject'
+    - '__OBJC_$_CATEGORY_INSTANCE_METHODS_NSManagedObject_$_Simple'
+    - '__OBJC_$_CATEGORY_NSManagedObject_$_Simple'
+    - '__OBJC_METACLASS_RO_$_HiddenClass'
+    - '__OBJC_CLASS_RO_$_HiddenClass'
+    - '__OBJC_$_CLASS_METHODS_Basic8'
+    - '__OBJC_METACLASS_RO_$_Basic8'
+    - '__OBJC_$_INSTANCE_METHODS_Basic8'
+    - '__OBJC_CLASS_RO_$_Basic8'
+    - '__OBJC_METACLASS_RO_$_A'
+    - '__OBJC_$_INSTANCE_METHODS_A'
+    - '__OBJC_CLASS_RO_$_A'
+    - '__OBJC_METACLASS_RO_$_Basic9'
+    - '__OBJC_$_INSTANCE_METHODS_Basic9'
+    - '__OBJC_$_INSTANCE_VARIABLES_Basic9'
+    - '__OBJC_$_PROP_LIST_Basic9'
+    - '__OBJC_CLASS_RO_$_Basic9'
+    - '__OBJC_$_PROP_LIST_Basic9_$_deprecated'
+    - '__OBJC_$_CATEGORY_Basic9_$_deprecated'
+    - '__OBJC_CLASS_RO_$_FooClass'
+    - '__OBJC_METACLASS_RO_$_FooClass'
+    - '__OBJC_$_PROTOCOL_INSTANCE_METHODS_BaseProtocol'
+    - '__OBJC_$_PROTOCOL_METHOD_TYPES_BaseProtocol'
+    - '__OBJC_$_PROTOCOL_REFS_FooProtocol'
+    - '__OBJC_$_PROTOCOL_INSTANCE_METHODS_FooProtocol'
+    - '__OBJC_$_PROTOCOL_METHOD_TYPES_FooProtocol'
+    - '__OBJC_$_PROTOCOL_INSTANCE_METHODS_BarProtocol'
+    - '__OBJC_$_PROTOCOL_METHOD_TYPES_BarProtocol'
+    - '__OBJC_CLASS_PROTOCOLS_$_FooClass'
+    - '__OBJC_$_INSTANCE_METHODS_FooClass'
+    - '__OBJC_$_CATEGORY_INSTANCE_METHODS_FooClass_$_Private'
+    - '__OBJC_$_PROTOCOL_INSTANCE_METHODS_PrivateProtocol'
+    - '__OBJC_$_PROTOCOL_METHOD_TYPES_PrivateProtocol'
+    - '__OBJC_CATEGORY_PROTOCOLS_$_FooClass_$_Private'
+    - '__OBJC_$_CATEGORY_FooClass_$_Private'
+    - _extraGlobalAPI1
+    - _extraGlobalAPI2
+    - _privateGlobalVariable
+    - _publicGlobalVariable
+    - '_OBJC_EHTYPE_$_Base'
+    - '_OBJC_EHTYPE_$_SubClass'
+    - '_OBJC_EHTYPE_$_SimpleInternalAPI'
+    - '_OBJC_EHTYPE_$_SimpleInternalSPI'
+    - '_OBJC_METACLASS_$_Simple'
+    - '_OBJC_CLASS_$_Simple'
+    - '_OBJC_METACLASS_$_Base'
+    - '_OBJC_CLASS_$_Base'
+    - '_OBJC_METACLASS_$_SubClass'
+    - '_OBJC_CLASS_$_SubClass'
+    - '_OBJC_METACLASS_$_SimpleInternalAPI'
+    - '_OBJC_CLASS_$_SimpleInternalAPI'
+    - '_OBJC_METACLASS_$_SimpleInternalSPI'
+    - '_OBJC_CLASS_$_SimpleInternalSPI'
+    - '_OBJC_CLASS_$_Basic1'
+    - '_OBJC_METACLASS_$_Basic1'
+    - '_OBJC_METACLASS_$_Basic2'
+    - '_OBJC_CLASS_$_Basic2'
+    - '_OBJC_METACLASS_$_Basic3'
+    - '_OBJC_CLASS_$_Basic3'
+    - '_OBJC_METACLASS_$_Basic4'
+    - '_OBJC_CLASS_$_Basic4'
+    - '_OBJC_METACLASS_$_Basic4_1'
+    - '_OBJC_CLASS_$_Basic4_1'
+    - '_OBJC_METACLASS_$_Basic4_2'
+    - '_OBJC_CLASS_$_Basic4_2'
+    - '_OBJC_METACLASS_$_Basic5'
+    - '_OBJC_CLASS_$_Basic5'
+    - '_OBJC_METACLASS_$_Basic6'
+    - '_OBJC_CLASS_$_Basic6'
+    - '_OBJC_METACLASS_$_Basic7'
+    - '_OBJC_CLASS_$_Basic7'
+    - '_OBJC_METACLASS_$_ExternalManagedObject'
+    - '_OBJC_CLASS_$_ExternalManagedObject'
+    - '_OBJC_METACLASS_$_HiddenClass'
+    - '_OBJC_CLASS_$_HiddenClass'
+    - '_OBJC_METACLASS_$_Basic8'
+    - '_OBJC_CLASS_$_Basic8'
+    - '_OBJC_METACLASS_$_A'
+    - '_OBJC_CLASS_$_A'
+    - '_OBJC_METACLASS_$_Basic9'
+    - '_OBJC_CLASS_$_Basic9'
+    - '_OBJC_CLASS_$_FooClass'
+    - '_OBJC_METACLASS_$_FooClass'
+    - '_OBJC_IVAR_$_Basic3.property1'
+    - '_OBJC_IVAR_$_Basic3.property2'
+    - '_OBJC_IVAR_$_Basic3.property3'
+    - '_OBJC_IVAR_$_Basic4.ivar1'
+    - '_OBJC_IVAR_$_Basic4.ivar2'
+    - '_OBJC_IVAR_$_Basic4.ivar3'
+    - '_OBJC_IVAR_$_Basic4.ivar4'
+    - '_OBJC_IVAR_$_Basic4_1.ivar1'
+    - '_OBJC_IVAR_$_Basic4_1.ivar2'
+    - '_OBJC_IVAR_$_Basic4_1.ivar3'
+    - '_OBJC_IVAR_$_Basic4_1.ivar4'
+    - '_OBJC_IVAR_$_Basic4_2.ivar4'
+    - '_OBJC_IVAR_$_Basic4_2.ivar3'
+    - '_OBJC_IVAR_$_Basic4_2.ivar2'
+    - '_OBJC_IVAR_$_Basic4_2.ivar1'
+    - '_OBJC_IVAR_$_Basic6.ivar1'
+    - '_OBJC_IVAR_$_Basic6.property1'
+    - '_OBJC_IVAR_$_Basic9._aProperty'
+    - '__OBJC_PROTOCOL_$_BaseProtocol'
+    - '__OBJC_PROTOCOL_$_FooProtocol'
+    - '__OBJC_PROTOCOL_$_BarProtocol'
+    - '__OBJC_PROTOCOL_$_PrivateProtocol'
+    - _weakPublicGlobalVariable
+    - _weakPrivateGlobalVariable
+    - '__OBJC_LABEL_PROTOCOL_$_BaseProtocol'
+    - '__OBJC_LABEL_PROTOCOL_$_FooProtocol'
+    - '__OBJC_LABEL_PROTOCOL_$_BarProtocol'
+    - '__OBJC_LABEL_PROTOCOL_$_PrivateProtocol'
+    - '_OBJC_CLASS_$_NSManagedObject'
+    - '_OBJC_CLASS_$_NSObject'
+    - '_OBJC_METACLASS_$_NSManagedObject'
+    - '_OBJC_METACLASS_$_NSObject'
+    - __objc_empty_cache
+    - _objc_ehtype_vtable
+    - dyld_stub_binder
+    - ''
+    - ''
+  FunctionStarts:  [ 0x1BC0, 0x1BCA, 0x1BD3, 0x1BDD, 0x1BE7, 0x1BF0, 0x1BF6, 
+                     0x1BFC, 0x1C02, 0x1C0C, 0x1C15, 0x1C20, 0x1C26, 0x1C2C, 
+                     0x1C32, 0x1C38, 0x1C40, 0x1C46, 0x1C4C, 0x1C52, 0x1C5C, 
+                     0x1C62, 0x1C68, 0x1C6E ]
+...
diff --git a/clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI.h b/clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI.h
new file mode 100644
index 000000000000..5dd416a0619c
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI.h
@@ -0,0 +1,3 @@
+#ifndef HAVE_SEEN_PROJECT_HEADER_FIRST
+#error "Project header was not included in the correct order!"
+#endif
diff --git a/clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI2.h b/clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI2.h
new file mode 100644
index 000000000000..9bbae52d7215
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI2.h
@@ -0,0 +1,7 @@
+#import <Foundation/Foundation.h>
+
+__attribute__((objc_exception))
+@interface SimpleInternalAPI : NSObject
+@end
+
+#define HAVE_SEEN_PROJECT_HEADER_FIRST 1
diff --git a/clang/test/InstallAPI/Inputs/Simple/SimpleInternalSPI.h b/clang/test/InstallAPI/Inputs/Simple/SimpleInternalSPI.h
new file mode 100644
index 000000000000..a816c01abeb0
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/SimpleInternalSPI.h
@@ -0,0 +1,5 @@
+#import <Foundation/Foundation.h>
+
+__attribute__((objc_exception))
+@interface SimpleInternalSPI : NSObject
+@end
diff --git a/clang/test/InstallAPI/extra-exclude-headers.test b/clang/test/InstallAPI/extra-exclude-headers.test
new file mode 100644
index 000000000000..663ca1a5d500
--- /dev/null
+++ b/clang/test/InstallAPI/extra-exclude-headers.test
@@ -0,0 +1,207 @@
+; RUN: rm -rf %t 
+; RUN: split-file %s %t
+; RUN: mkdir -p %t/System/Library/Frameworks 
+; RUN: cp -r %S/Inputs/Simple/Simple.framework %t/System/Library/Frameworks/
+; RUN: sed -e "s|DSTROOT|%/t|g" %t/inputs.json.in > %t/inputs.json
+; RUN: yaml2obj %S/Inputs/Simple/Simple.yaml -o %t/Simple 
+
+// Add exclude options.
+; RUN: clang-installapi -target x86_64-apple-macosx10.12 \
+; RUN: -install_name /System/Library/Frameworks/Simple.framework/Versions/A/Simple \
+; RUN: -current_version 1.2.3 -compatibility_version 1 \
+; RUN: -F%t/System/Library/Frameworks \
+; RUN: %t/inputs.json -o %t/Simple.tbd \
+; RUN: --verify-against=%t/Simple --verify-mode=ErrorsAndWarnings \
+; RUN: --exclude-public-header=**/SimpleAPI.h \
+; RUN: --exclude-private-header=**/SimplePrivateSPI.h 2>&1 | FileCheck -check-prefix=WARNINGS %s
+; RUN: llvm-readtapi -compare %t/Simple.tbd %t/expected-excluded.tbd
+
+// Add extra options.
+; RUN: clang-installapi -target x86_64-apple-macosx10.12 \
+; RUN: -install_name /System/Library/Frameworks/Simple.framework/Versions/A/Simple \
+; RUN: -current_version 1.2.3 -compatibility_version 1 \
+; RUN: -F%t/System/Library/Frameworks \
+; RUN: %t/inputs.json -o %t/Simple.tbd \
+; RUN: --verify-against=%t/Simple --verify-mode=Pedantic \
+; RUN: --extra-project-header=%S/Inputs/Simple/SimpleInternalAPI2.h \
+; RUN: --extra-project-header=%S/Inputs/Simple/SimpleInternalAPI.h \
+; RUN: --extra-public-header=%S/Inputs/Simple/Extra \
+; RUN: --extra-private-header=%S/Inputs/Simple/SimpleInternalSPI.h \
+; RUN: --exclude-public-header=**/SimpleAPI.h \
+; RUN: --exclude-private-header=**/SimplePrivateSPI.h 2>&1 | FileCheck -check-prefix=PEDANTIC -allow-empty %s
+; RUN: llvm-readtapi -compare %t/Simple.tbd %t/expected-extra.tbd
+
+// Check fatal missing file input.
+; RUN: not clang-installapi -target x86_64-apple-macosx10.12 \
+; RUN: -install_name /System/Library/Frameworks/Simple.framework/Versions/A/Simple \
+; RUN: -current_version 1.2.3 -compatibility_version 1 \
+; RUN: -F%t/System/Library/Frameworks \
+; RUN: %t/inputs.json -o %t/Simple.tbd \
+; RUN: --extra-public-header=%S/Inputs/Simple/NoSuchFile.h 2>&1 | FileCheck -allow-empty -check-prefix=NOPUBLIC %s
+
+; WARNINGS: warning: no declaration was found for exported symbol '_extraGlobalAPI1' in dynamic library
+; WARNINGS: warning: no declaration was found for exported symbol '_extraGlobalAPI2' in dynamic library
+; WARNINGS: warning: no declaration was found for exported symbol '(ObjC Class) SimpleInternalSPI' in dynamic library
+; WARNINGS: warning: no declaration was found for exported symbol '(ObjC Class) SimpleInternalAPI' in dynamic library
+
+; PEDANTIC-NOT: error
+; PEDANTIC: warning: cannot find protocol definition for 'ForwardProcotol'
+
+; NOPUBLIC: error: no such public header file:
+
+;--- expected-excluded.tbd
+{
+  "main_library": {
+    "current_versions": [
+      {
+        "version": "1.2.3"
+      }
+    ],
+    "exported_symbols": [
+      {
+        "data": {
+          "global": [
+            "_publicGlobalVariable",
+            "_privateGlobalVariable"
+          ],
+          "objc_class": [
+            "ExternalManagedObject", "Basic6",
+            "Basic1", "Base", "Basic3",
+            "FooClass", "Simple",
+            "Basic4_2", "Basic5",
+            "Basic9","Basic8",
+            "Basic2", "Basic4", "A", "SubClass"
+          ],
+          "objc_eh_type": [
+            "SubClass", "Base"
+          ],
+          "objc_ivar": [
+            "Basic4.ivar2", "Basic4_2.ivar1", "Basic6.ivar1",
+            "Basic4.ivar1", "Basic4_2.ivar2"
+          ],
+          "weak": [
+            "_weakPrivateGlobalVariable", "_weakPublicGlobalVariable"
+          ]
+        }
+      }
+    ],
+    "flags": [
+      {
+        "attributes": ["not_app_extension_safe"]
+      }
+    ],
+    "install_names": [
+      {
+        "name": "/System/Library/Frameworks/Simple.framework/Versions/A/Simple"
+      }
+    ],
+    "target_info": [
+      {"min_deployment": "10.12", "target": "x86_64-macos"}
+    ]
+  },
+  "tapi_tbd_version": 5
+}
+
+;--- expected-extra.tbd
+{
+  "main_library": {
+    "current_versions": [
+      { "version": "1.2.3" }
+    ],
+    "exported_symbols": [
+      {
+        "data": {
+          "global": [
+            "_publicGlobalVariable", "_extraGlobalAPI2",
+            "_extraGlobalAPI1", "_privateGlobalVariable"
+          ],
+          "objc_class": [
+            "SubClass", "SimpleInternalSPI",
+            "Basic6", "Basic1", "Base",
+            "Basic3", "Simple", "Basic4_2",
+            "Basic5", "FooClass", "Basic9",
+            "Basic8", "Basic2", "Basic4",
+            "A", "SimpleInternalAPI",
+            "ExternalManagedObject"
+          ],
+          "objc_eh_type": [
+            "SubClass", "SimpleInternalAPI",
+            "Base", "SimpleInternalSPI"
+          ],
+          "objc_ivar": [
+            "Basic4.ivar2", "Basic4_2.ivar1",
+            "Basic6.ivar1", "Basic4.ivar1",
+            "Basic4_2.ivar2"
+          ],
+          "weak": [
+            "_weakPrivateGlobalVariable", "_weakPublicGlobalVariable"
+          ]
+        }
+      }
+    ],
+    "flags": [
+      {
+        "attributes": [ "not_app_extension_safe"]
+      }
+    ],
+    "install_names": [
+      { "name": "/System/Library/Frameworks/Simple.framework/Versions/A/Simple" }
+    ],
+    "target_info": [
+      { "min_deployment": "10.12", "target": "x86_64-macos" }
+    ]
+  },
+  "tapi_tbd_version": 5
+}
+
+;--- inputs.json.in
+{
+  "headers": [ 
+  {
+    "path" : "DSTROOT/System/Library/Frameworks/Simple.framework/Headers/Basic.h",
+    "type" : "public"
+  }, 
+  {
+    "path" : "DSTROOT/System/Library/Frameworks/Simple.framework/Headers/External.h",
+    "type" : "public"
+  }, 
+  {
+    "path" : "DSTROOT/System/Library/Frameworks/Simple.framework/Headers/Simple.h",
+    "type" : "public"
+  }, 
+  {
+    "path" : "DSTROOT/System/Library/Frameworks/Simple.framework/Headers/SimpleAPI.h",
+    "type" : "public"
+  }, 
+  {
+    "path" : "DSTROOT/System/Library/Frameworks/Simple.framework/PrivateHeaders/SimplePrivate.h",
+    "type" : "private"
+  }, 
+  {
+    "path" : "DSTROOT/System/Library/Frameworks/Simple.framework/PrivateHeaders/SimplePrivateSPI.h",
+    "type" : "private"
+  }
+  ],
+  "version": "3"
+}
+
+;--- System/Library/Frameworks/Foundation.framework/Headers/Foundation.h
+@interface NSObject 
+@end
+
+typedef unsigned char BOOL; 
+#ifndef NS_AVAILABLE
+#define NS_AVAILABLE(x,y) __attribute__((availability(macosx,introduced=x)))
+#endif 
+#ifndef NS_UNAVAILABLE
+#define NS_UNAVAILABLE  __attribute__((unavailable))
+#endif 
+#ifndef NS_DEPRECATED_MAC
+#define NS_DEPRECATED_MAC(x,y) __attribute__((availability(macosx,introduced=x,deprecated=y,message="" )));
+#endif 
+
+@interface NSManagedObject
+@end 
+
+@interface NSSet 
+@end 
diff --git a/clang/tools/clang-installapi/InstallAPIOpts.td b/clang/tools/clang-installapi/InstallAPIOpts.td
index 87f4c3327e84..ab9e1fe7f2f9 100644
--- a/clang/tools/clang-installapi/InstallAPIOpts.td
+++ b/clang/tools/clang-installapi/InstallAPIOpts.td
@@ -29,3 +29,35 @@ def verify_mode_EQ : Joined<["--"], "verify-mode=">,
   HelpText<"Specify the severity and extend of the validation. Valid modes are ErrorsOnly, ErrorsAndWarnings, and Pedantic.">;
 def demangle : Flag<["--", "-"], "demangle">,
   HelpText<"Demangle symbols when printing warnings and errors">;
+
+// Additional input options.
+def extra_project_header : Separate<["-"], "extra-project-header">,
+  MetaVarName<"<path>">,
+  HelpText<"Add additional project header location for parsing">;
+def extra_project_header_EQ : Joined<["--"], "extra-project-header=">,
+  Alias<extra_project_header>;
+def exclude_project_header : Separate<["-"], "exclude-project-header">,
+  MetaVarName<"<glob>">,
+  HelpText<"Exclude project header from parsing">;
+def exclude_project_header_EQ : Joined<["--"], "exclude-project-header=">,
+  Alias<exclude_project_header>;
+def extra_public_header : Separate<["-"], "extra-public-header">,
+  MetaVarName<"<path>">,
+  HelpText<"Add additional public header location for parsing">;
+def extra_public_header_EQ : Joined<["--"], "extra-public-header=">,
+  Alias<extra_public_header>;
+def extra_private_header : Separate<["-"], "extra-private-header">,
+  MetaVarName<"<path>">,
+  HelpText<"Add additional private header location for parsing">;
+def extra_private_header_EQ : Joined<["--"], "extra-private-header=">,
+  Alias<extra_private_header>;
+def exclude_public_header : Separate<["-"], "exclude-public-header">,
+  MetaVarName<"<glob>">,
+  HelpText<"Exclude public header from parsing">;
+def exclude_public_header_EQ : Joined<["--"], "exclude-public-header=">,
+  Alias<exclude_public_header>;
+def exclude_private_header : Separate<["-"], "exclude-private-header">,
+  MetaVarName<"<glob>">,
+  HelpText<"Exclude private header from parsing">;
+def exclude_private_header_EQ : Joined<["--"], "exclude-private-header=">,
+  Alias<exclude_private_header>;
diff --git a/clang/tools/clang-installapi/Options.cpp b/clang/tools/clang-installapi/Options.cpp
index b8696bb7896d..4f79c62724a6 100644
--- a/clang/tools/clang-installapi/Options.cpp
+++ b/clang/tools/clang-installapi/Options.cpp
@@ -10,6 +10,7 @@
 #include "clang/Driver/Driver.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/InstallAPI/FileList.h"
+#include "clang/InstallAPI/HeaderFile.h"
 #include "clang/InstallAPI/InstallAPIDiagnostic.h"
 #include "llvm/Support/Program.h"
 #include "llvm/TargetParser/Host.h"
@@ -181,6 +182,26 @@ bool Options::processFrontendOptions(InputArgList &Args) {
   return true;
 }
 
+bool Options::addFilePaths(InputArgList &Args, PathSeq &Headers,
+                           OptSpecifier ID) {
+  for (const StringRef Path : Args.getAllArgValues(ID)) {
+    if ((bool)FM->getDirectory(Path, /*CacheFailure=*/false)) {
+      auto InputHeadersOrErr = enumerateFiles(*FM, Path);
+      if (!InputHeadersOrErr) {
+        Diags->Report(diag::err_cannot_open_file)
+            << Path << toString(InputHeadersOrErr.takeError());
+        return false;
+      }
+      // Sort headers to ensure deterministic behavior.
+      sort(*InputHeadersOrErr);
+      for (std::string &H : *InputHeadersOrErr)
+        Headers.emplace_back(std::move(H));
+    } else
+      Headers.emplace_back(Path);
+  }
+  return true;
+}
+
 std::vector<const char *>
 Options::processAndFilterOutInstallAPIOptions(ArrayRef<const char *> Args) {
   std::unique_ptr<llvm::opt::OptTable> Table;
@@ -220,6 +241,35 @@ Options::processAndFilterOutInstallAPIOptions(ArrayRef<const char *> Args) {
   if (const Arg *A = ParsedArgs.getLastArg(OPT_verify_against))
     DriverOpts.DylibToVerify = A->getValue();
 
+  // Handle exclude & extra header directories or files.
+  auto handleAdditionalInputArgs = [&](PathSeq &Headers,
+                                       clang::installapi::ID OptID) {
+    if (ParsedArgs.hasArgNoClaim(OptID))
+      Headers.clear();
+    return addFilePaths(ParsedArgs, Headers, OptID);
+  };
+
+  if (!handleAdditionalInputArgs(DriverOpts.ExtraPublicHeaders,
+                                 OPT_extra_public_header))
+    return {};
+
+  if (!handleAdditionalInputArgs(DriverOpts.ExtraPrivateHeaders,
+                                 OPT_extra_private_header))
+    return {};
+  if (!handleAdditionalInputArgs(DriverOpts.ExtraProjectHeaders,
+                                 OPT_extra_project_header))
+    return {};
+
+  if (!handleAdditionalInputArgs(DriverOpts.ExcludePublicHeaders,
+                                 OPT_exclude_public_header))
+    return {};
+  if (!handleAdditionalInputArgs(DriverOpts.ExcludePrivateHeaders,
+                                 OPT_exclude_private_header))
+    return {};
+  if (!handleAdditionalInputArgs(DriverOpts.ExcludeProjectHeaders,
+                                 OPT_exclude_project_header))
+    return {};
+
   /// Any unclaimed arguments should be forwarded to the clang driver.
   std::vector<const char *> ClangDriverArgs(ParsedArgs.size());
   for (const Arg *A : ParsedArgs) {
@@ -302,6 +352,77 @@ InstallAPIContext Options::createContext() {
       return Ctx;
     }
   }
+  // After initial input has been processed, add any extra headers.
+  auto HandleExtraHeaders = [&](PathSeq &Headers, HeaderType Type) -> bool {
+    assert(Type != HeaderType::Unknown && "Missing header type.");
+    for (const StringRef Path : Headers) {
+      if (!FM->getOptionalFileRef(Path)) {
+        Diags->Report(diag::err_no_such_header_file)
+            << Path << (unsigned)Type - 1;
+        return false;
+      }
+      SmallString<PATH_MAX> FullPath(Path);
+      FM->makeAbsolutePath(FullPath);
+
+      auto IncludeName = createIncludeHeaderName(FullPath);
+      Ctx.InputHeaders.emplace_back(
+          FullPath, Type, IncludeName.has_value() ? *IncludeName : "");
+      Ctx.InputHeaders.back().setExtra();
+    }
+    return true;
+  };
+
+  if (!HandleExtraHeaders(DriverOpts.ExtraPublicHeaders, HeaderType::Public) ||
+      !HandleExtraHeaders(DriverOpts.ExtraPrivateHeaders,
+                          HeaderType::Private) ||
+      !HandleExtraHeaders(DriverOpts.ExtraProjectHeaders, HeaderType::Project))
+    return Ctx;
+
+  // After all headers have been added, consider excluded headers.
+  std::vector<std::unique_ptr<HeaderGlob>> ExcludedHeaderGlobs;
+  std::set<FileEntryRef> ExcludedHeaderFiles;
+  auto ParseGlobs = [&](const PathSeq &Paths, HeaderType Type) {
+    for (const StringRef Path : Paths) {
+      auto Glob = HeaderGlob::create(Path, Type);
+      if (Glob)
+        ExcludedHeaderGlobs.emplace_back(std::move(Glob.get()));
+      else {
+        consumeError(Glob.takeError());
+        if (auto File = FM->getFileRef(Path))
+          ExcludedHeaderFiles.emplace(*File);
+        else {
+          Diags->Report(diag::err_no_such_header_file)
+              << Path << (unsigned)Type;
+          return false;
+        }
+      }
+    }
+    return true;
+  };
+
+  if (!ParseGlobs(DriverOpts.ExcludePublicHeaders, HeaderType::Public) ||
+      !ParseGlobs(DriverOpts.ExcludePrivateHeaders, HeaderType::Private) ||
+      !ParseGlobs(DriverOpts.ExcludeProjectHeaders, HeaderType::Project))
+    return Ctx;
+
+  for (HeaderFile &Header : Ctx.InputHeaders) {
+    for (auto &Glob : ExcludedHeaderGlobs)
+      if (Glob->match(Header))
+        Header.setExcluded();
+  }
+  if (!ExcludedHeaderFiles.empty()) {
+    for (HeaderFile &Header : Ctx.InputHeaders) {
+      auto FileRef = FM->getFileRef(Header.getPath());
+      if (!FileRef)
+        continue;
+      if (ExcludedHeaderFiles.count(*FileRef))
+        Header.setExcluded();
+    }
+  }
+  // Report if glob was ignored.
+  for (const auto &Glob : ExcludedHeaderGlobs)
+    if (!Glob->didMatch())
+      Diags->Report(diag::warn_glob_did_not_match) << Glob->str();
 
   // Parse binary dylib and initialize verifier.
   if (DriverOpts.DylibToVerify.empty()) {
diff --git a/clang/tools/clang-installapi/Options.h b/clang/tools/clang-installapi/Options.h
index 2beeafc86bb0..c18309f69370 100644
--- a/clang/tools/clang-installapi/Options.h
+++ b/clang/tools/clang-installapi/Options.h
@@ -31,6 +31,24 @@ struct DriverOptions {
   /// \brief Path to input file lists (JSON).
   llvm::MachO::PathSeq FileLists;
 
+  /// \brief Paths of extra public headers.
+  PathSeq ExtraPublicHeaders;
+
+  /// \brief Paths of extra private headers.
+  PathSeq ExtraPrivateHeaders;
+
+  /// \brief Paths of extra project headers.
+  PathSeq ExtraProjectHeaders;
+
+  /// \brief List of excluded public headers.
+  PathSeq ExcludePublicHeaders;
+
+  /// \brief List of excluded private headers.
+  PathSeq ExcludePrivateHeaders;
+
+  /// \brief List of excluded project headers.
+  PathSeq ExcludeProjectHeaders;
+
   /// \brief Mappings of target triples & tapi targets to build for.
   std::map<llvm::MachO::Target, llvm::Triple> Targets;
 
@@ -103,6 +121,9 @@ public:
   std::vector<std::string> &getClangFrontendArgs() { return FrontendArgs; }
 
 private:
+  bool addFilePaths(llvm::opt::InputArgList &Args, PathSeq &Headers,
+                    llvm::opt::OptSpecifier ID);
+
   DiagnosticsEngine *Diags;
   FileManager *FM;
   std::vector<std::string> FrontendArgs;
diff --git a/llvm/include/llvm/TextAPI/Utils.h b/llvm/include/llvm/TextAPI/Utils.h
index 31d3c45f9e29..353859ca4cea 100644
--- a/llvm/include/llvm/TextAPI/Utils.h
+++ b/llvm/include/llvm/TextAPI/Utils.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/Regex.h"
 
 #if !defined(PATH_MAX)
 #define PATH_MAX 1024
@@ -68,5 +69,10 @@ std::error_code make_relative(StringRef From, StringRef To,
 /// \param IsSymLink Whether path points to a symlink.
 bool isPrivateLibrary(StringRef Path, bool IsSymLink = false);
 
+/// Create a regex rule from provided glob string.
+/// \param Glob String that represents glob input.
+/// \return The equivalent regex rule.
+llvm::Expected<llvm::Regex> createRegexFromGlob(llvm::StringRef Glob);
+
 } // namespace llvm::MachO
 #endif // LLVM_TEXTAPI_UTILS_H
diff --git a/llvm/lib/TextAPI/Utils.cpp b/llvm/lib/TextAPI/Utils.cpp
index b5c3999a86ea..61f4df1d4a6b 100644
--- a/llvm/lib/TextAPI/Utils.cpp
+++ b/llvm/lib/TextAPI/Utils.cpp
@@ -152,3 +152,49 @@ bool llvm::MachO::isPrivateLibrary(StringRef Path, bool IsSymLink) {
   }
   return false;
 }
+
+static StringLiteral RegexMetachars = "()^$|+.[]\\{}";
+
+Expected<Regex> llvm::MachO::createRegexFromGlob(StringRef Glob) {
+  SmallString<128> RegexString("^");
+  unsigned NumWildcards = 0;
+  for (unsigned i = 0; i < Glob.size(); ++i) {
+    char C = Glob[i];
+    switch (C) {
+    case '?':
+      RegexString += '.';
+      break;
+    case '*': {
+      const char *PrevChar = i > 0 ? Glob.data() + i - 1 : nullptr;
+      NumWildcards = 1;
+      ++i;
+      while (i < Glob.size() && Glob[i] == '*') {
+        ++NumWildcards;
+        ++i;
+      }
+      const char *NextChar = i < Glob.size() ? Glob.data() + i : nullptr;
+
+      if ((NumWildcards > 1) && (PrevChar == nullptr || *PrevChar == '/') &&
+          (NextChar == nullptr || *NextChar == '/')) {
+        RegexString += "(([^/]*(/|$))*)";
+      } else
+        RegexString += "([^/]*)";
+      break;
+    }
+    default:
+      if (RegexMetachars.find(C) != StringRef::npos)
+        RegexString.push_back('\\');
+      RegexString.push_back(C);
+    }
+  }
+  RegexString.push_back('$');
+  if (NumWildcards == 0)
+    return make_error<StringError>("not a glob", inconvertibleErrorCode());
+
+  auto Rule = Regex(RegexString);
+  std::string Error;
+  if (!Rule.isValid(Error))
+    return make_error<StringError>(Error, inconvertibleErrorCode());
+
+  return Rule;
+}
-- 
GitLab


From 25bcdab47807d56ba8a06211348c5f44f030f407 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 25 Mar 2024 12:35:37 -0700
Subject: [PATCH 178/404] [RISCV] Fix indentation and 80 columns in
 RISCVInstrInfoZvk.td. NFC

I didn't fix all the 80 column errors just the ones in template
arguments.
---
 llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td | 61 ++++++++++++----------
 1 file changed, 33 insertions(+), 28 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
index b4bd074b7101..67b2ad459360 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
@@ -273,7 +273,8 @@ multiclass VPseudoVALU_VV_NoMask_Zvk<string Constraint = ""> {
   }
 }
 
-multiclass VPseudoVALU_VI_NoMask_Zvk<Operand ImmType = simm5, string Constraint = ""> {
+multiclass VPseudoVALU_VI_NoMask_Zvk<Operand ImmType = simm5,
+                                     string Constraint = ""> {
   foreach m = MxListVF4 in {
     defvar mx = m.MX;
     defvar WriteVIALUV_MX = !cast<SchedWrite>("WriteVIALUV_" # mx);
@@ -285,7 +286,8 @@ multiclass VPseudoVALU_VI_NoMask_Zvk<Operand ImmType = simm5, string Constraint
   }
 }
 
-multiclass VPseudoVALU_VI_NoMaskTU_Zvk<Operand ImmType = uimm5, string Constraint = ""> {
+multiclass VPseudoVALU_VI_NoMaskTU_Zvk<Operand ImmType = uimm5,
+                                       string Constraint = ""> {
   foreach m = MxListVF4 in {
     defvar mx = m.MX;
     defvar WriteVIALUV_MX = !cast<SchedWrite>("WriteVIALUV_" # mx);
@@ -668,14 +670,14 @@ foreach vtiToWti = AllWidenableIntVectors in {
 //===----------------------------------------------------------------------===//
 
 class VPatUnaryNoMask_Zvk<string intrinsic_name,
-                      string inst,
-                      string kind,
-                      ValueType result_type,
-                      ValueType op2_type,
-                      int sew,
-                      LMULInfo vlmul,
-                      VReg result_reg_class,
-                      VReg op2_reg_class> :
+                          string inst,
+                          string kind,
+                          ValueType result_type,
+                          ValueType op2_type,
+                          int sew,
+                          LMULInfo vlmul,
+                          VReg result_reg_class,
+                          VReg op2_reg_class> :
   Pat<(result_type (!cast<Intrinsic>(intrinsic_name)
                    (result_type result_reg_class:$merge),
                    (op2_type op2_reg_class:$rs2),
@@ -686,15 +688,15 @@ class VPatUnaryNoMask_Zvk<string intrinsic_name,
                    GPR:$vl, sew, (XLenVT timm:$policy))>;
 
 class VPatUnaryNoMask_VS_Zvk<string intrinsic_name,
-                      string inst,
-                      string kind,
-                      ValueType result_type,
-                      ValueType op2_type,
-                      int sew,
-                      LMULInfo vlmul,
-                      LMULInfo vs2_lmul,
-                      VReg result_reg_class,
-                      VReg op2_reg_class> :
+                             string inst,
+                             string kind,
+                             ValueType result_type,
+                             ValueType op2_type,
+                             int sew,
+                             LMULInfo vlmul,
+                             LMULInfo vs2_lmul,
+                             VReg result_reg_class,
+                             VReg op2_reg_class> :
   Pat<(result_type (!cast<Intrinsic>(intrinsic_name)
                    (result_type result_reg_class:$merge),
                    (op2_type op2_reg_class:$rs2),
@@ -705,7 +707,7 @@ class VPatUnaryNoMask_VS_Zvk<string intrinsic_name,
                    GPR:$vl, sew, (XLenVT timm:$policy))>;
 
 multiclass VPatUnaryV_V_NoMask_Zvk<string intrinsic, string instruction,
-                              list<VTypeInfo> vtilist> {
+                                   list<VTypeInfo> vtilist> {
   foreach vti = vtilist in
     def : VPatUnaryNoMask_Zvk<intrinsic # "_vv", instruction, "VV",
                           vti.Vector, vti.Vector, vti.Log2SEW,
@@ -713,7 +715,7 @@ multiclass VPatUnaryV_V_NoMask_Zvk<string intrinsic, string instruction,
 }
 
 multiclass VPatUnaryV_S_NoMaskVectorCrypto<string intrinsic, string instruction,
-                               list<VTypeInfo> vtilist> {
+                                           list<VTypeInfo> vtilist> {
   foreach vti = vtilist in
     foreach vti_vs2 = ZvkI32IntegerVectors<vti.LMul.MX>.vs2_types in
       def : VPatUnaryNoMask_VS_Zvk<intrinsic # "_vs", instruction, "VS",
@@ -722,13 +724,13 @@ multiclass VPatUnaryV_S_NoMaskVectorCrypto<string intrinsic, string instruction,
 }
 
 multiclass VPatUnaryV_V_S_NoMask_Zvk<string intrinsic, string instruction,
-                                list<VTypeInfo> vtilist> {
+                                     list<VTypeInfo> vtilist> {
   defm : VPatUnaryV_V_NoMask_Zvk<intrinsic, instruction, vtilist>;
   defm : VPatUnaryV_S_NoMaskVectorCrypto<intrinsic, instruction, vtilist>;
 }
 
 multiclass VPatBinaryV_VV_NoMask<string intrinsic, string instruction,
-                          list<VTypeInfo> vtilist> {
+                                 list<VTypeInfo> vtilist> {
   foreach vti = vtilist in
     def : VPatTernaryNoMaskWithPolicy<intrinsic, instruction, "VV",
                                       vti.Vector, vti.Vector, vti.Vector,
@@ -737,7 +739,8 @@ multiclass VPatBinaryV_VV_NoMask<string intrinsic, string instruction,
 }
 
 multiclass VPatBinaryV_VI_NoMask<string intrinsic, string instruction,
-                          list<VTypeInfo> vtilist, Operand imm_type = tuimm5> {
+                                 list<VTypeInfo> vtilist,
+                                 Operand imm_type = tuimm5> {
   foreach vti = vtilist in
     def : VPatTernaryNoMaskWithPolicy<intrinsic, instruction, "VI",
                                       vti.Vector, vti.Vector, XLenVT,
@@ -746,7 +749,8 @@ multiclass VPatBinaryV_VI_NoMask<string intrinsic, string instruction,
 }
 
 multiclass VPatBinaryV_VI_NoMaskTU<string intrinsic, string instruction,
-                          list<VTypeInfo> vtilist, Operand imm_type = tuimm5> {
+                                   list<VTypeInfo> vtilist,
+                                   Operand imm_type = tuimm5> {
   foreach vti = vtilist in
     def : VPatBinaryNoMaskTU<intrinsic, instruction # "_VI_" # vti.LMul.MX,
                              vti.Vector, vti.Vector, XLenVT, vti.Log2SEW,
@@ -754,7 +758,7 @@ multiclass VPatBinaryV_VI_NoMaskTU<string intrinsic, string instruction,
 }
 
 multiclass VPatBinaryV_VV_NoMaskTU<string intrinsic, string instruction,
-                          list<VTypeInfo> vtilist> {
+                                   list<VTypeInfo> vtilist> {
   foreach vti = vtilist in
     def : VPatBinaryNoMaskTU<intrinsic, instruction # "_VV_" # vti.LMul.MX,
                              vti.Vector, vti.Vector, vti.Vector, vti.Log2SEW,
@@ -812,13 +816,14 @@ multiclass VPatBinaryV_VI_VROL<string intrinsic, string instruction,
 }
 
 multiclass VPatBinaryV_VV_VX_VROL<string intrinsic, string instruction,
-				  string instruction2, list<VTypeInfo> vtilist>
+                                  string instruction2, list<VTypeInfo> vtilist>
     : VPatBinaryV_VV<intrinsic, instruction, vtilist>,
       VPatBinaryV_VX_VROTATE<intrinsic, instruction, vtilist>,
       VPatBinaryV_VI_VROL<intrinsic, instruction2, vtilist>;
 
 multiclass VPatBinaryV_VV_VX_VI_VROR<string intrinsic, string instruction,
-                                     list<VTypeInfo> vtilist, Operand ImmType = uimm6>
+                                     list<VTypeInfo> vtilist,
+                                     Operand ImmType = uimm6>
     : VPatBinaryV_VV<intrinsic, instruction, vtilist>,
       VPatBinaryV_VX_VROTATE<intrinsic, instruction, vtilist>,
       VPatBinaryV_VI<intrinsic, instruction, vtilist, ImmType>;
-- 
GitLab


From f530eee4484e1b1e840238dbbd609ecac92d7c1e Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Mon, 25 Mar 2024 12:44:58 -0700
Subject: [PATCH 179/404] [bazel] Fix minor buildifier warnings (NFC) (#86566)

---
 utils/bazel/llvm-project-overlay/llvm/BUILD.bazel        | 9 +++++----
 .../llvm-project-overlay/llvm/unittests/BUILD.bazel      | 5 +++--
 utils/bazel/llvm-project-overlay/mlir/BUILD.bazel        | 2 +-
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index 65173f986585..26fc1041cf2e 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -623,7 +623,7 @@ cc_library(
     srcs = [
         "utils/TableGen/Common/GlobalISel/CodeExpander.cpp",
     ],
-    hdrs = glob([
+    hdrs = [
         # We have to include these headers here as well as in the `hdrs` below
         # to allow the `.cpp` files to use file-relative-inclusion to find
         # them, even though consumers of this library use inclusion relative to
@@ -631,7 +631,7 @@ cc_library(
         # This mixture appears to be incompatible with header modules.
         "utils/TableGen/Common/GlobalISel/CodeExpander.h",
         "utils/TableGen/Common/GlobalISel/CodeExpansions.h",
-    ]),
+    ],
     copts = llvm_copts,
     features = ["-header_modules"],
     strip_include_prefix = "utils/TableGen",
@@ -665,10 +665,11 @@ cc_binary(
             # listed here. MC uses headers produced by tablegen, so it cannot be a
             # regular dependency.
             "include/llvm/MC/*.h",
-            "include/llvm/TargetParser/SubtargetFeature.h",
         ],
         exclude = ["utils/TableGen/Common/GlobalISel/CodeExpander.cpp"],
-    ),
+    ) + [
+        "include/llvm/TargetParser/SubtargetFeature.h",
+    ],
     copts = llvm_copts,
     includes = ["utils/TableGen"],
     stamp = 0,
diff --git a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
index c8863af43c40..dd42f84d16dc 100644
--- a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
@@ -357,10 +357,11 @@ cc_test(
         [
             "IR/*.cpp",
             "IR/*.h",
-            "Support/KnownBitsTest.h",
         ],
         allow_empty = False,
-    ),
+    ) + [
+        "Support/KnownBitsTest.h",
+    ],
     shard_count = 20,
     deps = [
         "//llvm:Analysis",
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 88b46bdb326c..71fb97bff62f 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -8197,8 +8197,8 @@ cc_library(
     deps = [
         ":ConversionPassIncGen",
         ":EmitCDialect",
-        ":MemRefDialect",
         ":IR",
+        ":MemRefDialect",
         ":Pass",
         ":Support",
         ":TransformUtils",
-- 
GitLab


From b7d8c6188986f62573b9516fe27fdd0c7df1aaf9 Mon Sep 17 00:00:00 2001
From: Cyndy Ishida <cyndy_ishida@apple.com>
Date: Mon, 25 Mar 2024 15:45:06 -0400
Subject: [PATCH 180/404] [TextAPI] Add missing include for Expected type

---
 llvm/include/llvm/TextAPI/Utils.h | 1 +
 llvm/lib/TextAPI/Utils.cpp        | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/TextAPI/Utils.h b/llvm/include/llvm/TextAPI/Utils.h
index 353859ca4cea..319e0abce25b 100644
--- a/llvm/include/llvm/TextAPI/Utils.h
+++ b/llvm/include/llvm/TextAPI/Utils.h
@@ -14,6 +14,7 @@
 #define LLVM_TEXTAPI_UTILS_H
 
 #include "llvm/ADT/Twine.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Regex.h"
diff --git a/llvm/lib/TextAPI/Utils.cpp b/llvm/lib/TextAPI/Utils.cpp
index 61f4df1d4a6b..56b63ee01b39 100644
--- a/llvm/lib/TextAPI/Utils.cpp
+++ b/llvm/lib/TextAPI/Utils.cpp
@@ -155,7 +155,7 @@ bool llvm::MachO::isPrivateLibrary(StringRef Path, bool IsSymLink) {
 
 static StringLiteral RegexMetachars = "()^$|+.[]\\{}";
 
-Expected<Regex> llvm::MachO::createRegexFromGlob(StringRef Glob) {
+llvm::Expected<Regex> llvm::MachO::createRegexFromGlob(StringRef Glob) {
   SmallString<128> RegexString("^");
   unsigned NumWildcards = 0;
   for (unsigned i = 0; i < Glob.size(); ++i) {
@@ -191,7 +191,7 @@ Expected<Regex> llvm::MachO::createRegexFromGlob(StringRef Glob) {
   if (NumWildcards == 0)
     return make_error<StringError>("not a glob", inconvertibleErrorCode());
 
-  auto Rule = Regex(RegexString);
+  llvm::Regex Rule = Regex(RegexString);
   std::string Error;
   if (!Rule.isValid(Error))
     return make_error<StringError>(Error, inconvertibleErrorCode());
-- 
GitLab


From 540bbe15e617991f40fcf7033f0635bd3400bf5c Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Mon, 25 Mar 2024 12:47:01 -0700
Subject: [PATCH 181/404] [bazel] Add td_library target for OptParser.td
 (#86363)

This allows downstream consumers to depend on this file. Without this
target the include paths to this are mangled by bazel.
---
 utils/bazel/llvm-project-overlay/llvm/BUILD.bazel | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index 26fc1041cf2e..e9484e8fffa0 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -4,6 +4,7 @@
 
 load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
 load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
+load("//mlir:tblgen.bzl", "td_library")
 load(":binary_alias.bzl", "binary_alias")
 load(":config.bzl", "llvm_config_defines")
 load(":enum_targets_gen.bzl", "enum_targets_gen")
@@ -155,6 +156,12 @@ exports_files([
     "include/llvm/Config/abi-breaking.h.cmake",
 ])
 
+td_library(
+    name = "OptParserTdFiles",
+    srcs = ["include/llvm/Option/OptParser.td"],
+    includes = ["include"],
+)
+
 cc_library(
     name = "config",
     hdrs = [
-- 
GitLab


From 2eb010ddce2a8e7813a8e289004102011b1138db Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Mon, 25 Mar 2024 12:47:16 -0700
Subject: [PATCH 182/404] [bazel] Restrict libpfm to linux (#86565)

This target doesn't build on macOS (even with the upstream make based
build system) so this encodes that in the build without requiring
non-linux users to disable it manually with the starlark flag. The flag
is still respected for linux users.
---
 utils/bazel/llvm-project-overlay/llvm/BUILD.bazel |  6 ++++--
 utils/bazel/third_party_build/pfm.BUILD           | 12 ++++++++++--
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index e9484e8fffa0..0e658353c36f 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -3161,8 +3161,10 @@ cc_library(
         ":Support",
         ":Target",
         ":config",
-        ":maybe_pfm",
-    ],
+    ] + select({
+        "@platforms//os:linux": [":maybe_pfm"],
+        "//conditions:default": [],
+    }),
 )
 
 ################################################################################
diff --git a/utils/bazel/third_party_build/pfm.BUILD b/utils/bazel/third_party_build/pfm.BUILD
index fe908d4744da..07a1dc48b857 100644
--- a/utils/bazel/third_party_build/pfm.BUILD
+++ b/utils/bazel/third_party_build/pfm.BUILD
@@ -14,8 +14,12 @@ make_variant(
     copts = ["-w"],
     lib_name = "libpfm",
     lib_source = ":sources",
+    target_compatible_with = select({
+        "@platforms//os:linux": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
     toolchain = "@rules_foreign_cc//toolchains:preinstalled_autoconf_toolchain",
-    visibility = ["//visibility:public"],
+    visibility = ["//visibility:private"],
 )
 
 alias(
@@ -27,5 +31,9 @@ alias(
 cc_library(
     name = "pfm_system",
     linkopts = ["-lpfm"],
-    visibility = ["//visibility:public"],
+    target_compatible_with = select({
+        "@platforms//os:linux": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
+    visibility = ["//visibility:private"],
 )
-- 
GitLab


From 6ee39fb4c180c1425d865b2f360a09aa115549d1 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 25 Mar 2024 12:48:16 -0700
Subject: [PATCH 183/404] [RISCV] Remove unused forceMergeOpRead from
 SchedTernary class. NFC

Ternary operations cannot have a merge op in addition to their 3 sources
so there is no need for this option.
---
 llvm/lib/Target/RISCV/RISCVInstrInfoV.td | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
index d7807c120378..469ceaf6418e 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
@@ -157,10 +157,8 @@ class SchedBinaryMC<string write, string read0, string read1,
 
 // For instructions with three operands.
 class SchedTernary<string write, string read0, string read1, string read2,
-                   string mx, int sew = 0, bit forceMasked = 0,
-                   bit forceMergeOpRead = 0>
-    : SchedNary<write, [read0, read1, read2], mx, sew, forceMasked,
-                forceMergeOpRead>;
+                   string mx, int sew = 0, bit forceMasked = 0>
+    : SchedNary<write, [read0, read1, read2], mx, sew, forceMasked>;
 class SchedTernaryMC<string write, string read0, string read1, string read2,
                      int sew = 0, bit forceMasked = 1>:
   SchedNary<write, [read0, read1, read2], "WorstCase", sew, forceMasked>;
-- 
GitLab


From 2d7df3167b12e3f3330a6b6e2f83715f8bbad67e Mon Sep 17 00:00:00 2001
From: Cyndy Ishida <cyndy_ishida@apple.com>
Date: Mon, 25 Mar 2024 15:49:08 -0400
Subject: [PATCH 184/404] Revert "[InstallAPI] Add --extra* and --exclude* cli
 options for header input (#86522)"
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This reverts commit b7d8c6188986f62573b9516fe27fdd0c7df1aaf9. And
This reverts commit 2d40f179124f874aca4cf1145fdbc42fb8fb17f3.

It caused a build failure i'll need to reproduce.
` error: could not convert ‘Rule’ from ‘llvm::Regex’ to ‘llvm::Expected<llvm::Regex>’`
---
 .../clang/Basic/DiagnosticInstallAPIKinds.td  |    3 -
 clang/include/clang/InstallAPI/HeaderFile.h   |   54 +-
 clang/include/clang/InstallAPI/MachO.h        |    1 -
 clang/lib/InstallAPI/Frontend.cpp             |    2 -
 clang/lib/InstallAPI/HeaderFile.cpp           |   51 -
 .../Inputs/Simple/Extra/SimpleExtraAPI1.h     |    1 -
 .../Inputs/Simple/Extra/SimpleExtraAPI2.h     |    1 -
 .../Simple/Simple.framework/Headers/Basic.h   |  103 -
 .../Simple.framework/Headers/External.h       |   19 -
 .../Simple/Simple.framework/Headers/Simple.h  |   45 -
 .../Simple.framework/Headers/SimpleAPI.h      |    1 -
 .../PrivateHeaders/SimplePrivate.h            |    5 -
 .../PrivateHeaders/SimplePrivateSPI.h         |    2 -
 .../test/InstallAPI/Inputs/Simple/Simple.yaml | 3196 -----------------
 .../Inputs/Simple/SimpleInternalAPI.h         |    3 -
 .../Inputs/Simple/SimpleInternalAPI2.h        |    7 -
 .../Inputs/Simple/SimpleInternalSPI.h         |    5 -
 .../InstallAPI/extra-exclude-headers.test     |  207 --
 .../tools/clang-installapi/InstallAPIOpts.td  |   32 -
 clang/tools/clang-installapi/Options.cpp      |  121 -
 clang/tools/clang-installapi/Options.h        |   21 -
 llvm/include/llvm/TextAPI/Utils.h             |    7 -
 llvm/lib/TextAPI/Utils.cpp                    |   46 -
 23 files changed, 2 insertions(+), 3931 deletions(-)
 delete mode 100644 clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI1.h
 delete mode 100644 clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI2.h
 delete mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Basic.h
 delete mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/External.h
 delete mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Simple.h
 delete mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/SimpleAPI.h
 delete mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivate.h
 delete mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivateSPI.h
 delete mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.yaml
 delete mode 100644 clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI.h
 delete mode 100644 clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI2.h
 delete mode 100644 clang/test/InstallAPI/Inputs/Simple/SimpleInternalSPI.h
 delete mode 100644 clang/test/InstallAPI/extra-exclude-headers.test

diff --git a/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td b/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td
index 27df731fa286..a4c6e630ac5f 100644
--- a/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td
+++ b/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td
@@ -15,9 +15,6 @@ let CategoryName = "Command line" in {
 def err_cannot_write_file : Error<"cannot write file '%0': %1">;
 def err_no_install_name : Error<"no install name specified: add -install_name <path>">;
 def err_no_output_file: Error<"no output file specified">;
-def err_no_such_header_file : Error<"no such %select{public|private|project}1 header file: '%0'">;
-def warn_no_such_excluded_header_file : Warning<"no such excluded %select{public|private}0 header file: '%1'">, InGroup<InstallAPIViolation>;
-def warn_glob_did_not_match: Warning<"glob '%0' did not match any header file">, InGroup<InstallAPIViolation>;
 } // end of command line category.
 
 let CategoryName = "Verification" in {
diff --git a/clang/include/clang/InstallAPI/HeaderFile.h b/clang/include/clang/InstallAPI/HeaderFile.h
index 235b4da3add8..70e83bbb3e76 100644
--- a/clang/include/clang/InstallAPI/HeaderFile.h
+++ b/clang/include/clang/InstallAPI/HeaderFile.h
@@ -13,9 +13,7 @@
 #ifndef LLVM_CLANG_INSTALLAPI_HEADERFILE_H
 #define LLVM_CLANG_INSTALLAPI_HEADERFILE_H
 
-#include "clang/Basic/FileManager.h"
 #include "clang/Basic/LangStandard.h"
-#include "clang/InstallAPI/MachO.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Regex.h"
@@ -58,10 +56,6 @@ class HeaderFile {
   std::string IncludeName;
   /// Supported language mode for header.
   std::optional<clang::Language> Language;
-  /// Exclude header file from processing.
-  bool Excluded{false};
-  /// Add header file to processing.
-  bool Extra{false};
 
 public:
   HeaderFile() = delete;
@@ -77,48 +71,17 @@ public:
   StringRef getIncludeName() const { return IncludeName; }
   StringRef getPath() const { return FullPath; }
 
-  void setExtra(bool V = true) { Extra = V; }
-  void setExcluded(bool V = true) { Excluded = V; }
-  bool isExtra() const { return Extra; }
-  bool isExcluded() const { return Excluded; }
-
   bool useIncludeName() const {
     return Type != HeaderType::Project && !IncludeName.empty();
   }
 
   bool operator==(const HeaderFile &Other) const {
-    return std::tie(Type, FullPath, IncludeName, Language, Excluded, Extra) ==
+    return std::tie(Type, FullPath, IncludeName, Language) ==
            std::tie(Other.Type, Other.FullPath, Other.IncludeName,
-                    Other.Language, Other.Excluded, Other.Extra);
+                    Other.Language);
   }
 };
 
-/// Glob that represents a pattern of header files to retreive.
-class HeaderGlob {
-private:
-  std::string GlobString;
-  llvm::Regex Rule;
-  HeaderType Type;
-  bool FoundMatch{false};
-
-public:
-  HeaderGlob(StringRef GlobString, llvm::Regex &&, HeaderType Type);
-
-  /// Create a header glob from string for the header access level.
-  static llvm::Expected<std::unique_ptr<HeaderGlob>>
-  create(StringRef GlobString, HeaderType Type);
-
-  /// Query if provided header matches glob.
-  bool match(const HeaderFile &Header);
-
-  /// Query if a header was matched in the glob, used primarily for error
-  /// reporting.
-  bool didMatch() { return FoundMatch; }
-
-  /// Provide back input glob string.
-  StringRef str() { return GlobString; }
-};
-
 /// Assemble expected way header will be included by clients.
 /// As in what maps inside the brackets of `#include <IncludeName.h>`
 /// For example,
@@ -130,19 +93,6 @@ public:
 std::optional<std::string> createIncludeHeaderName(const StringRef FullPath);
 using HeaderSeq = std::vector<HeaderFile>;
 
-/// Determine if Path is a header file.
-/// It does not touch the file system.
-///
-/// \param  Path File path to file.
-bool isHeaderFile(StringRef Path);
-
-/// Given input directory, collect all header files.
-///
-/// \param FM FileManager for finding input files.
-/// \param Directory Path to directory file.
-llvm::Expected<PathSeq> enumerateFiles(clang::FileManager &FM,
-                                       StringRef Directory);
-
 } // namespace clang::installapi
 
 #endif // LLVM_CLANG_INSTALLAPI_HEADERFILE_H
diff --git a/clang/include/clang/InstallAPI/MachO.h b/clang/include/clang/InstallAPI/MachO.h
index 4961c596fd68..f0dea8bbd24c 100644
--- a/clang/include/clang/InstallAPI/MachO.h
+++ b/clang/include/clang/InstallAPI/MachO.h
@@ -40,7 +40,6 @@ using SymbolSet = llvm::MachO::SymbolSet;
 using SimpleSymbol = llvm::MachO::SimpleSymbol;
 using FileType = llvm::MachO::FileType;
 using PackedVersion = llvm::MachO::PackedVersion;
-using PathSeq = llvm::MachO::PathSeq;
 using Target = llvm::MachO::Target;
 using TargetList = llvm::MachO::TargetList;
 
diff --git a/clang/lib/InstallAPI/Frontend.cpp b/clang/lib/InstallAPI/Frontend.cpp
index e07ccb14e0b8..12cd5fcbc22b 100644
--- a/clang/lib/InstallAPI/Frontend.cpp
+++ b/clang/lib/InstallAPI/Frontend.cpp
@@ -138,8 +138,6 @@ std::unique_ptr<MemoryBuffer> createInputBuffer(InstallAPIContext &Ctx) {
   SmallString<4096> Contents;
   raw_svector_ostream OS(Contents);
   for (const HeaderFile &H : Ctx.InputHeaders) {
-    if (H.isExcluded())
-      continue;
     if (H.getType() != Ctx.Type)
       continue;
     if (Ctx.LangMode == Language::C || Ctx.LangMode == Language::CXX)
diff --git a/clang/lib/InstallAPI/HeaderFile.cpp b/clang/lib/InstallAPI/HeaderFile.cpp
index 0b7041ec8147..c2d8372741ee 100644
--- a/clang/lib/InstallAPI/HeaderFile.cpp
+++ b/clang/lib/InstallAPI/HeaderFile.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/InstallAPI/HeaderFile.h"
-#include "llvm/TextAPI/Utils.h"
 
 using namespace llvm;
 namespace clang::installapi {
@@ -35,54 +34,4 @@ std::optional<std::string> createIncludeHeaderName(const StringRef FullPath) {
   return Matches[1].drop_front(Matches[1].rfind('/') + 1).str() + "/" +
          Matches[3].str();
 }
-
-bool isHeaderFile(StringRef Path) {
-  return StringSwitch<bool>(sys::path::extension(Path))
-      .Cases(".h", ".H", ".hh", ".hpp", ".hxx", true)
-      .Default(false);
-}
-
-llvm::Expected<PathSeq> enumerateFiles(FileManager &FM, StringRef Directory) {
-  PathSeq Files;
-  std::error_code EC;
-  auto &FS = FM.getVirtualFileSystem();
-  for (llvm::vfs::recursive_directory_iterator i(FS, Directory, EC), ie;
-       i != ie; i.increment(EC)) {
-    if (EC)
-      return errorCodeToError(EC);
-
-    // Skip files that do not exist. This usually happens for broken symlinks.
-    if (FS.status(i->path()) == std::errc::no_such_file_or_directory)
-      continue;
-
-    StringRef Path = i->path();
-    if (isHeaderFile(Path))
-      Files.emplace_back(Path);
-  }
-
-  return Files;
-}
-
-HeaderGlob::HeaderGlob(StringRef GlobString, Regex &&Rule, HeaderType Type)
-    : GlobString(GlobString), Rule(std::move(Rule)), Type(Type) {}
-
-bool HeaderGlob::match(const HeaderFile &Header) {
-  if (Header.getType() != Type)
-    return false;
-
-  bool Match = Rule.match(Header.getPath());
-  if (Match)
-    FoundMatch = true;
-  return Match;
-}
-
-Expected<std::unique_ptr<HeaderGlob>> HeaderGlob::create(StringRef GlobString,
-                                                         HeaderType Type) {
-  auto Rule = MachO::createRegexFromGlob(GlobString);
-  if (!Rule)
-    return Rule.takeError();
-
-  return std::make_unique<HeaderGlob>(GlobString, std::move(*Rule), Type);
-}
-
 } // namespace clang::installapi
diff --git a/clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI1.h b/clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI1.h
deleted file mode 100644
index 83a5b9507de3..000000000000
--- a/clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI1.h
+++ /dev/null
@@ -1 +0,0 @@
-extern int extraGlobalAPI1;
diff --git a/clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI2.h b/clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI2.h
deleted file mode 100644
index 34fe3364bba8..000000000000
--- a/clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI2.h
+++ /dev/null
@@ -1 +0,0 @@
-extern int extraGlobalAPI2;
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Basic.h b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Basic.h
deleted file mode 100644
index 08412bb2de28..000000000000
--- a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Basic.h
+++ /dev/null
@@ -1,103 +0,0 @@
-#import <Foundation/Foundation.h>
-
-// Basic class with no super class
-@interface Basic1
-@end
-
-@interface Basic2 : NSObject
-@end
-
-@interface Basic3 : NSObject
-@property BOOL property1;
-@property(readonly) BOOL property2;
-@property(getter=isProperty3) BOOL property3;
-@property BOOL dynamicProp;
-@end
-
-@interface Basic4 : NSObject {
-@public
-  BOOL ivar1;
-@protected
-  BOOL ivar2;
-@package
-  BOOL ivar3;
-@private
-  BOOL ivar4;
-}
-@end
-
-__attribute__((visibility("hidden"))) @interface Basic4_1 : NSObject {
-@public
-  BOOL ivar1;
-@protected
-  BOOL ivar2;
-@package
-  BOOL ivar3;
-@private
-  BOOL ivar4;
-}
-@end
-
-@interface Basic4_2 : NSObject {
-@private
-  BOOL ivar4;
-@package
-  BOOL ivar3;
-@protected
-  BOOL ivar2;
-@public
-  BOOL ivar1;
-}
-@end
-
-@interface Basic5 : NSObject
-+ (void)aClassMethod;
-- (void)anInstanceMethod;
-@end
-
-@interface Basic6 : NSObject
-@end
-
-@interface Basic6 () {
-@public
-  BOOL ivar1;
-}
-@property BOOL property1;
-- (void)anInstanceMethodFromAnExtension;
-@end
-
-@interface Basic6 (Foo)
-@property BOOL property2;
-- (void)anInstanceMethodFromACategory;
-@end
-
-__attribute__((visibility("hidden")))
-@interface Basic7 : NSObject
-@end
-
-@interface Basic7 ()
-- (void) anInstanceMethodFromAnHiddenExtension;
-@end
-
-@interface Basic8 : NSObject
-+ (void)useSameName;
-@end
-
-// Classes and protocols can have the same name. For now they would only clash
-// in the selector map if the protocl starts with '_'.
-@protocol _A
-- (void)aMethod;
-@end
-
-@interface A : NSObject
-- (void)aMethod NS_AVAILABLE(10_11, 9_0);
-- (void)bMethod NS_UNAVAILABLE;
-@end
-
-@interface Basic9 : NSObject
-@property(readonly) BOOL aProperty NS_AVAILABLE(10_10, 8_0);
-@end
-
-@interface Basic9 (deprecated)
-@property(readwrite) BOOL aProperty NS_DEPRECATED_MAC(10_8, 10_10);
-@end
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/External.h b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/External.h
deleted file mode 100644
index 5dc3c92f34c2..000000000000
--- a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/External.h
+++ /dev/null
@@ -1,19 +0,0 @@
-#import <Foundation/Foundation.h>
-
-// Sub-class an external defined ObjC Class.
-@interface ExternalManagedObject : NSManagedObject
-- (void)foo;
-@end
-
-// Add category to external defined ObjC Class.
-@interface NSManagedObject (Simple)
-- (int)supportsSimple;
-@end
-
-// CoreData Accessors are dynamically generated and have no implementation.
-@interface ExternalManagedObject (CoreDataGeneratedAccessors)
-- (void)addChildObject:(ExternalManagedObject *)value;
-- (void)removeChildObject:(ExternalManagedObject *)value;
-- (void)addChild:(NSSet *)values;
-- (void)removeChild:(NSSet *)values;
-@end
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Simple.h b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Simple.h
deleted file mode 100644
index 12c77098a8d9..000000000000
--- a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Simple.h
+++ /dev/null
@@ -1,45 +0,0 @@
-#import <Foundation/Foundation.h>
-
-// Useless forward declaration. This is used for testing.
-@class FooBar;
-@protocol FooProtocol;
-
-@protocol ForwardProcotol;
-
-// Test public global.
-extern int publicGlobalVariable;
-
-// Test weak public global.
-extern int weakPublicGlobalVariable __attribute__((weak));
-
-// Test public ObjC class
-@interface Simple : NSObject
-@end
-
-__attribute__((objc_exception))
-@interface Base : NSObject
-@end
-
-@interface SubClass : Base
-@end
-
-@protocol BaseProtocol
-- (void) baseMethod;
-@end
-
-NS_AVAILABLE(10_11, 9_0)
-@protocol FooProtocol <BaseProtocol>
-- (void) protocolMethod;
-@end
-
-@protocol BarProtocol
-- (void) barMethod;
-@end
-
-@interface FooClass <FooProtocol, BarProtocol>
-@end
-
-// Create an empty category conforms to a forward declared protocol.
-// <rdar://problem/35605892>
-@interface FooClass (Test) <ForwardProcotol>
-@end
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/SimpleAPI.h b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/SimpleAPI.h
deleted file mode 100644
index d953fac966da..000000000000
--- a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/SimpleAPI.h
+++ /dev/null
@@ -1 +0,0 @@
-extern int otherFrameworkAPI;
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivate.h b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivate.h
deleted file mode 100644
index 5a28cda3928e..000000000000
--- a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivate.h
+++ /dev/null
@@ -1,5 +0,0 @@
-// Test private global variable.
-extern int privateGlobalVariable;
-
-// Test weak private global.
-extern int weakPrivateGlobalVariable __attribute__((weak));
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivateSPI.h b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivateSPI.h
deleted file mode 100644
index c9aca30fa82f..000000000000
--- a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivateSPI.h
+++ /dev/null
@@ -1,2 +0,0 @@
-// Test private global variable.
-extern int otherFrameworkSPI;
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.yaml b/clang/test/InstallAPI/Inputs/Simple/Simple.yaml
deleted file mode 100644
index 998e51f1a67d..000000000000
--- a/clang/test/InstallAPI/Inputs/Simple/Simple.yaml
+++ /dev/null
@@ -1,3196 +0,0 @@
---- !mach-o
-FileHeader:
-  magic:           0xFEEDFACF
-  cputype:         0x1000007
-  cpusubtype:      0x3
-  filetype:        0x6
-  ncmds:           15
-  sizeofcmds:      1952
-  flags:           0x118085
-  reserved:        0x0
-LoadCommands:
-  - cmd:             LC_SEGMENT_64
-    cmdsize:         472
-    segname:         __TEXT
-    vmaddr:          0
-    vmsize:          12288
-    fileoff:         0
-    filesize:        12288
-    maxprot:         5
-    initprot:        5
-    nsects:          5
-    flags:           0
-    Sections:
-      - sectname:        __text
-        segname:         __TEXT
-        addr:            0x1BC0
-        size:            180
-        offset:          0x1BC0
-        align:           0
-        reloff:          0x0
-        nreloc:          0
-        flags:           0x80000400
-        reserved1:       0x0
-        reserved2:       0x0
-        reserved3:       0x0
-        content:         554889E50FBE47085DC3554889E58857085DC3554889E50FBE47095DC3554889E50FBE470A5DC3554889E588570A5DC3554889E55DC3554889E55DC3554889E55DC3554889E50FBE47095DC3554889E58857095DC3554889E5B8010000005DC3554889E55DC3554889E55DC3554889E55DC3554889E55DC3554889E5B0015DC3554889E55DC3554889E55DC3554889E55DC3554889E50FBE47085DC3554889E55DC3554889E55DC3554889E55DC3554889E55DC3
-      - sectname:        __cstring
-        segname:         __TEXT
-        addr:            0x1C74
-        size:            296
-        offset:          0x1C74
-        align:           0
-        reloff:          0x0
-        nreloc:          0
-        flags:           0x2
-        reserved1:       0x0
-        reserved2:       0x0
-        reserved3:       0x0
-        content:         53696D706C65004261736500537562436C6173730053696D706C65496E7465726E616C4150490053696D706C65496E7465726E616C53504900426173696331004261736963320042617369633300426173696334004261736963345F31004261736963345F32004261736963350042617369633600466F6F004261736963370045787465726E616C4D616E616765644F626A6563740048696464656E436C61737300426173696338004100426173696339006465707265636174656400466F6F436C61737300466F6F50726F746F636F6C004261736550726F746F636F6C0042617250726F746F636F6C0050726976617465005072697661746550726F746F636F6C0063313640303A380076323040303A3863313600630076313640303A380042313640303A3800
-      - sectname:        __objc_methname
-        segname:         __TEXT
-        addr:            0x1D9C
-        size:            450
-        offset:          0x1D9C
-        align:           0
-        reloff:          0x0
-        nreloc:          0
-        flags:           0x2
-        reserved1:       0x0
-        reserved2:       0x0
-        reserved3:       0x0
-        content:         70726F7065727479310073657450726F7065727479313A0070726F70657274793200697350726F7065727479330073657450726F7065727479333A0070726F7065727479330054632C5670726F7065727479310054632C522C5670726F7065727479320054632C47697350726F7065727479332C5670726F7065727479330064796E616D696350726F700054632C440069766172310069766172320069766172330069766172340061436C6173734D6574686F6400616E496E7374616E63654D6574686F6400616E496E7374616E63654D6574686F6446726F6D416E457874656E73696F6E0073657450726F7065727479323A00616E496E7374616E63654D6574686F6446726F6D4143617465676F727900546300616E496E7374616E63654D6574686F6446726F6D416E48696464656E457874656E73696F6E00666F6F00737570706F72747353696D706C650075736553616D654E616D6500614D6574686F64006150726F7065727479005F6150726F70657274790054632C522C565F6150726F706572747900626173654D6574686F640070726F746F636F6C4D6574686F64006261724D6574686F64007072697661746550726F636F746F6C4D6574686F6400
-      - sectname:        __unwind_info
-        segname:         __TEXT
-        addr:            0x1F60
-        size:            4152
-        offset:          0x1F60
-        align:           2
-        reloff:          0x0
-        nreloc:          0
-        flags:           0x0
-        reserved1:       0x0
-        reserved2:       0x0
-        reserved3:       0x0
-        content:         010000001C000000010000002000000000000000200000000200000000000001C01B00003800000038000000741C00000000000038000000030000000C0001001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
-      - sectname:        __eh_frame
-        segname:         __TEXT
-        addr:            0x2F98
-        size:            24
-        offset:          0x2F98
-        align:           3
-        reloff:          0x0
-        nreloc:          0
-        flags:           0x6000000B
-        reserved1:       0x0
-        reserved2:       0x0
-        reserved3:       0x0
-        content:         1400000000000000017A520001781001100C070890010000
-  - cmd:             LC_SEGMENT_64
-    cmdsize:         792
-    segname:         __DATA
-    vmaddr:          12288
-    vmsize:          8192
-    fileoff:         12288
-    filesize:        8192
-    maxprot:         3
-    initprot:        3
-    nsects:          9
-    flags:           0
-    Sections:
-      - sectname:        __objc_const
-        segname:         __DATA
-        addr:            0x3000
-        size:            4952
-        offset:          0x3000
-        align:           3
-        reloff:          0x0
-        nreloc:          0
-        flags:           0x0
-        reserved1:       0x0
-        reserved2:       0x0
-        reserved3:       0x0
-        content:         010000002800000028000000000000000000000000000000741C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000008000000000000000000000000000000741C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000028000000280000000000000000000000000000007B1C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000008000000080000000000000000000000000000007B1C0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000007B1C000000000000D043000000000000010000002800000028000000000000000000000000000000801C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000800000008000000000000000000000000000000801C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000801C0000000000002044000000000000010000002800000028000000000000000000000000000000891C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000800000008000000000000000000000000000000891C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000891C00000000000070440000000000000100000028000000280000000000000000000000000000009B1C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000008000000080000000000000000000000000000009B1C0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000009B1C000000000000C044000000000000030000002800000028000000000000000000000000000000AD1C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000AD1C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000002800000028000000000000000000000000000000B41C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000008000000000000000000000000000000B41C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000002800000028000000000000000000000000000000BB1C0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000018000000050000009C1D000000000000771D000000000000C01B000000000000A61D0000000000007F1D000000000000CA1B000000000000B41D000000000000771D000000000000D31B000000000000BE1D000000000000771D000000000000DD1B000000000000CA1D0000000000007F1D000000000000E71B000000000000200000000300000098490000000000009C1D0000000000008A1D0000000000000000000001000000A049000000000000B41D0000000000008A1D0000000000000000000001000000A849000000000000D81D0000000000008A1D000000000000000000000100000010000000040000009C1D000000000000E21D000000000000B41D000000000000F01D000000000000D81D000000000000001E0000000000001B1E000000000000271E00000000000000000000080000000B000000000000000000000000000000BB1C00000000000098340000000000000000000000000000183500000000000000000000000000008035000000000000010000002800000028000000000000000000000000000000C21C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000004000000B0490000000000002C1E0000000000008A1D0000000000000000000001000000B849000000000000321E0000000000008A1D0000000000000000000001000000C049000000000000381E0000000000008A1D0000000000000000000001000000C8490000000000003E1E0000000000008A1D000000000000000000000100000000000000080000000C000000000000000000000000000000C21C00000000000000000000000000000000000000000000583600000000000000000000000000000000000000000000110000002800000028000000000000000000000000000000C91C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000004000000D0490000000000002C1E0000000000008A1D0000000000000000000001000000D849000000000000321E0000000000008A1D0000000000000000000001000000E049000000000000381E0000000000008A1D0000000000000000000001000000E8490000000000003E1E0000000000008A1D000000000000000000000100000010000000080000000C000000000000000000000000000000C91C00000000000000000000000000000000000000000000703700000000000000000000000000000000000000000000010000002800000028000000000000000000000000000000D21C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000004000000F0490000000000003E1E0000000000008A1D0000000000000000000001000000F849000000000000381E0000000000008A1D0000000000000000000001000000004A000000000000321E0000000000008A1D0000000000000000000001000000084A0000000000002C1E0000000000008A1D000000000000000000000100000000000000080000000C000000000000000000000000000000D21C000000000000000000000000000000000000000000008838000000000000000000000000000000000000000000001800000001000000441E0000000000008C1D000000000000F01B000000000000010000002800000028000000000000000000000000000000DB1C000000000000583900000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000511E0000000000008C1D000000000000F61B000000000000000000000800000008000000000000000000000000000000DB1C000000000000C0390000000000000000000000000000000000000000000000000000000000000000000000000000010000002800000028000000000000000000000000000000E21C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001800000003000000621E0000000000008C1D000000000000FC1B0000000000009C1D000000000000771D000000000000021C000000000000A61D0000000000007F1D0000000000000C1C0000000000002000000002000000104A0000000000002C1E0000000000008A1D0000000000000000000001000000184A0000000000009C1D0000000000008A1D000000000000000000000100000010000000010000009C1D000000000000E21D00000000000000000000080000000A000000000000000000000000000000E21C000000000000703A0000000000000000000000000000C03A0000000000000000000000000000083B0000000000001800000003000000B41D000000000000771D000000000000151C000000000000821E0000000000007F1D000000000000201C000000000000901E0000000000008C1D000000000000261C0000000000001000000001000000B41D000000000000AE1E000000000000E91C0000000000004047000000000000683B00000000000000000000000000000000000000000000B83B00000000000000000000000000004000000000000000110000002800000028000000000000000000000000000000ED1C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000B11E0000000000008C1D0000000000002C1C000000000000100000000800000008000000000000000000000000000000ED1C000000000000583C0000000000000000000000000000000000000000000000000000000000000000000000000000010000002800000028000000000000000000000000000000F41C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000D71E0000000000008C1D000000000000321C000000000000000000000800000008000000000000000000000000000000F41C000000000000083D00000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000DB1E000000000000941D000000000000381C000000000000741C0000000000000000000000000000703D000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000001100000028000000280000000000000000000000000000000A1D000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000008000000080000000000000000000000000000000A1D000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000EA1E0000000000008C1D000000000000401C000000000000010000002800000028000000000000000000000000000000161D000000000000603E00000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000EA1E0000000000008C1D000000000000461C000000000000000000000800000008000000000000000000000000000000161D000000000000C83E00000000000000000000000000000000000000000000000000000000000000000000000000000100000028000000280000000000000000000000000000001D1D000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000F61E0000000000008C1D0000000000004C1C0000000000000000000008000000080000000000000000000000000000001D1D000000000000783F00000000000000000000000000000000000000000000000000000000000000000000000000000100000028000000280000000000000000000000000000001F1D000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000FE1E000000000000771D000000000000521C0000000000002000000001000000204A000000000000081F0000000000008A1D00000000000000000000010000001000000001000000FE1E000000000000131F0000000000000000000008000000090000000000000000000000000000001F1D000000000000284000000000000000000000000000004840000000000000000000000000000070400000000000001000000001000000FE1E000000000000AE1E000000000000261D0000000000002049000000000000000000000000000000000000000000000000000000000000D040000000000000000000000000000040000000000000001800000001000000241F0000000000008C1D00000000000000000000000000008C1D0000000000000100000000000000284A000000000000000000000000000018000000010000002F1F0000000000008C1D00000000000000000000000000008C1D00000000000018000000010000003E1F0000000000008C1D00000000000000000000000000008C1D0000000000000200000000000000884A000000000000E84A0000000000000000000000000000030000002800000028000000000000000000000000000000311D0000000000000000000000000000B8410000000000000000000000000000000000000000000000000000000000001800000003000000241F0000000000008C1D0000000000005C1C0000000000002F1F0000000000008C1D000000000000621C0000000000003E1F0000000000008C1D000000000000681C000000000000020000000000000000000000000000000000000000000000311D0000000000002042000000000000B8410000000000000000000000000000000000000000000000000000000000001800000001000000481F0000000000008C1D0000000000006E1C0000000000001800000001000000481F0000000000008C1D00000000000000000000000000008C1D0000000000000100000000000000484B00000000000000000000000000005F1D0000000000004849000000000000B84200000000000000000000000000000043000000000000000000000000000000000000000000004000000000000000
-      - sectname:        __objc_data
-        segname:         __DATA
-        addr:            0x4358
-        size:            1600
-        offset:          0x4358
-        align:           3
-        reloff:          0x0
-        nreloc:          0
-        flags:           0x0
-        reserved1:       0x0
-        reserved2:       0x0
-        reserved3:       0x0
-        content:         000000000000000000000000000000000000000000000000000000000000000000300000000000005843000000000000000000000000000000000000000000000000000000000000483000000000000000000000000000000000000000000000000000000000000000000000000000009030000000000000A843000000000000000000000000000000000000000000000000000000000000D8300000000000000000000000000000A843000000000000000000000000000000000000000000003831000000000000F843000000000000D0430000000000000000000000000000000000000000000080310000000000000000000000000000000000000000000000000000000000000000000000000000E03100000000000048440000000000000000000000000000000000000000000000000000000000002832000000000000000000000000000000000000000000000000000000000000000000000000000088320000000000009844000000000000000000000000000000000000000000000000000000000000D032000000000000104500000000000000000000000000000000000000000000000000000000000078330000000000001045000000000000E8440000000000000000000000000000000000000000000030330000000000000000000000000000000000000000000000000000000000000000000000000000C03300000000000038450000000000000000000000000000000000000000000000000000000000000834000000000000000000000000000000000000000000000000000000000000000000000000000050340000000000008845000000000000000000000000000000000000000000000000000000000000C83500000000000000000000000000000000000000000000000000000000000000000000000000001036000000000000D845000000000000000000000000000000000000000000000000000000000000E036000000000000000000000000000000000000000000000000000000000000000000000000000028370000000000002846000000000000000000000000000000000000000000000000000000000000F837000000000000000000000000000000000000000000000000000000000000000000000000000040380000000000007846000000000000000000000000000000000000000000000000000000000000103900000000000000000000000000000000000000000000000000000000000000000000000000007839000000000000C846000000000000000000000000000000000000000000000000000000000000E0390000000000000000000000000000000000000000000000000000000000000000000000000000283A0000000000001847000000000000000000000000000000000000000000000000000000000000203B0000000000000000000000000000000000000000000000000000000000000000000000000000103C0000000000006847000000000000000000000000000000000000000000000000000000000000783C0000000000000000000000000000000000000000000000000000000000000000000000000000C03C000000000000B847000000000000000000000000000000000000000000000000000000000000283D0000000000000000000000000000000000000000000000000000000000000000000000000000D03D0000000000000848000000000000000000000000000000000000000000000000000000000000183E0000000000000000000000000000000000000000000000000000000000000000000000000000803E0000000000005848000000000000000000000000000000000000000000000000000000000000E83E0000000000000000000000000000000000000000000000000000000000000000000000000000303F000000000000A848000000000000000000000000000000000000000000000000000000000000983F0000000000000000000000000000000000000000000000000000000000000000000000000000E03F000000000000F8480000000000000000000000000000000000000000000000000000000000008840000000000000704900000000000000000000000000000000000000000000000000000000000070420000000000007049000000000000484900000000000000000000000000000000000000000000D841000000000000
-      - sectname:        __objc_ivar
-        segname:         __DATA
-        addr:            0x4998
-        size:            144
-        offset:          0x4998
-        align:           3
-        reloff:          0x0
-        nreloc:          0
-        flags:           0x0
-        reserved1:       0x0
-        reserved2:       0x0
-        reserved3:       0x0
-        content:         080000000000000009000000000000000A00000000000000080000000000000009000000000000000A000000000000000B00000000000000080000000000000009000000000000000A000000000000000B00000000000000080000000000000009000000000000000A000000000000000B00000000000000080000000000000009000000000000000800000000000000
-      - sectname:        __data
-        segname:         __DATA
-        addr:            0x4A28
-        size:            392
-        offset:          0x4A28
-        align:           3
-        reloff:          0x0
-        nreloc:          0
-        flags:           0x0
-        reserved1:       0x0
-        reserved2:       0x0
-        reserved3:       0x0
-        content:         0000000000000000461D000000000000000000000000000028410000000000000000000000000000000000000000000000000000000000000000000000000000600000000000000048410000000000000000000000000000000000000000000000000000000000003A1D00000000000050410000000000006841000000000000000000000000000000000000000000000000000000000000000000000000000060000000000000008841000000000000000000000000000000000000000000000000000000000000531D0000000000000000000000000000904100000000000000000000000000000000000000000000000000000000000000000000000000006000000000000000B041000000000000000000000000000000000000000000000000000000000000671D0000000000000000000000000000D84200000000000000000000000000000000000000000000000000000000000000000000000000006000000000000000F842000000000000000000000000000000000000000000000000000000000000
-      - sectname:        __objc_protolist
-        segname:         __DATA
-        addr:            0x4BB0
-        size:            32
-        offset:          0x4BB0
-        align:           3
-        reloff:          0x0
-        nreloc:          0
-        flags:           0x1000000B
-        reserved1:       0x0
-        reserved2:       0x0
-        reserved3:       0x0
-        content:         284A000000000000884A000000000000E84A000000000000484B000000000000
-      - sectname:        __objc_classlist
-        segname:         __DATA
-        addr:            0x4BD0
-        size:            160
-        offset:          0x4BD0
-        align:           3
-        reloff:          0x0
-        nreloc:          0
-        flags:           0x10000000
-        reserved1:       0x0
-        reserved2:       0x0
-        reserved3:       0x0
-        content:         8043000000000000D04300000000000020440000000000007044000000000000C044000000000000E8440000000000006045000000000000B04500000000000000460000000000005046000000000000A046000000000000F04600000000000040470000000000009047000000000000E04700000000000030480000000000008048000000000000D04800000000000020490000000000004849000000000000
-      - sectname:        __objc_catlist
-        segname:         __DATA
-        addr:            0x4C70
-        size:            32
-        offset:          0x4C70
-        align:           3
-        reloff:          0x0
-        nreloc:          0
-        flags:           0x10000000
-        reserved1:       0x0
-        reserved2:       0x0
-        reserved3:       0x0
-        content:         D03B000000000000903D000000000000E8400000000000001843000000000000
-      - sectname:        __objc_imageinfo
-        segname:         __DATA
-        addr:            0x4C90
-        size:            8
-        offset:          0x4C90
-        align:           0
-        reloff:          0x0
-        nreloc:          0
-        flags:           0x0
-        reserved1:       0x0
-        reserved2:       0x0
-        reserved3:       0x0
-        content:         '0000000040000000'
-      - sectname:        __common
-        segname:         __DATA
-        addr:            0x4C98
-        size:            16
-        offset:          0x0
-        align:           2
-        reloff:          0x0
-        nreloc:          0
-        flags:           0x1
-        reserved1:       0x0
-        reserved2:       0x0
-        reserved3:       0x0
-  - cmd:             LC_SEGMENT_64
-    cmdsize:         72
-    segname:         __LINKEDIT
-    vmaddr:          20480
-    vmsize:          10272
-    fileoff:         20480
-    filesize:        10272
-    maxprot:         1
-    initprot:        1
-    nsects:          0
-    flags:           0
-  - cmd:             LC_DYLD_INFO_ONLY
-    cmdsize:         48
-    rebase_off:      20480
-    rebase_size:     320
-    bind_off:        20800
-    bind_size:       480
-    weak_bind_off:   0
-    weak_bind_size:  0
-    lazy_bind_off:   0
-    lazy_bind_size:  0
-    export_off:      21280
-    export_size:     896
-  - cmd:             LC_SYMTAB
-    cmdsize:         24
-    symoff:          22208
-    nsyms:           187
-    stroff:          25200
-    strsize:         5552
-  - cmd:             LC_DYSYMTAB
-    cmdsize:         80
-    ilocalsym:       0
-    nlocalsym:       131
-    iextdefsym:      131
-    nextdefsym:      49
-    iundefsym:       180
-    nundefsym:       7
-    tocoff:          0
-    ntoc:            0
-    modtaboff:       0
-    nmodtab:         0
-    extrefsymoff:    0
-    nextrefsyms:     0
-    indirectsymoff:  0
-    nindirectsyms:   0
-    extreloff:       0
-    nextrel:         0
-    locreloff:       0
-    nlocrel:         0
-  - cmd:             LC_ID_DYLIB
-    cmdsize:         88
-    dylib:
-      name:            24
-      timestamp:       0
-      current_version: 66051
-      compatibility_version: 65536
-    Content:         '/System/Library/Frameworks/Simple.framework/Versions/A/Simple'
-    ZeroPadBytes:    3
-  - cmd:             LC_UUID
-    cmdsize:         24
-    uuid:            4C4C441D-5555-3144-A104-DD1AF4EF8FE7
-  - cmd:             LC_VERSION_MIN_MACOSX
-    cmdsize:         16
-    version:         658432
-    sdk:             983040
-  - cmd:             LC_LOAD_DYLIB
-    cmdsize:         96
-    dylib:
-      name:            24
-      timestamp:       0
-      current_version: 197722368
-      compatibility_version: 19660800
-    Content:         '/System/Library/Frameworks/Foundation.framework/Versions/C/Foundation'
-    ZeroPadBytes:    3
-  - cmd:             LC_LOAD_DYLIB
-    cmdsize:         56
-    dylib:
-      name:            24
-      timestamp:       0
-      current_version: 14942208
-      compatibility_version: 65536
-    Content:         '/usr/lib/libobjc.A.dylib'
-    ZeroPadBytes:    8
-  - cmd:             LC_LOAD_DYLIB
-    cmdsize:         96
-    dylib:
-      name:            24
-      timestamp:       0
-      current_version: 91750400
-      compatibility_version: 65536
-    Content:         '/System/Library/Frameworks/CoreData.framework/Versions/A/CoreData'
-    ZeroPadBytes:    7
-  - cmd:             LC_LOAD_DYLIB
-    cmdsize:         56
-    dylib:
-      name:            24
-      timestamp:       0
-      current_version: 88539136
-      compatibility_version: 65536
-    Content:         '/usr/lib/libSystem.B.dylib'
-    ZeroPadBytes:    6
-  - cmd:             LC_FUNCTION_STARTS
-    cmdsize:         16
-    dataoff:         22176
-    datasize:        32
-  - cmd:             LC_DATA_IN_CODE
-    cmdsize:         16
-    dataoff:         22208
-    datasize:        0
-LinkEditData:
-  RebaseOpcodes:
-    - Opcode:          REBASE_OPCODE_SET_TYPE_IMM
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB
-      Imm:             1
-      ExtraData:       [ 0x18 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
-      Imm:             0
-      ExtraData:       [ 0x3, 0x40 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x30 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x40 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x30 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x40 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x30 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x40 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x30 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
-      Imm:             0
-      ExtraData:       [ 0x4, 0x40 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x30 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             15
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             8
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
-      Imm:             0
-      ExtraData:       [ 0x2, 0x8 ]
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x30 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             4
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
-      Imm:             0
-      ExtraData:       [ 0x2, 0x10 ]
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x30 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             4
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
-      Imm:             0
-      ExtraData:       [ 0x2, 0x10 ]
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x30 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             4
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
-      Imm:             0
-      ExtraData:       [ 0x2, 0x10 ]
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             5
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             7
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x30 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             9
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
-      Imm:             0
-      ExtraData:       [ 0x2, 0x8 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             9
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             5
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
-      Imm:             0
-      ExtraData:       [ 0x2, 0x28 ]
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             7
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x30 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             5
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             4
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
-      Imm:             0
-      ExtraData:       [ 0x2, 0x40 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x30 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             5
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             7
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x30 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             7
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x30 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
-      Imm:             0
-      ExtraData:       [ 0x2, 0x8 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             4
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x18 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
-      Imm:             0
-      ExtraData:       [ 0x2, 0x8 ]
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x8 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x8 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             4
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
-      Imm:             0
-      ExtraData:       [ 0x2, 0x8 ]
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             9
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             4
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
-      Imm:             0
-      ExtraData:       [ 0x2, 0x8 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x38 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x20 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
-      Imm:             0
-      ExtraData:       [ 0x2, 0x8 ]
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             1
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x20 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x20 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x20 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x20 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x20 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x20 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x20 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x20 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x20 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x20 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x20 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x20 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x20 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x20 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x20 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             2
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x98 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
-      Imm:             0
-      ExtraData:       [ 0x2, 0x8 ]
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             4
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x18 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
-      Imm:             3
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             5
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x18 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
-      Imm:             0
-      ExtraData:       [ 0x2, 0x8 ]
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             4
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x18 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
-      Imm:             0
-      ExtraData:       [ 0x2, 0x8 ]
-    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
-      Imm:             4
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
-      Imm:             0
-      ExtraData:       [ 0x18 ]
-    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES
-      Imm:             0
-      ExtraData:       [ 0x1C ]
-    - Opcode:          REBASE_OPCODE_DONE
-      Imm:             0
-  BindOpcodes:
-    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
-      Imm:             0
-      Symbol:          _objc_ehtype_vtable
-    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
-      Imm:             1
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_SET_DYLIB_ORDINAL_IMM
-      Imm:             2
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB
-      Imm:             1
-      ULEBExtraData:   [ 0x120 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_SET_ADDEND_SLEB
-      Imm:             0
-      SLEBExtraData:   [ 16 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0xA0 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0xA0 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0xA0 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
-      Imm:             0
-      Symbol:          '_OBJC_CLASS_$_NSManagedObject'
-    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
-      Imm:             1
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_SET_DYLIB_ORDINAL_IMM
-      Imm:             3
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0xA78 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_SET_ADDEND_SLEB
-      Imm:             0
-      SLEBExtraData:   [ 0 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0xA48 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
-      Imm:             0
-      Symbol:          '_OBJC_METACLASS_$_NSObject'
-    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
-      Imm:             1
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_SET_DYLIB_ORDINAL_IMM
-      Imm:             2
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0xFFFFFFFFFFFFFB68 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x40 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x40 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x48 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x40 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x90 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x40 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x40 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x40 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x40 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x40 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x40 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x40 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x40 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x48 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x40 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x40 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x40 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
-      Imm:             0
-      Symbol:          __objc_empty_cache
-    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
-      Imm:             1
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0xFFFFFFFFFFFFFA60 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x20 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
-      Imm:             0
-      Symbol:          '_OBJC_CLASS_$_NSObject'
-    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
-      Imm:             1
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0xFFFFFFFFFFFFFA00 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x48 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x98 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x48 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x98 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x48 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x48 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x48 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x48 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x48 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x48 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x48 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x98 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x48 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x48 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0x48 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
-      Imm:             0
-      Symbol:          '_OBJC_METACLASS_$_NSManagedObject'
-    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
-      Imm:             1
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_SET_DYLIB_ORDINAL_IMM
-      Imm:             3
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
-      Imm:             0
-      ULEBExtraData:   [ 0xFFFFFFFFFFFFFE90 ]
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DO_BIND
-      Imm:             0
-      Symbol:          ''
-    - Opcode:          BIND_OPCODE_DONE
-      Imm:             0
-      Symbol:          ''
-  ExportTrie:
-    TerminalSize:    0
-    NodeOffset:      0
-    Name:            ''
-    Flags:           0x0
-    Address:         0x0
-    Other:           0x0
-    ImportName:      ''
-    Children:
-      - TerminalSize:    0
-        NodeOffset:      5
-        Name:            _
-        Flags:           0x0
-        Address:         0x0
-        Other:           0x0
-        ImportName:      ''
-        Children:
-          - TerminalSize:    0
-            NodeOffset:      41
-            Name:            p
-            Flags:           0x0
-            Address:         0x0
-            Other:           0x0
-            ImportName:      ''
-            Children:
-              - TerminalSize:    4
-                NodeOffset:      86
-                Name:            rivateGlobalVariable
-                Flags:           0x0
-                Address:         0x4CA0
-                Other:           0x0
-                ImportName:      ''
-              - TerminalSize:    4
-                NodeOffset:      92
-                Name:            ublicGlobalVariable
-                Flags:           0x0
-                Address:         0x4CA4
-                Other:           0x0
-                ImportName:      ''
-          - TerminalSize:    0
-            NodeOffset:      98
-            Name:            extraGlobalAPI
-            Flags:           0x0
-            Address:         0x0
-            Other:           0x0
-            ImportName:      ''
-            Children:
-              - TerminalSize:    4
-                NodeOffset:      106
-                Name:            '1'
-                Flags:           0x0
-                Address:         0x4C98
-                Other:           0x0
-                ImportName:      ''
-              - TerminalSize:    4
-                NodeOffset:      112
-                Name:            '2'
-                Flags:           0x0
-                Address:         0x4C9C
-                Other:           0x0
-                ImportName:      ''
-          - TerminalSize:    0
-            NodeOffset:      118
-            Name:            weakP
-            Flags:           0x0
-            Address:         0x0
-            Other:           0x0
-            ImportName:      ''
-            Children:
-              - TerminalSize:    4
-                NodeOffset:      165
-                Name:            rivateGlobalVariable
-                Flags:           0x4
-                Address:         0x4BAC
-                Other:           0x0
-                ImportName:      ''
-              - TerminalSize:    4
-                NodeOffset:      171
-                Name:            ublicGlobalVariable
-                Flags:           0x4
-                Address:         0x4BA8
-                Other:           0x0
-                ImportName:      ''
-          - TerminalSize:    0
-            NodeOffset:      177
-            Name:            OBJC_
-            Flags:           0x0
-            Address:         0x0
-            Other:           0x0
-            ImportName:      ''
-            Children:
-              - TerminalSize:    0
-                NodeOffset:      232
-                Name:            'IVAR_$_Basic'
-                Flags:           0x0
-                Address:         0x0
-                Other:           0x0
-                ImportName:      ''
-                Children:
-                  - TerminalSize:    4
-                    NodeOffset:      248
-                    Name:            6.ivar1
-                    Flags:           0x0
-                    Address:         0x4A10
-                    Other:           0x0
-                    ImportName:      ''
-                  - TerminalSize:    0
-                    NodeOffset:      254
-                    Name:            '4'
-                    Flags:           0x0
-                    Address:         0x0
-                    Other:           0x0
-                    ImportName:      ''
-                    Children:
-                      - TerminalSize:    0
-                        NodeOffset:      274
-                        Name:            .ivar
-                        Flags:           0x0
-                        Address:         0x0
-                        Other:           0x0
-                        ImportName:      ''
-                        Children:
-                          - TerminalSize:    4
-                            NodeOffset:      284
-                            Name:            '2'
-                            Flags:           0x0
-                            Address:         0x49B8
-                            Other:           0x0
-                            ImportName:      ''
-                          - TerminalSize:    4
-                            NodeOffset:      290
-                            Name:            '1'
-                            Flags:           0x0
-                            Address:         0x49B0
-                            Other:           0x0
-                            ImportName:      ''
-                      - TerminalSize:    0
-                        NodeOffset:      296
-                        Name:            _2.ivar
-                        Flags:           0x0
-                        Address:         0x0
-                        Other:           0x0
-                        ImportName:      ''
-                        Children:
-                          - TerminalSize:    4
-                            NodeOffset:      306
-                            Name:            '1'
-                            Flags:           0x0
-                            Address:         0x4A08
-                            Other:           0x0
-                            ImportName:      ''
-                          - TerminalSize:    4
-                            NodeOffset:      312
-                            Name:            '2'
-                            Flags:           0x0
-                            Address:         0x4A00
-                            Other:           0x0
-                            ImportName:      ''
-              - TerminalSize:    0
-                NodeOffset:      318
-                Name:            'METACLASS_$_'
-                Flags:           0x0
-                Address:         0x0
-                Other:           0x0
-                ImportName:      ''
-                Children:
-                  - TerminalSize:    4
-                    NodeOffset:      369
-                    Name:            FooClass
-                    Flags:           0x0
-                    Address:         0x4970
-                    Other:           0x0
-                    ImportName:      ''
-                  - TerminalSize:    4
-                    NodeOffset:      375
-                    Name:            ExternalManagedObject
-                    Flags:           0x0
-                    Address:         0x47B8
-                    Other:           0x0
-                    ImportName:      ''
-                  - TerminalSize:    0
-                    NodeOffset:      381
-                    Name:            S
-                    Flags:           0x0
-                    Address:         0x0
-                    Other:           0x0
-                    ImportName:      ''
-                    Children:
-                      - TerminalSize:    4
-                        NodeOffset:      401
-                        Name:            imple
-                        Flags:           0x0
-                        Address:         0x4358
-                        Other:           0x0
-                        ImportName:      ''
-                        Children:
-                          - TerminalSize:    0
-                            NodeOffset:      418
-                            Name:            Internal
-                            Flags:           0x0
-                            Address:         0x0
-                            Other:           0x0
-                            ImportName:      ''
-                            Children:
-                              - TerminalSize:    4
-                                NodeOffset:      432
-                                Name:            SPI
-                                Flags:           0x0
-                                Address:         0x4498
-                                Other:           0x0
-                                ImportName:      ''
-                              - TerminalSize:    4
-                                NodeOffset:      438
-                                Name:            API
-                                Flags:           0x0
-                                Address:         0x4448
-                                Other:           0x0
-                                ImportName:      ''
-                      - TerminalSize:    4
-                        NodeOffset:      444
-                        Name:            ubClass
-                        Flags:           0x0
-                        Address:         0x43F8
-                        Other:           0x0
-                        ImportName:      ''
-                  - TerminalSize:    0
-                    NodeOffset:      450
-                    Name:            Bas
-                    Flags:           0x0
-                    Address:         0x0
-                    Other:           0x0
-                    ImportName:      ''
-                    Children:
-                      - TerminalSize:    4
-                        NodeOffset:      461
-                        Name:            e
-                        Flags:           0x0
-                        Address:         0x43A8
-                        Other:           0x0
-                        ImportName:      ''
-                      - TerminalSize:    0
-                        NodeOffset:      467
-                        Name:            ic
-                        Flags:           0x0
-                        Address:         0x0
-                        Other:           0x0
-                        ImportName:      ''
-                        Children:
-                          - TerminalSize:    4
-                            NodeOffset:      501
-                            Name:            '2'
-                            Flags:           0x0
-                            Address:         0x4538
-                            Other:           0x0
-                            ImportName:      ''
-                          - TerminalSize:    4
-                            NodeOffset:      507
-                            Name:            '3'
-                            Flags:           0x0
-                            Address:         0x4588
-                            Other:           0x0
-                            ImportName:      ''
-                          - TerminalSize:    4
-                            NodeOffset:      513
-                            Name:            '5'
-                            Flags:           0x0
-                            Address:         0x46C8
-                            Other:           0x0
-                            ImportName:      ''
-                          - TerminalSize:    4
-                            NodeOffset:      519
-                            Name:            '4'
-                            Flags:           0x0
-                            Address:         0x45D8
-                            Other:           0x0
-                            ImportName:      ''
-                            Children:
-                              - TerminalSize:    4
-                                NodeOffset:      530
-                                Name:            _2
-                                Flags:           0x0
-                                Address:         0x4678
-                                Other:           0x0
-                                ImportName:      ''
-                          - TerminalSize:    4
-                            NodeOffset:      536
-                            Name:            '9'
-                            Flags:           0x0
-                            Address:         0x48F8
-                            Other:           0x0
-                            ImportName:      ''
-                          - TerminalSize:    4
-                            NodeOffset:      542
-                            Name:            '8'
-                            Flags:           0x0
-                            Address:         0x4858
-                            Other:           0x0
-                            ImportName:      ''
-                          - TerminalSize:    4
-                            NodeOffset:      548
-                            Name:            '6'
-                            Flags:           0x0
-                            Address:         0x4718
-                            Other:           0x0
-                            ImportName:      ''
-                          - TerminalSize:    4
-                            NodeOffset:      554
-                            Name:            '1'
-                            Flags:           0x0
-                            Address:         0x4510
-                            Other:           0x0
-                            ImportName:      ''
-                  - TerminalSize:    4
-                    NodeOffset:      560
-                    Name:            A
-                    Flags:           0x0
-                    Address:         0x48A8
-                    Other:           0x0
-                    ImportName:      ''
-              - TerminalSize:    0
-                NodeOffset:      566
-                Name:            'EHTYPE_$_'
-                Flags:           0x0
-                Address:         0x0
-                Other:           0x0
-                ImportName:      ''
-                Children:
-                  - TerminalSize:    3
-                    NodeOffset:      579
-                    Name:            Base
-                    Flags:           0x0
-                    Address:         0x3120
-                    Other:           0x0
-                    ImportName:      ''
-                  - TerminalSize:    0
-                    NodeOffset:      584
-                    Name:            S
-                    Flags:           0x0
-                    Address:         0x0
-                    Other:           0x0
-                    ImportName:      ''
-                    Children:
-                      - TerminalSize:    3
-                        NodeOffset:      612
-                        Name:            ubClass
-                        Flags:           0x0
-                        Address:         0x31C8
-                        Other:           0x0
-                        ImportName:      ''
-                      - TerminalSize:    0
-                        NodeOffset:      617
-                        Name:            impleInternal
-                        Flags:           0x0
-                        Address:         0x0
-                        Other:           0x0
-                        ImportName:      ''
-                        Children:
-                          - TerminalSize:    3
-                            NodeOffset:      631
-                            Name:            SPI
-                            Flags:           0x0
-                            Address:         0x3318
-                            Other:           0x0
-                            ImportName:      ''
-                          - TerminalSize:    3
-                            NodeOffset:      636
-                            Name:            API
-                            Flags:           0x0
-                            Address:         0x3270
-                            Other:           0x0
-                            ImportName:      ''
-              - TerminalSize:    0
-                NodeOffset:      641
-                Name:            'CLASS_$_'
-                Flags:           0x0
-                Address:         0x0
-                Other:           0x0
-                ImportName:      ''
-                Children:
-                  - TerminalSize:    4
-                    NodeOffset:      692
-                    Name:            A
-                    Flags:           0x0
-                    Address:         0x48D0
-                    Other:           0x0
-                    ImportName:      ''
-                  - TerminalSize:    4
-                    NodeOffset:      698
-                    Name:            ExternalManagedObject
-                    Flags:           0x0
-                    Address:         0x47E0
-                    Other:           0x0
-                    ImportName:      ''
-                  - TerminalSize:    4
-                    NodeOffset:      704
-                    Name:            FooClass
-                    Flags:           0x0
-                    Address:         0x4948
-                    Other:           0x0
-                    ImportName:      ''
-                  - TerminalSize:    0
-                    NodeOffset:      710
-                    Name:            S
-                    Flags:           0x0
-                    Address:         0x0
-                    Other:           0x0
-                    ImportName:      ''
-                    Children:
-                      - TerminalSize:    4
-                        NodeOffset:      730
-                        Name:            ubClass
-                        Flags:           0x0
-                        Address:         0x4420
-                        Other:           0x0
-                        ImportName:      ''
-                      - TerminalSize:    4
-                        NodeOffset:      736
-                        Name:            imple
-                        Flags:           0x0
-                        Address:         0x4380
-                        Other:           0x0
-                        ImportName:      ''
-                        Children:
-                          - TerminalSize:    0
-                            NodeOffset:      753
-                            Name:            Internal
-                            Flags:           0x0
-                            Address:         0x0
-                            Other:           0x0
-                            ImportName:      ''
-                            Children:
-                              - TerminalSize:    4
-                                NodeOffset:      767
-                                Name:            API
-                                Flags:           0x0
-                                Address:         0x4470
-                                Other:           0x0
-                                ImportName:      ''
-                              - TerminalSize:    4
-                                NodeOffset:      773
-                                Name:            SPI
-                                Flags:           0x0
-                                Address:         0x44C0
-                                Other:           0x0
-                                ImportName:      ''
-                  - TerminalSize:    0
-                    NodeOffset:      779
-                    Name:            Bas
-                    Flags:           0x0
-                    Address:         0x0
-                    Other:           0x0
-                    ImportName:      ''
-                    Children:
-                      - TerminalSize:    4
-                        NodeOffset:      790
-                        Name:            e
-                        Flags:           0x0
-                        Address:         0x43D0
-                        Other:           0x0
-                        ImportName:      ''
-                      - TerminalSize:    0
-                        NodeOffset:      796
-                        Name:            ic
-                        Flags:           0x0
-                        Address:         0x0
-                        Other:           0x0
-                        ImportName:      ''
-                        Children:
-                          - TerminalSize:    4
-                            NodeOffset:      830
-                            Name:            '1'
-                            Flags:           0x0
-                            Address:         0x44E8
-                            Other:           0x0
-                            ImportName:      ''
-                          - TerminalSize:    4
-                            NodeOffset:      836
-                            Name:            '3'
-                            Flags:           0x0
-                            Address:         0x45B0
-                            Other:           0x0
-                            ImportName:      ''
-                          - TerminalSize:    4
-                            NodeOffset:      842
-                            Name:            '4'
-                            Flags:           0x0
-                            Address:         0x4600
-                            Other:           0x0
-                            ImportName:      ''
-                            Children:
-                              - TerminalSize:    4
-                                NodeOffset:      853
-                                Name:            _2
-                                Flags:           0x0
-                                Address:         0x46A0
-                                Other:           0x0
-                                ImportName:      ''
-                          - TerminalSize:    4
-                            NodeOffset:      859
-                            Name:            '2'
-                            Flags:           0x0
-                            Address:         0x4560
-                            Other:           0x0
-                            ImportName:      ''
-                          - TerminalSize:    4
-                            NodeOffset:      865
-                            Name:            '8'
-                            Flags:           0x0
-                            Address:         0x4880
-                            Other:           0x0
-                            ImportName:      ''
-                          - TerminalSize:    4
-                            NodeOffset:      871
-                            Name:            '9'
-                            Flags:           0x0
-                            Address:         0x4920
-                            Other:           0x0
-                            ImportName:      ''
-                          - TerminalSize:    4
-                            NodeOffset:      877
-                            Name:            '6'
-                            Flags:           0x0
-                            Address:         0x4740
-                            Other:           0x0
-                            ImportName:      ''
-                          - TerminalSize:    4
-                            NodeOffset:      883
-                            Name:            '5'
-                            Flags:           0x0
-                            Address:         0x46F0
-                            Other:           0x0
-                            ImportName:      ''
-  NameList:
-    - n_strx:          2
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7104
-    - n_strx:          22
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7114
-    - n_strx:          46
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7123
-    - n_strx:          66
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7133
-    - n_strx:          88
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7143
-    - n_strx:          112
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7152
-    - n_strx:          135
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7158
-    - n_strx:          162
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7164
-    - n_strx:          204
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7170
-    - n_strx:          224
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7180
-    - n_strx:          248
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7189
-    - n_strx:          273
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7200
-    - n_strx:          302
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7206
-    - n_strx:          347
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7212
-    - n_strx:          395
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7218
-    - n_strx:          424
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7224
-    - n_strx:          466
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7232
-    - n_strx:          488
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7238
-    - n_strx:          510
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7244
-    - n_strx:          523
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7250
-    - n_strx:          543
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7260
-    - n_strx:          566
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7266
-    - n_strx:          593
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7272
-    - n_strx:          615
-      n_type:          0xE
-      n_sect:          1
-      n_desc:          0
-      n_value:         7278
-    - n_strx:          658
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         12288
-    - n_strx:          687
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         12360
-    - n_strx:          712
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         12432
-    - n_strx:          739
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         12504
-    - n_strx:          762
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         12600
-    - n_strx:          793
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         12672
-    - n_strx:          820
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         12768
-    - n_strx:          860
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         12840
-    - n_strx:          896
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         12936
-    - n_strx:          936
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         13008
-    - n_strx:          972
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         13176
-    - n_strx:          997
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         13104
-    - n_strx:          1026
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         13248
-    - n_strx:          1055
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         13320
-    - n_strx:          1080
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         13392
-    - n_strx:          1109
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         13464
-    - n_strx:          1142
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         13592
-    - n_strx:          1177
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         13696
-    - n_strx:          1203
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         13768
-    - n_strx:          1228
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         13840
-    - n_strx:          1257
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         13912
-    - n_strx:          1292
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         14048
-    - n_strx:          1317
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         14120
-    - n_strx:          1348
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         14192
-    - n_strx:          1385
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         14328
-    - n_strx:          1412
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         14400
-    - n_strx:          1443
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         14472
-    - n_strx:          1480
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         14608
-    - n_strx:          1507
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         14680
-    - n_strx:          1537
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         14712
-    - n_strx:          1566
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         14784
-    - n_strx:          1599
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         14816
-    - n_strx:          1624
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         14888
-    - n_strx:          1653
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         14960
-    - n_strx:          1686
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         15040
-    - n_strx:          1721
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         15112
-    - n_strx:          1747
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         15136
-    - n_strx:          1772
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         15208
-    - n_strx:          1820
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         15288
-    - n_strx:          1852
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         15312
-    - n_strx:          1883
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         15376
-    - n_strx:          1912
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         15448
-    - n_strx:          1945
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         15480
-    - n_strx:          1970
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         15552
-    - n_strx:          2014
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         15624
-    - n_strx:          2062
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         15656
-    - n_strx:          2102
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         15728
-    - n_strx:          2162
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         15760
-    - n_strx:          2205
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         15824
-    - n_strx:          2239
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         15896
-    - n_strx:          2269
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         15968
-    - n_strx:          2299
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16000
-    - n_strx:          2328
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16072
-    - n_strx:          2361
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16104
-    - n_strx:          2386
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16176
-    - n_strx:          2410
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16248
-    - n_strx:          2438
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16280
-    - n_strx:          2458
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16352
-    - n_strx:          2487
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16424
-    - n_strx:          2520
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16456
-    - n_strx:          2555
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16496
-    - n_strx:          2581
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16520
-    - n_strx:          2606
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16592
-    - n_strx:          2645
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16616
-    - n_strx:          2683
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         17008
-    - n_strx:          2710
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16856
-    - n_strx:          2741
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16680
-    - n_strx:          2789
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16712
-    - n_strx:          2833
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16720
-    - n_strx:          2868
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16744
-    - n_strx:          2915
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16776
-    - n_strx:          2958
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16784
-    - n_strx:          3005
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16816
-    - n_strx:          3048
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16824
-    - n_strx:          3082
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         16928
-    - n_strx:          3117
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         17080
-    - n_strx:          3171
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         17112
-    - n_strx:          3222
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         17144
-    - n_strx:          3269
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         17152
-    - n_strx:          3316
-      n_type:          0xE
-      n_sect:          6
-      n_desc:          0
-      n_value:         17176
-    - n_strx:          4000
-      n_type:          0x1E
-      n_sect:          7
-      n_desc:          0
-      n_value:         17960
-    - n_strx:          4027
-      n_type:          0x1E
-      n_sect:          7
-      n_desc:          0
-      n_value:         18000
-    - n_strx:          4192
-      n_type:          0x1E
-      n_sect:          7
-      n_desc:          0
-      n_value:         18280
-    - n_strx:          4217
-      n_type:          0x1E
-      n_sect:          7
-      n_desc:          0
-      n_value:         18320
-    - n_strx:          4314
-      n_type:          0x1E
-      n_sect:          7
-      n_desc:          0
-      n_value:         18440
-    - n_strx:          4344
-      n_type:          0x1E
-      n_sect:          7
-      n_desc:          0
-      n_value:         18480
-    - n_strx:          4548
-      n_type:          0x1E
-      n_sect:          8
-      n_desc:          0
-      n_value:         18840
-    - n_strx:          4578
-      n_type:          0x1E
-      n_sect:          8
-      n_desc:          0
-      n_value:         18848
-    - n_strx:          4608
-      n_type:          0x1E
-      n_sect:          8
-      n_desc:          0
-      n_value:         18856
-    - n_strx:          4690
-      n_type:          0x1E
-      n_sect:          8
-      n_desc:          0
-      n_value:         18880
-    - n_strx:          4716
-      n_type:          0x1E
-      n_sect:          8
-      n_desc:          0
-      n_value:         18888
-    - n_strx:          4742
-      n_type:          0x1E
-      n_sect:          8
-      n_desc:          0
-      n_value:         18896
-    - n_strx:          4770
-      n_type:          0x1E
-      n_sect:          8
-      n_desc:          0
-      n_value:         18904
-    - n_strx:          4798
-      n_type:          0x1E
-      n_sect:          8
-      n_desc:          0
-      n_value:         18912
-    - n_strx:          4826
-      n_type:          0x1E
-      n_sect:          8
-      n_desc:          0
-      n_value:         18920
-    - n_strx:          4854
-      n_type:          0x1E
-      n_sect:          8
-      n_desc:          0
-      n_value:         18928
-    - n_strx:          4882
-      n_type:          0x1E
-      n_sect:          8
-      n_desc:          0
-      n_value:         18936
-    - n_strx:          4992
-      n_type:          0x1E
-      n_sect:          8
-      n_desc:          0
-      n_value:         18968
-    - n_strx:          5022
-      n_type:          0x1E
-      n_sect:          8
-      n_desc:          0
-      n_value:         18976
-    - n_strx:          5053
-      n_type:          0x1E
-      n_sect:          9
-      n_desc:          0
-      n_value:         18984
-    - n_strx:          5084
-      n_type:          0x1E
-      n_sect:          9
-      n_desc:          0
-      n_value:         19080
-    - n_strx:          5114
-      n_type:          0x1E
-      n_sect:          9
-      n_desc:          0
-      n_value:         19176
-    - n_strx:          5144
-      n_type:          0x1E
-      n_sect:          9
-      n_desc:          0
-      n_value:         19272
-    - n_strx:          5231
-      n_type:          0x1E
-      n_sect:          10
-      n_desc:          0
-      n_value:         19376
-    - n_strx:          5268
-      n_type:          0x1E
-      n_sect:          10
-      n_desc:          0
-      n_value:         19384
-    - n_strx:          5304
-      n_type:          0x1E
-      n_sect:          10
-      n_desc:          0
-      n_value:         19392
-    - n_strx:          5340
-      n_type:          0x1E
-      n_sect:          10
-      n_desc:          0
-      n_value:         19400
-    - n_strx:          3353
-      n_type:          0xF
-      n_sect:          14
-      n_desc:          0
-      n_value:         19608
-    - n_strx:          3370
-      n_type:          0xF
-      n_sect:          14
-      n_desc:          0
-      n_value:         19612
-    - n_strx:          3387
-      n_type:          0xF
-      n_sect:          14
-      n_desc:          0
-      n_value:         19616
-    - n_strx:          3410
-      n_type:          0xF
-      n_sect:          14
-      n_desc:          0
-      n_value:         19620
-    - n_strx:          3432
-      n_type:          0xF
-      n_sect:          6
-      n_desc:          0
-      n_value:         12576
-    - n_strx:          3452
-      n_type:          0xF
-      n_sect:          6
-      n_desc:          0
-      n_value:         12744
-    - n_strx:          3476
-      n_type:          0xF
-      n_sect:          6
-      n_desc:          0
-      n_value:         12912
-    - n_strx:          3509
-      n_type:          0xF
-      n_sect:          6
-      n_desc:          0
-      n_value:         13080
-    - n_strx:          3542
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         17240
-    - n_strx:          3567
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         17280
-    - n_strx:          3588
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         17320
-    - n_strx:          3611
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         17360
-    - n_strx:          3630
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         17400
-    - n_strx:          3657
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         17440
-    - n_strx:          3680
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         17480
-    - n_strx:          3716
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         17520
-    - n_strx:          3748
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         17560
-    - n_strx:          3784
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         17600
-    - n_strx:          3816
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         17640
-    - n_strx:          3837
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         17680
-    - n_strx:          3862
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         17720
-    - n_strx:          3887
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         17760
-    - n_strx:          3908
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         17800
-    - n_strx:          3933
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         17840
-    - n_strx:          3954
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         17880
-    - n_strx:          3979
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         17920
-    - n_strx:          4050
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         18040
-    - n_strx:          4077
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         18080
-    - n_strx:          4100
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         18120
-    - n_strx:          4125
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         18160
-    - n_strx:          4146
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         18200
-    - n_strx:          4171
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         18240
-    - n_strx:          4238
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         18360
-    - n_strx:          4278
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         18400
-    - n_strx:          4370
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         18520
-    - n_strx:          4395
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         18560
-    - n_strx:          4416
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         18600
-    - n_strx:          4436
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         18640
-    - n_strx:          4452
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         18680
-    - n_strx:          4477
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         18720
-    - n_strx:          4498
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         18760
-    - n_strx:          4521
-      n_type:          0xF
-      n_sect:          7
-      n_desc:          0
-      n_value:         18800
-    - n_strx:          4638
-      n_type:          0xF
-      n_sect:          8
-      n_desc:          0
-      n_value:         18864
-    - n_strx:          4664
-      n_type:          0xF
-      n_sect:          8
-      n_desc:          0
-      n_value:         18872
-    - n_strx:          4910
-      n_type:          0xF
-      n_sect:          8
-      n_desc:          0
-      n_value:         18944
-    - n_strx:          4938
-      n_type:          0xF
-      n_sect:          8
-      n_desc:          0
-      n_value:         18952
-    - n_strx:          4966
-      n_type:          0xF
-      n_sect:          8
-      n_desc:          0
-      n_value:         18960
-    - n_strx:          5178
-      n_type:          0xF
-      n_sect:          9
-      n_desc:          128
-      n_value:         19368
-    - n_strx:          5204
-      n_type:          0xF
-      n_sect:          9
-      n_desc:          128
-      n_value:         19372
-    - n_strx:          5380
-      n_type:          0x1
-      n_sect:          0
-      n_desc:          768
-      n_value:         0
-    - n_strx:          5410
-      n_type:          0x1
-      n_sect:          0
-      n_desc:          512
-      n_value:         0
-    - n_strx:          5433
-      n_type:          0x1
-      n_sect:          0
-      n_desc:          768
-      n_value:         0
-    - n_strx:          5467
-      n_type:          0x1
-      n_sect:          0
-      n_desc:          512
-      n_value:         0
-    - n_strx:          5494
-      n_type:          0x1
-      n_sect:          0
-      n_desc:          512
-      n_value:         0
-    - n_strx:          5513
-      n_type:          0x1
-      n_sect:          0
-      n_desc:          512
-      n_value:         0
-    - n_strx:          5533
-      n_type:          0x1
-      n_sect:          0
-      n_desc:          1024
-      n_value:         0
-  StringTable:
-    - ' '
-    - '-[Basic3 property1]'
-    - '-[Basic3 setProperty1:]'
-    - '-[Basic3 property2]'
-    - '-[Basic3 isProperty3]'
-    - '-[Basic3 setProperty3:]'
-    - '+[Basic5 aClassMethod]'
-    - '-[Basic5 anInstanceMethod]'
-    - '-[Basic6 anInstanceMethodFromAnExtension]'
-    - '-[Basic6 property1]'
-    - '-[Basic6 setProperty1:]'
-    - '-[Basic6(Foo) property2]'
-    - '-[Basic6(Foo) setProperty2:]'
-    - '-[Basic6(Foo) anInstanceMethodFromACategory]'
-    - '-[Basic7 anInstanceMethodFromAnHiddenExtension]'
-    - '-[ExternalManagedObject foo]'
-    - '-[NSManagedObject(Simple) supportsSimple]'
-    - '+[Basic8 useSameName]'
-    - '-[Basic8 useSameName]'
-    - '-[A aMethod]'
-    - '-[Basic9 aProperty]'
-    - '-[FooClass baseMethod]'
-    - '-[FooClass protocolMethod]'
-    - '-[FooClass barMethod]'
-    - '-[FooClass(Private) privateProcotolMethod]'
-    - '__OBJC_METACLASS_RO_$_Simple'
-    - '__OBJC_CLASS_RO_$_Simple'
-    - '__OBJC_METACLASS_RO_$_Base'
-    - '__OBJC_CLASS_RO_$_Base'
-    - '__OBJC_METACLASS_RO_$_SubClass'
-    - '__OBJC_CLASS_RO_$_SubClass'
-    - '__OBJC_METACLASS_RO_$_SimpleInternalAPI'
-    - '__OBJC_CLASS_RO_$_SimpleInternalAPI'
-    - '__OBJC_METACLASS_RO_$_SimpleInternalSPI'
-    - '__OBJC_CLASS_RO_$_SimpleInternalSPI'
-    - '__OBJC_CLASS_RO_$_Basic1'
-    - '__OBJC_METACLASS_RO_$_Basic1'
-    - '__OBJC_METACLASS_RO_$_Basic2'
-    - '__OBJC_CLASS_RO_$_Basic2'
-    - '__OBJC_METACLASS_RO_$_Basic3'
-    - '__OBJC_$_INSTANCE_METHODS_Basic3'
-    - '__OBJC_$_INSTANCE_VARIABLES_Basic3'
-    - '__OBJC_$_PROP_LIST_Basic3'
-    - '__OBJC_CLASS_RO_$_Basic3'
-    - '__OBJC_METACLASS_RO_$_Basic4'
-    - '__OBJC_$_INSTANCE_VARIABLES_Basic4'
-    - '__OBJC_CLASS_RO_$_Basic4'
-    - '__OBJC_METACLASS_RO_$_Basic4_1'
-    - '__OBJC_$_INSTANCE_VARIABLES_Basic4_1'
-    - '__OBJC_CLASS_RO_$_Basic4_1'
-    - '__OBJC_METACLASS_RO_$_Basic4_2'
-    - '__OBJC_$_INSTANCE_VARIABLES_Basic4_2'
-    - '__OBJC_CLASS_RO_$_Basic4_2'
-    - '__OBJC_$_CLASS_METHODS_Basic5'
-    - '__OBJC_METACLASS_RO_$_Basic5'
-    - '__OBJC_$_INSTANCE_METHODS_Basic5'
-    - '__OBJC_CLASS_RO_$_Basic5'
-    - '__OBJC_METACLASS_RO_$_Basic6'
-    - '__OBJC_$_INSTANCE_METHODS_Basic6'
-    - '__OBJC_$_INSTANCE_VARIABLES_Basic6'
-    - '__OBJC_$_PROP_LIST_Basic6'
-    - '__OBJC_CLASS_RO_$_Basic6'
-    - '__OBJC_$_CATEGORY_INSTANCE_METHODS_Basic6_$_Foo'
-    - '__OBJC_$_PROP_LIST_Basic6_$_Foo'
-    - '__OBJC_$_CATEGORY_Basic6_$_Foo'
-    - '__OBJC_METACLASS_RO_$_Basic7'
-    - '__OBJC_$_INSTANCE_METHODS_Basic7'
-    - '__OBJC_CLASS_RO_$_Basic7'
-    - '__OBJC_METACLASS_RO_$_ExternalManagedObject'
-    - '__OBJC_$_INSTANCE_METHODS_ExternalManagedObject'
-    - '__OBJC_CLASS_RO_$_ExternalManagedObject'
-    - '__OBJC_$_CATEGORY_INSTANCE_METHODS_NSManagedObject_$_Simple'
-    - '__OBJC_$_CATEGORY_NSManagedObject_$_Simple'
-    - '__OBJC_METACLASS_RO_$_HiddenClass'
-    - '__OBJC_CLASS_RO_$_HiddenClass'
-    - '__OBJC_$_CLASS_METHODS_Basic8'
-    - '__OBJC_METACLASS_RO_$_Basic8'
-    - '__OBJC_$_INSTANCE_METHODS_Basic8'
-    - '__OBJC_CLASS_RO_$_Basic8'
-    - '__OBJC_METACLASS_RO_$_A'
-    - '__OBJC_$_INSTANCE_METHODS_A'
-    - '__OBJC_CLASS_RO_$_A'
-    - '__OBJC_METACLASS_RO_$_Basic9'
-    - '__OBJC_$_INSTANCE_METHODS_Basic9'
-    - '__OBJC_$_INSTANCE_VARIABLES_Basic9'
-    - '__OBJC_$_PROP_LIST_Basic9'
-    - '__OBJC_CLASS_RO_$_Basic9'
-    - '__OBJC_$_PROP_LIST_Basic9_$_deprecated'
-    - '__OBJC_$_CATEGORY_Basic9_$_deprecated'
-    - '__OBJC_CLASS_RO_$_FooClass'
-    - '__OBJC_METACLASS_RO_$_FooClass'
-    - '__OBJC_$_PROTOCOL_INSTANCE_METHODS_BaseProtocol'
-    - '__OBJC_$_PROTOCOL_METHOD_TYPES_BaseProtocol'
-    - '__OBJC_$_PROTOCOL_REFS_FooProtocol'
-    - '__OBJC_$_PROTOCOL_INSTANCE_METHODS_FooProtocol'
-    - '__OBJC_$_PROTOCOL_METHOD_TYPES_FooProtocol'
-    - '__OBJC_$_PROTOCOL_INSTANCE_METHODS_BarProtocol'
-    - '__OBJC_$_PROTOCOL_METHOD_TYPES_BarProtocol'
-    - '__OBJC_CLASS_PROTOCOLS_$_FooClass'
-    - '__OBJC_$_INSTANCE_METHODS_FooClass'
-    - '__OBJC_$_CATEGORY_INSTANCE_METHODS_FooClass_$_Private'
-    - '__OBJC_$_PROTOCOL_INSTANCE_METHODS_PrivateProtocol'
-    - '__OBJC_$_PROTOCOL_METHOD_TYPES_PrivateProtocol'
-    - '__OBJC_CATEGORY_PROTOCOLS_$_FooClass_$_Private'
-    - '__OBJC_$_CATEGORY_FooClass_$_Private'
-    - _extraGlobalAPI1
-    - _extraGlobalAPI2
-    - _privateGlobalVariable
-    - _publicGlobalVariable
-    - '_OBJC_EHTYPE_$_Base'
-    - '_OBJC_EHTYPE_$_SubClass'
-    - '_OBJC_EHTYPE_$_SimpleInternalAPI'
-    - '_OBJC_EHTYPE_$_SimpleInternalSPI'
-    - '_OBJC_METACLASS_$_Simple'
-    - '_OBJC_CLASS_$_Simple'
-    - '_OBJC_METACLASS_$_Base'
-    - '_OBJC_CLASS_$_Base'
-    - '_OBJC_METACLASS_$_SubClass'
-    - '_OBJC_CLASS_$_SubClass'
-    - '_OBJC_METACLASS_$_SimpleInternalAPI'
-    - '_OBJC_CLASS_$_SimpleInternalAPI'
-    - '_OBJC_METACLASS_$_SimpleInternalSPI'
-    - '_OBJC_CLASS_$_SimpleInternalSPI'
-    - '_OBJC_CLASS_$_Basic1'
-    - '_OBJC_METACLASS_$_Basic1'
-    - '_OBJC_METACLASS_$_Basic2'
-    - '_OBJC_CLASS_$_Basic2'
-    - '_OBJC_METACLASS_$_Basic3'
-    - '_OBJC_CLASS_$_Basic3'
-    - '_OBJC_METACLASS_$_Basic4'
-    - '_OBJC_CLASS_$_Basic4'
-    - '_OBJC_METACLASS_$_Basic4_1'
-    - '_OBJC_CLASS_$_Basic4_1'
-    - '_OBJC_METACLASS_$_Basic4_2'
-    - '_OBJC_CLASS_$_Basic4_2'
-    - '_OBJC_METACLASS_$_Basic5'
-    - '_OBJC_CLASS_$_Basic5'
-    - '_OBJC_METACLASS_$_Basic6'
-    - '_OBJC_CLASS_$_Basic6'
-    - '_OBJC_METACLASS_$_Basic7'
-    - '_OBJC_CLASS_$_Basic7'
-    - '_OBJC_METACLASS_$_ExternalManagedObject'
-    - '_OBJC_CLASS_$_ExternalManagedObject'
-    - '_OBJC_METACLASS_$_HiddenClass'
-    - '_OBJC_CLASS_$_HiddenClass'
-    - '_OBJC_METACLASS_$_Basic8'
-    - '_OBJC_CLASS_$_Basic8'
-    - '_OBJC_METACLASS_$_A'
-    - '_OBJC_CLASS_$_A'
-    - '_OBJC_METACLASS_$_Basic9'
-    - '_OBJC_CLASS_$_Basic9'
-    - '_OBJC_CLASS_$_FooClass'
-    - '_OBJC_METACLASS_$_FooClass'
-    - '_OBJC_IVAR_$_Basic3.property1'
-    - '_OBJC_IVAR_$_Basic3.property2'
-    - '_OBJC_IVAR_$_Basic3.property3'
-    - '_OBJC_IVAR_$_Basic4.ivar1'
-    - '_OBJC_IVAR_$_Basic4.ivar2'
-    - '_OBJC_IVAR_$_Basic4.ivar3'
-    - '_OBJC_IVAR_$_Basic4.ivar4'
-    - '_OBJC_IVAR_$_Basic4_1.ivar1'
-    - '_OBJC_IVAR_$_Basic4_1.ivar2'
-    - '_OBJC_IVAR_$_Basic4_1.ivar3'
-    - '_OBJC_IVAR_$_Basic4_1.ivar4'
-    - '_OBJC_IVAR_$_Basic4_2.ivar4'
-    - '_OBJC_IVAR_$_Basic4_2.ivar3'
-    - '_OBJC_IVAR_$_Basic4_2.ivar2'
-    - '_OBJC_IVAR_$_Basic4_2.ivar1'
-    - '_OBJC_IVAR_$_Basic6.ivar1'
-    - '_OBJC_IVAR_$_Basic6.property1'
-    - '_OBJC_IVAR_$_Basic9._aProperty'
-    - '__OBJC_PROTOCOL_$_BaseProtocol'
-    - '__OBJC_PROTOCOL_$_FooProtocol'
-    - '__OBJC_PROTOCOL_$_BarProtocol'
-    - '__OBJC_PROTOCOL_$_PrivateProtocol'
-    - _weakPublicGlobalVariable
-    - _weakPrivateGlobalVariable
-    - '__OBJC_LABEL_PROTOCOL_$_BaseProtocol'
-    - '__OBJC_LABEL_PROTOCOL_$_FooProtocol'
-    - '__OBJC_LABEL_PROTOCOL_$_BarProtocol'
-    - '__OBJC_LABEL_PROTOCOL_$_PrivateProtocol'
-    - '_OBJC_CLASS_$_NSManagedObject'
-    - '_OBJC_CLASS_$_NSObject'
-    - '_OBJC_METACLASS_$_NSManagedObject'
-    - '_OBJC_METACLASS_$_NSObject'
-    - __objc_empty_cache
-    - _objc_ehtype_vtable
-    - dyld_stub_binder
-    - ''
-    - ''
-  FunctionStarts:  [ 0x1BC0, 0x1BCA, 0x1BD3, 0x1BDD, 0x1BE7, 0x1BF0, 0x1BF6, 
-                     0x1BFC, 0x1C02, 0x1C0C, 0x1C15, 0x1C20, 0x1C26, 0x1C2C, 
-                     0x1C32, 0x1C38, 0x1C40, 0x1C46, 0x1C4C, 0x1C52, 0x1C5C, 
-                     0x1C62, 0x1C68, 0x1C6E ]
-...
diff --git a/clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI.h b/clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI.h
deleted file mode 100644
index 5dd416a0619c..000000000000
--- a/clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI.h
+++ /dev/null
@@ -1,3 +0,0 @@
-#ifndef HAVE_SEEN_PROJECT_HEADER_FIRST
-#error "Project header was not included in the correct order!"
-#endif
diff --git a/clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI2.h b/clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI2.h
deleted file mode 100644
index 9bbae52d7215..000000000000
--- a/clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI2.h
+++ /dev/null
@@ -1,7 +0,0 @@
-#import <Foundation/Foundation.h>
-
-__attribute__((objc_exception))
-@interface SimpleInternalAPI : NSObject
-@end
-
-#define HAVE_SEEN_PROJECT_HEADER_FIRST 1
diff --git a/clang/test/InstallAPI/Inputs/Simple/SimpleInternalSPI.h b/clang/test/InstallAPI/Inputs/Simple/SimpleInternalSPI.h
deleted file mode 100644
index a816c01abeb0..000000000000
--- a/clang/test/InstallAPI/Inputs/Simple/SimpleInternalSPI.h
+++ /dev/null
@@ -1,5 +0,0 @@
-#import <Foundation/Foundation.h>
-
-__attribute__((objc_exception))
-@interface SimpleInternalSPI : NSObject
-@end
diff --git a/clang/test/InstallAPI/extra-exclude-headers.test b/clang/test/InstallAPI/extra-exclude-headers.test
deleted file mode 100644
index 663ca1a5d500..000000000000
--- a/clang/test/InstallAPI/extra-exclude-headers.test
+++ /dev/null
@@ -1,207 +0,0 @@
-; RUN: rm -rf %t 
-; RUN: split-file %s %t
-; RUN: mkdir -p %t/System/Library/Frameworks 
-; RUN: cp -r %S/Inputs/Simple/Simple.framework %t/System/Library/Frameworks/
-; RUN: sed -e "s|DSTROOT|%/t|g" %t/inputs.json.in > %t/inputs.json
-; RUN: yaml2obj %S/Inputs/Simple/Simple.yaml -o %t/Simple 
-
-// Add exclude options.
-; RUN: clang-installapi -target x86_64-apple-macosx10.12 \
-; RUN: -install_name /System/Library/Frameworks/Simple.framework/Versions/A/Simple \
-; RUN: -current_version 1.2.3 -compatibility_version 1 \
-; RUN: -F%t/System/Library/Frameworks \
-; RUN: %t/inputs.json -o %t/Simple.tbd \
-; RUN: --verify-against=%t/Simple --verify-mode=ErrorsAndWarnings \
-; RUN: --exclude-public-header=**/SimpleAPI.h \
-; RUN: --exclude-private-header=**/SimplePrivateSPI.h 2>&1 | FileCheck -check-prefix=WARNINGS %s
-; RUN: llvm-readtapi -compare %t/Simple.tbd %t/expected-excluded.tbd
-
-// Add extra options.
-; RUN: clang-installapi -target x86_64-apple-macosx10.12 \
-; RUN: -install_name /System/Library/Frameworks/Simple.framework/Versions/A/Simple \
-; RUN: -current_version 1.2.3 -compatibility_version 1 \
-; RUN: -F%t/System/Library/Frameworks \
-; RUN: %t/inputs.json -o %t/Simple.tbd \
-; RUN: --verify-against=%t/Simple --verify-mode=Pedantic \
-; RUN: --extra-project-header=%S/Inputs/Simple/SimpleInternalAPI2.h \
-; RUN: --extra-project-header=%S/Inputs/Simple/SimpleInternalAPI.h \
-; RUN: --extra-public-header=%S/Inputs/Simple/Extra \
-; RUN: --extra-private-header=%S/Inputs/Simple/SimpleInternalSPI.h \
-; RUN: --exclude-public-header=**/SimpleAPI.h \
-; RUN: --exclude-private-header=**/SimplePrivateSPI.h 2>&1 | FileCheck -check-prefix=PEDANTIC -allow-empty %s
-; RUN: llvm-readtapi -compare %t/Simple.tbd %t/expected-extra.tbd
-
-// Check fatal missing file input.
-; RUN: not clang-installapi -target x86_64-apple-macosx10.12 \
-; RUN: -install_name /System/Library/Frameworks/Simple.framework/Versions/A/Simple \
-; RUN: -current_version 1.2.3 -compatibility_version 1 \
-; RUN: -F%t/System/Library/Frameworks \
-; RUN: %t/inputs.json -o %t/Simple.tbd \
-; RUN: --extra-public-header=%S/Inputs/Simple/NoSuchFile.h 2>&1 | FileCheck -allow-empty -check-prefix=NOPUBLIC %s
-
-; WARNINGS: warning: no declaration was found for exported symbol '_extraGlobalAPI1' in dynamic library
-; WARNINGS: warning: no declaration was found for exported symbol '_extraGlobalAPI2' in dynamic library
-; WARNINGS: warning: no declaration was found for exported symbol '(ObjC Class) SimpleInternalSPI' in dynamic library
-; WARNINGS: warning: no declaration was found for exported symbol '(ObjC Class) SimpleInternalAPI' in dynamic library
-
-; PEDANTIC-NOT: error
-; PEDANTIC: warning: cannot find protocol definition for 'ForwardProcotol'
-
-; NOPUBLIC: error: no such public header file:
-
-;--- expected-excluded.tbd
-{
-  "main_library": {
-    "current_versions": [
-      {
-        "version": "1.2.3"
-      }
-    ],
-    "exported_symbols": [
-      {
-        "data": {
-          "global": [
-            "_publicGlobalVariable",
-            "_privateGlobalVariable"
-          ],
-          "objc_class": [
-            "ExternalManagedObject", "Basic6",
-            "Basic1", "Base", "Basic3",
-            "FooClass", "Simple",
-            "Basic4_2", "Basic5",
-            "Basic9","Basic8",
-            "Basic2", "Basic4", "A", "SubClass"
-          ],
-          "objc_eh_type": [
-            "SubClass", "Base"
-          ],
-          "objc_ivar": [
-            "Basic4.ivar2", "Basic4_2.ivar1", "Basic6.ivar1",
-            "Basic4.ivar1", "Basic4_2.ivar2"
-          ],
-          "weak": [
-            "_weakPrivateGlobalVariable", "_weakPublicGlobalVariable"
-          ]
-        }
-      }
-    ],
-    "flags": [
-      {
-        "attributes": ["not_app_extension_safe"]
-      }
-    ],
-    "install_names": [
-      {
-        "name": "/System/Library/Frameworks/Simple.framework/Versions/A/Simple"
-      }
-    ],
-    "target_info": [
-      {"min_deployment": "10.12", "target": "x86_64-macos"}
-    ]
-  },
-  "tapi_tbd_version": 5
-}
-
-;--- expected-extra.tbd
-{
-  "main_library": {
-    "current_versions": [
-      { "version": "1.2.3" }
-    ],
-    "exported_symbols": [
-      {
-        "data": {
-          "global": [
-            "_publicGlobalVariable", "_extraGlobalAPI2",
-            "_extraGlobalAPI1", "_privateGlobalVariable"
-          ],
-          "objc_class": [
-            "SubClass", "SimpleInternalSPI",
-            "Basic6", "Basic1", "Base",
-            "Basic3", "Simple", "Basic4_2",
-            "Basic5", "FooClass", "Basic9",
-            "Basic8", "Basic2", "Basic4",
-            "A", "SimpleInternalAPI",
-            "ExternalManagedObject"
-          ],
-          "objc_eh_type": [
-            "SubClass", "SimpleInternalAPI",
-            "Base", "SimpleInternalSPI"
-          ],
-          "objc_ivar": [
-            "Basic4.ivar2", "Basic4_2.ivar1",
-            "Basic6.ivar1", "Basic4.ivar1",
-            "Basic4_2.ivar2"
-          ],
-          "weak": [
-            "_weakPrivateGlobalVariable", "_weakPublicGlobalVariable"
-          ]
-        }
-      }
-    ],
-    "flags": [
-      {
-        "attributes": [ "not_app_extension_safe"]
-      }
-    ],
-    "install_names": [
-      { "name": "/System/Library/Frameworks/Simple.framework/Versions/A/Simple" }
-    ],
-    "target_info": [
-      { "min_deployment": "10.12", "target": "x86_64-macos" }
-    ]
-  },
-  "tapi_tbd_version": 5
-}
-
-;--- inputs.json.in
-{
-  "headers": [ 
-  {
-    "path" : "DSTROOT/System/Library/Frameworks/Simple.framework/Headers/Basic.h",
-    "type" : "public"
-  }, 
-  {
-    "path" : "DSTROOT/System/Library/Frameworks/Simple.framework/Headers/External.h",
-    "type" : "public"
-  }, 
-  {
-    "path" : "DSTROOT/System/Library/Frameworks/Simple.framework/Headers/Simple.h",
-    "type" : "public"
-  }, 
-  {
-    "path" : "DSTROOT/System/Library/Frameworks/Simple.framework/Headers/SimpleAPI.h",
-    "type" : "public"
-  }, 
-  {
-    "path" : "DSTROOT/System/Library/Frameworks/Simple.framework/PrivateHeaders/SimplePrivate.h",
-    "type" : "private"
-  }, 
-  {
-    "path" : "DSTROOT/System/Library/Frameworks/Simple.framework/PrivateHeaders/SimplePrivateSPI.h",
-    "type" : "private"
-  }
-  ],
-  "version": "3"
-}
-
-;--- System/Library/Frameworks/Foundation.framework/Headers/Foundation.h
-@interface NSObject 
-@end
-
-typedef unsigned char BOOL; 
-#ifndef NS_AVAILABLE
-#define NS_AVAILABLE(x,y) __attribute__((availability(macosx,introduced=x)))
-#endif 
-#ifndef NS_UNAVAILABLE
-#define NS_UNAVAILABLE  __attribute__((unavailable))
-#endif 
-#ifndef NS_DEPRECATED_MAC
-#define NS_DEPRECATED_MAC(x,y) __attribute__((availability(macosx,introduced=x,deprecated=y,message="" )));
-#endif 
-
-@interface NSManagedObject
-@end 
-
-@interface NSSet 
-@end 
diff --git a/clang/tools/clang-installapi/InstallAPIOpts.td b/clang/tools/clang-installapi/InstallAPIOpts.td
index ab9e1fe7f2f9..87f4c3327e84 100644
--- a/clang/tools/clang-installapi/InstallAPIOpts.td
+++ b/clang/tools/clang-installapi/InstallAPIOpts.td
@@ -29,35 +29,3 @@ def verify_mode_EQ : Joined<["--"], "verify-mode=">,
   HelpText<"Specify the severity and extend of the validation. Valid modes are ErrorsOnly, ErrorsAndWarnings, and Pedantic.">;
 def demangle : Flag<["--", "-"], "demangle">,
   HelpText<"Demangle symbols when printing warnings and errors">;
-
-// Additional input options.
-def extra_project_header : Separate<["-"], "extra-project-header">,
-  MetaVarName<"<path>">,
-  HelpText<"Add additional project header location for parsing">;
-def extra_project_header_EQ : Joined<["--"], "extra-project-header=">,
-  Alias<extra_project_header>;
-def exclude_project_header : Separate<["-"], "exclude-project-header">,
-  MetaVarName<"<glob>">,
-  HelpText<"Exclude project header from parsing">;
-def exclude_project_header_EQ : Joined<["--"], "exclude-project-header=">,
-  Alias<exclude_project_header>;
-def extra_public_header : Separate<["-"], "extra-public-header">,
-  MetaVarName<"<path>">,
-  HelpText<"Add additional public header location for parsing">;
-def extra_public_header_EQ : Joined<["--"], "extra-public-header=">,
-  Alias<extra_public_header>;
-def extra_private_header : Separate<["-"], "extra-private-header">,
-  MetaVarName<"<path>">,
-  HelpText<"Add additional private header location for parsing">;
-def extra_private_header_EQ : Joined<["--"], "extra-private-header=">,
-  Alias<extra_private_header>;
-def exclude_public_header : Separate<["-"], "exclude-public-header">,
-  MetaVarName<"<glob>">,
-  HelpText<"Exclude public header from parsing">;
-def exclude_public_header_EQ : Joined<["--"], "exclude-public-header=">,
-  Alias<exclude_public_header>;
-def exclude_private_header : Separate<["-"], "exclude-private-header">,
-  MetaVarName<"<glob>">,
-  HelpText<"Exclude private header from parsing">;
-def exclude_private_header_EQ : Joined<["--"], "exclude-private-header=">,
-  Alias<exclude_private_header>;
diff --git a/clang/tools/clang-installapi/Options.cpp b/clang/tools/clang-installapi/Options.cpp
index 4f79c62724a6..b8696bb7896d 100644
--- a/clang/tools/clang-installapi/Options.cpp
+++ b/clang/tools/clang-installapi/Options.cpp
@@ -10,7 +10,6 @@
 #include "clang/Driver/Driver.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/InstallAPI/FileList.h"
-#include "clang/InstallAPI/HeaderFile.h"
 #include "clang/InstallAPI/InstallAPIDiagnostic.h"
 #include "llvm/Support/Program.h"
 #include "llvm/TargetParser/Host.h"
@@ -182,26 +181,6 @@ bool Options::processFrontendOptions(InputArgList &Args) {
   return true;
 }
 
-bool Options::addFilePaths(InputArgList &Args, PathSeq &Headers,
-                           OptSpecifier ID) {
-  for (const StringRef Path : Args.getAllArgValues(ID)) {
-    if ((bool)FM->getDirectory(Path, /*CacheFailure=*/false)) {
-      auto InputHeadersOrErr = enumerateFiles(*FM, Path);
-      if (!InputHeadersOrErr) {
-        Diags->Report(diag::err_cannot_open_file)
-            << Path << toString(InputHeadersOrErr.takeError());
-        return false;
-      }
-      // Sort headers to ensure deterministic behavior.
-      sort(*InputHeadersOrErr);
-      for (std::string &H : *InputHeadersOrErr)
-        Headers.emplace_back(std::move(H));
-    } else
-      Headers.emplace_back(Path);
-  }
-  return true;
-}
-
 std::vector<const char *>
 Options::processAndFilterOutInstallAPIOptions(ArrayRef<const char *> Args) {
   std::unique_ptr<llvm::opt::OptTable> Table;
@@ -241,35 +220,6 @@ Options::processAndFilterOutInstallAPIOptions(ArrayRef<const char *> Args) {
   if (const Arg *A = ParsedArgs.getLastArg(OPT_verify_against))
     DriverOpts.DylibToVerify = A->getValue();
 
-  // Handle exclude & extra header directories or files.
-  auto handleAdditionalInputArgs = [&](PathSeq &Headers,
-                                       clang::installapi::ID OptID) {
-    if (ParsedArgs.hasArgNoClaim(OptID))
-      Headers.clear();
-    return addFilePaths(ParsedArgs, Headers, OptID);
-  };
-
-  if (!handleAdditionalInputArgs(DriverOpts.ExtraPublicHeaders,
-                                 OPT_extra_public_header))
-    return {};
-
-  if (!handleAdditionalInputArgs(DriverOpts.ExtraPrivateHeaders,
-                                 OPT_extra_private_header))
-    return {};
-  if (!handleAdditionalInputArgs(DriverOpts.ExtraProjectHeaders,
-                                 OPT_extra_project_header))
-    return {};
-
-  if (!handleAdditionalInputArgs(DriverOpts.ExcludePublicHeaders,
-                                 OPT_exclude_public_header))
-    return {};
-  if (!handleAdditionalInputArgs(DriverOpts.ExcludePrivateHeaders,
-                                 OPT_exclude_private_header))
-    return {};
-  if (!handleAdditionalInputArgs(DriverOpts.ExcludeProjectHeaders,
-                                 OPT_exclude_project_header))
-    return {};
-
   /// Any unclaimed arguments should be forwarded to the clang driver.
   std::vector<const char *> ClangDriverArgs(ParsedArgs.size());
   for (const Arg *A : ParsedArgs) {
@@ -352,77 +302,6 @@ InstallAPIContext Options::createContext() {
       return Ctx;
     }
   }
-  // After initial input has been processed, add any extra headers.
-  auto HandleExtraHeaders = [&](PathSeq &Headers, HeaderType Type) -> bool {
-    assert(Type != HeaderType::Unknown && "Missing header type.");
-    for (const StringRef Path : Headers) {
-      if (!FM->getOptionalFileRef(Path)) {
-        Diags->Report(diag::err_no_such_header_file)
-            << Path << (unsigned)Type - 1;
-        return false;
-      }
-      SmallString<PATH_MAX> FullPath(Path);
-      FM->makeAbsolutePath(FullPath);
-
-      auto IncludeName = createIncludeHeaderName(FullPath);
-      Ctx.InputHeaders.emplace_back(
-          FullPath, Type, IncludeName.has_value() ? *IncludeName : "");
-      Ctx.InputHeaders.back().setExtra();
-    }
-    return true;
-  };
-
-  if (!HandleExtraHeaders(DriverOpts.ExtraPublicHeaders, HeaderType::Public) ||
-      !HandleExtraHeaders(DriverOpts.ExtraPrivateHeaders,
-                          HeaderType::Private) ||
-      !HandleExtraHeaders(DriverOpts.ExtraProjectHeaders, HeaderType::Project))
-    return Ctx;
-
-  // After all headers have been added, consider excluded headers.
-  std::vector<std::unique_ptr<HeaderGlob>> ExcludedHeaderGlobs;
-  std::set<FileEntryRef> ExcludedHeaderFiles;
-  auto ParseGlobs = [&](const PathSeq &Paths, HeaderType Type) {
-    for (const StringRef Path : Paths) {
-      auto Glob = HeaderGlob::create(Path, Type);
-      if (Glob)
-        ExcludedHeaderGlobs.emplace_back(std::move(Glob.get()));
-      else {
-        consumeError(Glob.takeError());
-        if (auto File = FM->getFileRef(Path))
-          ExcludedHeaderFiles.emplace(*File);
-        else {
-          Diags->Report(diag::err_no_such_header_file)
-              << Path << (unsigned)Type;
-          return false;
-        }
-      }
-    }
-    return true;
-  };
-
-  if (!ParseGlobs(DriverOpts.ExcludePublicHeaders, HeaderType::Public) ||
-      !ParseGlobs(DriverOpts.ExcludePrivateHeaders, HeaderType::Private) ||
-      !ParseGlobs(DriverOpts.ExcludeProjectHeaders, HeaderType::Project))
-    return Ctx;
-
-  for (HeaderFile &Header : Ctx.InputHeaders) {
-    for (auto &Glob : ExcludedHeaderGlobs)
-      if (Glob->match(Header))
-        Header.setExcluded();
-  }
-  if (!ExcludedHeaderFiles.empty()) {
-    for (HeaderFile &Header : Ctx.InputHeaders) {
-      auto FileRef = FM->getFileRef(Header.getPath());
-      if (!FileRef)
-        continue;
-      if (ExcludedHeaderFiles.count(*FileRef))
-        Header.setExcluded();
-    }
-  }
-  // Report if glob was ignored.
-  for (const auto &Glob : ExcludedHeaderGlobs)
-    if (!Glob->didMatch())
-      Diags->Report(diag::warn_glob_did_not_match) << Glob->str();
 
   // Parse binary dylib and initialize verifier.
   if (DriverOpts.DylibToVerify.empty()) {
diff --git a/clang/tools/clang-installapi/Options.h b/clang/tools/clang-installapi/Options.h
index c18309f69370..2beeafc86bb0 100644
--- a/clang/tools/clang-installapi/Options.h
+++ b/clang/tools/clang-installapi/Options.h
@@ -31,24 +31,6 @@ struct DriverOptions {
   /// \brief Path to input file lists (JSON).
   llvm::MachO::PathSeq FileLists;
 
-  /// \brief Paths of extra public headers.
-  PathSeq ExtraPublicHeaders;
-
-  /// \brief Paths of extra private headers.
-  PathSeq ExtraPrivateHeaders;
-
-  /// \brief Paths of extra project headers.
-  PathSeq ExtraProjectHeaders;
-
-  /// \brief List of excluded public headers.
-  PathSeq ExcludePublicHeaders;
-
-  /// \brief List of excluded private headers.
-  PathSeq ExcludePrivateHeaders;
-
-  /// \brief List of excluded project headers.
-  PathSeq ExcludeProjectHeaders;
-
   /// \brief Mappings of target triples & tapi targets to build for.
   std::map<llvm::MachO::Target, llvm::Triple> Targets;
 
@@ -121,9 +103,6 @@ public:
   std::vector<std::string> &getClangFrontendArgs() { return FrontendArgs; }
 
 private:
-  bool addFilePaths(llvm::opt::InputArgList &Args, PathSeq &Headers,
-                    llvm::opt::OptSpecifier ID);
-
   DiagnosticsEngine *Diags;
   FileManager *FM;
   std::vector<std::string> FrontendArgs;
diff --git a/llvm/include/llvm/TextAPI/Utils.h b/llvm/include/llvm/TextAPI/Utils.h
index 319e0abce25b..31d3c45f9e29 100644
--- a/llvm/include/llvm/TextAPI/Utils.h
+++ b/llvm/include/llvm/TextAPI/Utils.h
@@ -14,10 +14,8 @@
 #define LLVM_TEXTAPI_UTILS_H
 
 #include "llvm/ADT/Twine.h"
-#include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
-#include "llvm/Support/Regex.h"
 
 #if !defined(PATH_MAX)
 #define PATH_MAX 1024
@@ -70,10 +68,5 @@ std::error_code make_relative(StringRef From, StringRef To,
 /// \param IsSymLink Whether path points to a symlink.
 bool isPrivateLibrary(StringRef Path, bool IsSymLink = false);
 
-/// Create a regex rule from provided glob string.
-/// \param Glob String that represents glob input.
-/// \return The equivalent regex rule.
-llvm::Expected<llvm::Regex> createRegexFromGlob(llvm::StringRef Glob);
-
 } // namespace llvm::MachO
 #endif // LLVM_TEXTAPI_UTILS_H
diff --git a/llvm/lib/TextAPI/Utils.cpp b/llvm/lib/TextAPI/Utils.cpp
index 56b63ee01b39..b5c3999a86ea 100644
--- a/llvm/lib/TextAPI/Utils.cpp
+++ b/llvm/lib/TextAPI/Utils.cpp
@@ -152,49 +152,3 @@ bool llvm::MachO::isPrivateLibrary(StringRef Path, bool IsSymLink) {
   }
   return false;
 }
-
-static StringLiteral RegexMetachars = "()^$|+.[]\\{}";
-
-llvm::Expected<Regex> llvm::MachO::createRegexFromGlob(StringRef Glob) {
-  SmallString<128> RegexString("^");
-  unsigned NumWildcards = 0;
-  for (unsigned i = 0; i < Glob.size(); ++i) {
-    char C = Glob[i];
-    switch (C) {
-    case '?':
-      RegexString += '.';
-      break;
-    case '*': {
-      const char *PrevChar = i > 0 ? Glob.data() + i - 1 : nullptr;
-      NumWildcards = 1;
-      ++i;
-      while (i < Glob.size() && Glob[i] == '*') {
-        ++NumWildcards;
-        ++i;
-      }
-      const char *NextChar = i < Glob.size() ? Glob.data() + i : nullptr;
-
-      if ((NumWildcards > 1) && (PrevChar == nullptr || *PrevChar == '/') &&
-          (NextChar == nullptr || *NextChar == '/')) {
-        RegexString += "(([^/]*(/|$))*)";
-      } else
-        RegexString += "([^/]*)";
-      break;
-    }
-    default:
-      if (RegexMetachars.find(C) != StringRef::npos)
-        RegexString.push_back('\\');
-      RegexString.push_back(C);
-    }
-  }
-  RegexString.push_back('$');
-  if (NumWildcards == 0)
-    return make_error<StringError>("not a glob", inconvertibleErrorCode());
-
-  llvm::Regex Rule = Regex(RegexString);
-  std::string Error;
-  if (!Rule.isValid(Error))
-    return make_error<StringError>(Error, inconvertibleErrorCode());
-
-  return Rule;
-}
-- 
GitLab


From cce18e40c0f1ca382e7b83ad2860430736dbf8c8 Mon Sep 17 00:00:00 2001
From: Christian Sigg <chsigg@users.noreply.github.com>
Date: Mon, 25 Mar 2024 20:57:22 +0100
Subject: [PATCH 185/404] [bazel] Fix BUILD after
 2eb010ddce2a8e7813a8e289004102011b1138db.

---
 utils/bazel/third_party_build/pfm.BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/bazel/third_party_build/pfm.BUILD b/utils/bazel/third_party_build/pfm.BUILD
index 07a1dc48b857..dba7464d17df 100644
--- a/utils/bazel/third_party_build/pfm.BUILD
+++ b/utils/bazel/third_party_build/pfm.BUILD
@@ -35,5 +35,5 @@ cc_library(
         "@platforms//os:linux": [],
         "//conditions:default": ["@platforms//:incompatible"],
     }),
-    visibility = ["//visibility:private"],
+    visibility = ["//visibility:public"],
 )
-- 
GitLab


From b7611370491873722e08e4ce9374312d0c936af1 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <jeffrey.byrnes@amd.com>
Date: Mon, 25 Mar 2024 13:11:58 -0700
Subject: [PATCH 186/404] [AMDGPU] Use correct VGPR threshold for flagging
 ExcessRP regions in unified register file case (#85860)

`ST.getMaxNumVGPRs(MF)` lowers to `AMDGPUBaseInfo.cpp:getTotalNumVGPRs`
which returns 512 for gfx90a. This is subsequently limited by
`AMDGPUBaseInfo:getAddressableNumVGPRs()`, which also returns 512 for
gfx90a. The ISA states we can have a total of 512 registers, but a
maximum of only 256 of each of AGPR and VGPR (gfx90a 3.6.4).

Therefore, in unified register file case, `ST.getMaxNumVGPRs(MF)`
calculates the maximum number of combined VGPR + AGPR. But, it is
currently used as the limit for accvgpr and as the limit for archvgpr.

This patch uses it as the combined limit, and accounts for the maximum addressable arch/acc VGPRs when calculating the per RegClass limits.

It is not unreasonable to think other clients of getTotalNumVGPRs are
using it in the wrong way.
---
 llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp           | 11 ++++++++---
 .../CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.single.2b.mir |  2 ++
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 9f419a7fbf68..94d93390d091 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -973,12 +973,17 @@ void GCNSchedStage::checkScheduling() {
     LLVM_DEBUG(dbgs() << "Occupancy lowered for the function to "
                       << DAG.MinOccupancy << ".\n");
   }
-
+  // The maximum number of arch VGPR on non-unified register file, or the
+  // maximum VGPR + AGPR in the unified register file case.
   unsigned MaxVGPRs = ST.getMaxNumVGPRs(MF);
+  // The maximum number of arch VGPR for both unified and non-unified register
+  // file.
+  unsigned MaxArchVGPRs = std::min(MaxVGPRs, ST.getAddressableNumArchVGPRs());
   unsigned MaxSGPRs = ST.getMaxNumSGPRs(MF);
 
-  if (PressureAfter.getVGPRNum(false) > MaxVGPRs ||
-      PressureAfter.getAGPRNum() > MaxVGPRs ||
+  if (PressureAfter.getVGPRNum(ST.hasGFX90AInsts()) > MaxVGPRs ||
+      PressureAfter.getVGPRNum(false) > MaxArchVGPRs ||
+      PressureAfter.getAGPRNum() > MaxArchVGPRs ||
       PressureAfter.getSGPRNum() > MaxSGPRs) {
     DAG.RescheduleRegions[RegionIdx] = true;
     DAG.RegionsWithHighRP[RegionIdx] = true;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.single.2b.mir b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.single.2b.mir
index 091b29c23d60..e93595b9ef27 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.single.2b.mir
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.single.2b.mir
@@ -4,6 +4,8 @@
 --- |
   define amdgpu_kernel void @single-wave-phase-2b(ptr addrspace(3) noalias %in0, ptr addrspace(3) noalias %in1, ptr addrspace(3) noalias %in2, ptr addrspace(3) noalias %in3, ptr addrspace(3) noalias %in4, ptr addrspace(3) noalias %in5, ptr addrspace(3) noalias %in6, ptr addrspace(3) noalias %in7, ptr addrspace(3) noalias %in8, ptr addrspace(3) noalias %in9, ptr addrspace(3) noalias %in10, ptr addrspace(3) noalias %in11, ptr addrspace(7) noalias %in12, ptr addrspace(7) noalias %in13, ptr addrspace(7) noalias %in14, ptr addrspace(7) noalias %in15, ptr addrspace(7) noalias %in16, ptr addrspace(7) noalias %in17, ptr addrspace(7) noalias %in18, ptr addrspace(7) noalias %in19, ptr addrspace(7) noalias %in20, ptr addrspace(7) noalias %in21, ptr addrspace(7) noalias %in22, ptr addrspace(7) noalias %in23, ptr addrspace(7) noalias %in24, ptr addrspace(7) noalias %in25, ptr addrspace(7) noalias %in26, ptr addrspace(7) noalias %in27, ptr addrspace(7) noalias %in28, ptr addrspace(7) noalias %in29) #0 { ret void }
 
+  attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="1,256" }
+
   !0 = distinct !{!0}
   !1 = !{!1, !0}
 ...
-- 
GitLab


From 2cad43c1ba7d9c83ae8fb809e60a57d347e09370 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Mon, 25 Mar 2024 15:24:39 -0500
Subject: [PATCH 187/404] [Libomptarget] Factor functions out of 'Plugin'
 interface (#86528)

Summary:
This patch factors common functions out of the `Plugin` interface prior
to its removal in a future patch. This simply temporarily renames it to
`PluginTy` so that we could re-use `Plugin::check` internally as this
needs to be defined statically per plugin now. We can refactor this
later.

The future patch will delete `PluginTy` and `PluginTy::get` entirely.
This simply tries to minimize a few changes to make it easier to land.
---
 openmp/libomptarget/CMakeLists.txt            |  3 +-
 .../plugins-nextgen/amdgpu/src/rtl.cpp        | 39 +++++----
 .../common/include/PluginInterface.h          | 70 ++++++++-------
 .../common/src/PluginInterface.cpp            | 86 ++++++++++---------
 .../plugins-nextgen/cuda/src/rtl.cpp          | 27 +++---
 .../plugins-nextgen/host/src/rtl.cpp          | 28 +++---
 6 files changed, 132 insertions(+), 121 deletions(-)

diff --git a/openmp/libomptarget/CMakeLists.txt b/openmp/libomptarget/CMakeLists.txt
index b382137b70ee..531198fae016 100644
--- a/openmp/libomptarget/CMakeLists.txt
+++ b/openmp/libomptarget/CMakeLists.txt
@@ -145,8 +145,7 @@ add_subdirectory(DeviceRTL)
 add_subdirectory(tools)
 
 # Build target agnostic offloading library.
-set(LIBOMPTARGET_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src)
-add_subdirectory(${LIBOMPTARGET_SRC_DIR})
+add_subdirectory(src)
 
 # Add tests.
 add_subdirectory(test)
diff --git a/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp b/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp
index fce7454bf280..14461f14dd67 100644
--- a/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -2088,7 +2088,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   /// Allocate and construct an AMDGPU kernel.
   Expected<GenericKernelTy &> constructKernel(const char *Name) override {
     // Allocate and construct the AMDGPU kernel.
-    AMDGPUKernelTy *AMDGPUKernel = Plugin::get().allocate<AMDGPUKernelTy>();
+    AMDGPUKernelTy *AMDGPUKernel = PluginTy::get().allocate<AMDGPUKernelTy>();
     if (!AMDGPUKernel)
       return Plugin::error("Failed to allocate memory for AMDGPU kernel");
 
@@ -2139,7 +2139,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
                                            int32_t ImageId) override {
     // Allocate and initialize the image object.
     AMDGPUDeviceImageTy *AMDImage =
-        Plugin::get().allocate<AMDGPUDeviceImageTy>();
+        PluginTy::get().allocate<AMDGPUDeviceImageTy>();
     new (AMDImage) AMDGPUDeviceImageTy(ImageId, *this, TgtImage);
 
     // Load the HSA executable.
@@ -2697,7 +2697,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   }
   Error setDeviceHeapSize(uint64_t Value) override {
     for (DeviceImageTy *Image : LoadedImages)
-      if (auto Err = setupDeviceMemoryPool(Plugin::get(), *Image, Value))
+      if (auto Err = setupDeviceMemoryPool(PluginTy::get(), *Image, Value))
         return Err;
     DeviceMemoryPoolSize = Value;
     return Plugin::success();
@@ -2737,7 +2737,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     return utils::iterateAgentMemoryPools(
         Agent, [&](hsa_amd_memory_pool_t HSAMemoryPool) {
           AMDGPUMemoryPoolTy *MemoryPool =
-              Plugin::get().allocate<AMDGPUMemoryPoolTy>();
+              PluginTy::get().allocate<AMDGPUMemoryPoolTy>();
           new (MemoryPool) AMDGPUMemoryPoolTy(HSAMemoryPool);
           AllMemoryPools.push_back(MemoryPool);
           return HSA_STATUS_SUCCESS;
@@ -3115,6 +3115,17 @@ struct AMDGPUPluginTy final : public GenericPluginTy {
     return Plugin::check(Status, "Error in hsa_shut_down: %s");
   }
 
+  /// Creates an AMDGPU device.
+  GenericDeviceTy *createDevice(int32_t DeviceId, int32_t NumDevices) override {
+    return new AMDGPUDeviceTy(DeviceId, NumDevices, getHostDevice(),
+                              getKernelAgent(DeviceId));
+  }
+
+  /// Creates an AMDGPU global handler.
+  GenericGlobalHandlerTy *createGlobalHandler() override {
+    return new AMDGPUGlobalHandlerTy();
+  }
+
   Triple::ArchType getTripleArch() const override { return Triple::amdgcn; }
 
   /// Get the ELF code for recognizing the compatible image binary.
@@ -3237,7 +3248,7 @@ Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
   // 56 bytes per allocation.
   uint32_t AllArgsSize = KernelArgsSize + ImplicitArgsSize;
 
-  AMDHostDeviceTy &HostDevice = Plugin::get<AMDGPUPluginTy>().getHostDevice();
+  AMDHostDeviceTy &HostDevice = PluginTy::get<AMDGPUPluginTy>().getHostDevice();
   AMDGPUMemoryManagerTy &ArgsMemoryManager = HostDevice.getArgsMemoryManager();
 
   void *AllArgs = nullptr;
@@ -3347,20 +3358,10 @@ Error AMDGPUKernelTy::printLaunchInfoDetails(GenericDeviceTy &GenericDevice,
   return Plugin::success();
 }
 
-GenericPluginTy *Plugin::createPlugin() { return new AMDGPUPluginTy(); }
-
-GenericDeviceTy *Plugin::createDevice(int32_t DeviceId, int32_t NumDevices) {
-  AMDGPUPluginTy &Plugin = get<AMDGPUPluginTy &>();
-  return new AMDGPUDeviceTy(DeviceId, NumDevices, Plugin.getHostDevice(),
-                            Plugin.getKernelAgent(DeviceId));
-}
-
-GenericGlobalHandlerTy *Plugin::createGlobalHandler() {
-  return new AMDGPUGlobalHandlerTy();
-}
+GenericPluginTy *PluginTy::createPlugin() { return new AMDGPUPluginTy(); }
 
 template <typename... ArgsTy>
-Error Plugin::check(int32_t Code, const char *ErrFmt, ArgsTy... Args) {
+static Error Plugin::check(int32_t Code, const char *ErrFmt, ArgsTy... Args) {
   hsa_status_t ResultCode = static_cast<hsa_status_t>(Code);
   if (ResultCode == HSA_STATUS_SUCCESS || ResultCode == HSA_STATUS_INFO_BREAK)
     return Error::success();
@@ -3384,7 +3385,7 @@ void *AMDGPUMemoryManagerTy::allocate(size_t Size, void *HstPtr,
   }
   assert(Ptr && "Invalid pointer");
 
-  auto &KernelAgents = Plugin::get<AMDGPUPluginTy>().getKernelAgents();
+  auto &KernelAgents = PluginTy::get<AMDGPUPluginTy>().getKernelAgents();
 
   // Allow all kernel agents to access the allocation.
   if (auto Err = MemoryPool->enableAccess(Ptr, Size, KernelAgents)) {
@@ -3427,7 +3428,7 @@ void *AMDGPUDeviceTy::allocate(size_t Size, void *, TargetAllocTy Kind) {
   }
 
   if (Alloc) {
-    auto &KernelAgents = Plugin::get<AMDGPUPluginTy>().getKernelAgents();
+    auto &KernelAgents = PluginTy::get<AMDGPUPluginTy>().getKernelAgents();
     // Inherently necessary for host or shared allocations
     // Also enabled for device memory to allow device to device memcpy
 
diff --git a/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h b/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h
index b7be7b645ba3..1440f8f678d8 100644
--- a/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h
+++ b/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h
@@ -976,6 +976,13 @@ struct GenericPluginTy {
   Error deinit();
   virtual Error deinitImpl() = 0;
 
+  /// Create a new device for the underlying plugin.
+  virtual GenericDeviceTy *createDevice(int32_t DeviceID,
+                                        int32_t NumDevices) = 0;
+
+  /// Create a new global handler for the underlying plugin.
+  virtual GenericGlobalHandlerTy *createGlobalHandler() = 0;
+
   /// Get the reference to the device with a certain device id.
   GenericDeviceTy &getDevice(int32_t DeviceId) {
     assert(isValidDeviceId(DeviceId) && "Invalid device id");
@@ -1085,29 +1092,53 @@ private:
   RPCServerTy *RPCServer;
 };
 
+namespace Plugin {
+/// Create a success error. This is the same as calling Error::success(), but
+/// it is recommended to use this one for consistency with Plugin::error() and
+/// Plugin::check().
+static Error success() { return Error::success(); }
+
+/// Create a string error.
+template <typename... ArgsTy>
+static Error error(const char *ErrFmt, ArgsTy... Args) {
+  return createStringError(inconvertibleErrorCode(), ErrFmt, Args...);
+}
+
+/// Check the plugin-specific error code and return an error or success
+/// accordingly. In case of an error, create a string error with the error
+/// description. The ErrFmt should follow the format:
+///     "Error in <function name>[<optional info>]: %s"
+/// The last format specifier "%s" is mandatory and will be used to place the
+/// error code's description. Notice this function should be only called from
+/// the plugin-specific code.
+/// TODO: Refactor this, must be defined individually by each plugin.
+template <typename... ArgsTy>
+static Error check(int32_t ErrorCode, const char *ErrFmt, ArgsTy... Args);
+} // namespace Plugin
+
 /// Class for simplifying the getter operation of the plugin. Anywhere on the
 /// code, the current plugin can be retrieved by Plugin::get(). The class also
 /// declares functions to create plugin-specific object instances. The check(),
 /// createPlugin(), createDevice() and createGlobalHandler() functions should be
 /// defined by each plugin implementation.
-class Plugin {
+class PluginTy {
   // Reference to the plugin instance.
   static GenericPluginTy *SpecificPlugin;
 
-  Plugin() {
+  PluginTy() {
     if (auto Err = init())
       REPORT("Failed to initialize plugin: %s\n",
              toString(std::move(Err)).data());
   }
 
-  ~Plugin() {
+  ~PluginTy() {
     if (auto Err = deinit())
       REPORT("Failed to deinitialize plugin: %s\n",
              toString(std::move(Err)).data());
   }
 
-  Plugin(const Plugin &) = delete;
-  void operator=(const Plugin &) = delete;
+  PluginTy(const PluginTy &) = delete;
+  void operator=(const PluginTy &) = delete;
 
   /// Create and intialize the plugin instance.
   static Error init() {
@@ -1158,7 +1189,7 @@ public:
     // This static variable will initialize the underlying plugin instance in
     // case there was no previous explicit initialization. The initialization is
     // thread safe.
-    static Plugin Plugin;
+    static PluginTy Plugin;
 
     assert(SpecificPlugin && "Plugin is not active");
     return *SpecificPlugin;
@@ -1170,35 +1201,8 @@ public:
   /// Indicate whether the plugin is active.
   static bool isActive() { return SpecificPlugin != nullptr; }
 
-  /// Create a success error. This is the same as calling Error::success(), but
-  /// it is recommended to use this one for consistency with Plugin::error() and
-  /// Plugin::check().
-  static Error success() { return Error::success(); }
-
-  /// Create a string error.
-  template <typename... ArgsTy>
-  static Error error(const char *ErrFmt, ArgsTy... Args) {
-    return createStringError(inconvertibleErrorCode(), ErrFmt, Args...);
-  }
-
-  /// Check the plugin-specific error code and return an error or success
-  /// accordingly. In case of an error, create a string error with the error
-  /// description. The ErrFmt should follow the format:
-  ///     "Error in <function name>[<optional info>]: %s"
-  /// The last format specifier "%s" is mandatory and will be used to place the
-  /// error code's description. Notice this function should be only called from
-  /// the plugin-specific code.
-  template <typename... ArgsTy>
-  static Error check(int32_t ErrorCode, const char *ErrFmt, ArgsTy... Args);
-
   /// Create a plugin instance.
   static GenericPluginTy *createPlugin();
-
-  /// Create a plugin-specific device.
-  static GenericDeviceTy *createDevice(int32_t DeviceId, int32_t NumDevices);
-
-  /// Create a plugin-specific global handler.
-  static GenericGlobalHandlerTy *createGlobalHandler();
 };
 
 /// Auxiliary interface class for GenericDeviceResourceManagerTy. This class
diff --git a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp
index f39f913d85ee..4db787a2ea91 100644
--- a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp
@@ -39,7 +39,7 @@ using namespace omp;
 using namespace target;
 using namespace plugin;
 
-GenericPluginTy *Plugin::SpecificPlugin = nullptr;
+GenericPluginTy *PluginTy::SpecificPlugin = nullptr;
 
 // TODO: Fix any thread safety issues for multi-threaded kernel recording.
 struct RecordReplayTy {
@@ -438,7 +438,7 @@ Error GenericKernelTy::init(GenericDeviceTy &GenericDevice,
   // Retrieve kernel environment object for the kernel.
   GlobalTy KernelEnv(std::string(Name) + "_kernel_environment",
                      sizeof(KernelEnvironment), &KernelEnvironment);
-  GenericGlobalHandlerTy &GHandler = Plugin::get().getGlobalHandler();
+  GenericGlobalHandlerTy &GHandler = PluginTy::get().getGlobalHandler();
   if (auto Err =
           GHandler.readGlobalFromImage(GenericDevice, *ImagePtr, KernelEnv)) {
     [[maybe_unused]] std::string ErrStr = toString(std::move(Err));
@@ -1488,7 +1488,7 @@ Error GenericPluginTy::init() {
   assert(Devices.size() == 0 && "Plugin already initialized");
   Devices.resize(NumDevices, nullptr);
 
-  GlobalHandler = Plugin::createGlobalHandler();
+  GlobalHandler = createGlobalHandler();
   assert(GlobalHandler && "Invalid global handler");
 
   RPCServer = new RPCServerTy(NumDevices);
@@ -1522,7 +1522,7 @@ Error GenericPluginTy::initDevice(int32_t DeviceId) {
   assert(!Devices[DeviceId] && "Device already initialized");
 
   // Create the device and save the reference.
-  GenericDeviceTy *Device = Plugin::createDevice(DeviceId, NumDevices);
+  GenericDeviceTy *Device = createDevice(DeviceId, NumDevices);
   assert(Device && "Invalid device");
 
   // Save the device reference into the list.
@@ -1581,7 +1581,7 @@ extern "C" {
 #endif
 
 int32_t __tgt_rtl_init_plugin() {
-  auto Err = Plugin::initIfNeeded();
+  auto Err = PluginTy::initIfNeeded();
   if (Err) {
     [[maybe_unused]] std::string ErrStr = toString(std::move(Err));
     DP("Failed to init plugin: %s", ErrStr.c_str());
@@ -1592,7 +1592,7 @@ int32_t __tgt_rtl_init_plugin() {
 }
 
 int32_t __tgt_rtl_is_valid_binary(__tgt_device_image *Image) {
-  if (!Plugin::isActive())
+  if (!PluginTy::isActive())
     return false;
 
   StringRef Buffer(reinterpret_cast<const char *>(Image->ImageStart),
@@ -1609,13 +1609,13 @@ int32_t __tgt_rtl_is_valid_binary(__tgt_device_image *Image) {
   case file_magic::elf_executable:
   case file_magic::elf_shared_object:
   case file_magic::elf_core: {
-    auto MatchOrErr = Plugin::get().checkELFImage(Buffer);
+    auto MatchOrErr = PluginTy::get().checkELFImage(Buffer);
     if (Error Err = MatchOrErr.takeError())
       return HandleError(std::move(Err));
     return *MatchOrErr;
   }
   case file_magic::bitcode: {
-    auto MatchOrErr = Plugin::get().getJIT().checkBitcodeImage(Buffer);
+    auto MatchOrErr = PluginTy::get().getJIT().checkBitcodeImage(Buffer);
     if (Error Err = MatchOrErr.takeError())
       return HandleError(std::move(Err));
     return *MatchOrErr;
@@ -1626,7 +1626,7 @@ int32_t __tgt_rtl_is_valid_binary(__tgt_device_image *Image) {
 }
 
 int32_t __tgt_rtl_init_device(int32_t DeviceId) {
-  auto Err = Plugin::get().initDevice(DeviceId);
+  auto Err = PluginTy::get().initDevice(DeviceId);
   if (Err) {
     REPORT("Failure to initialize device %d: %s\n", DeviceId,
            toString(std::move(Err)).data());
@@ -1636,23 +1636,25 @@ int32_t __tgt_rtl_init_device(int32_t DeviceId) {
   return OFFLOAD_SUCCESS;
 }
 
-int32_t __tgt_rtl_number_of_devices() { return Plugin::get().getNumDevices(); }
+int32_t __tgt_rtl_number_of_devices() {
+  return PluginTy::get().getNumDevices();
+}
 
 int64_t __tgt_rtl_init_requires(int64_t RequiresFlags) {
-  Plugin::get().setRequiresFlag(RequiresFlags);
+  PluginTy::get().setRequiresFlag(RequiresFlags);
   return OFFLOAD_SUCCESS;
 }
 
 int32_t __tgt_rtl_is_data_exchangable(int32_t SrcDeviceId,
                                       int32_t DstDeviceId) {
-  return Plugin::get().isDataExchangable(SrcDeviceId, DstDeviceId);
+  return PluginTy::get().isDataExchangable(SrcDeviceId, DstDeviceId);
 }
 
 int32_t __tgt_rtl_initialize_record_replay(int32_t DeviceId, int64_t MemorySize,
                                            void *VAddr, bool isRecord,
                                            bool SaveOutput,
                                            uint64_t &ReqPtrArgOffset) {
-  GenericPluginTy &Plugin = Plugin::get();
+  GenericPluginTy &Plugin = PluginTy::get();
   GenericDeviceTy &Device = Plugin.getDevice(DeviceId);
   RecordReplayTy::RRStatusTy Status =
       isRecord ? RecordReplayTy::RRStatusTy::RRRecording
@@ -1674,7 +1676,7 @@ int32_t __tgt_rtl_initialize_record_replay(int32_t DeviceId, int64_t MemorySize,
 
 int32_t __tgt_rtl_load_binary(int32_t DeviceId, __tgt_device_image *TgtImage,
                               __tgt_device_binary *Binary) {
-  GenericPluginTy &Plugin = Plugin::get();
+  GenericPluginTy &Plugin = PluginTy::get();
   GenericDeviceTy &Device = Plugin.getDevice(DeviceId);
 
   auto ImageOrErr = Device.loadBinary(Plugin, TgtImage);
@@ -1695,7 +1697,7 @@ int32_t __tgt_rtl_load_binary(int32_t DeviceId, __tgt_device_image *TgtImage,
 
 void *__tgt_rtl_data_alloc(int32_t DeviceId, int64_t Size, void *HostPtr,
                            int32_t Kind) {
-  auto AllocOrErr = Plugin::get().getDevice(DeviceId).dataAlloc(
+  auto AllocOrErr = PluginTy::get().getDevice(DeviceId).dataAlloc(
       Size, HostPtr, (TargetAllocTy)Kind);
   if (!AllocOrErr) {
     auto Err = AllocOrErr.takeError();
@@ -1709,8 +1711,8 @@ void *__tgt_rtl_data_alloc(int32_t DeviceId, int64_t Size, void *HostPtr,
 }
 
 int32_t __tgt_rtl_data_delete(int32_t DeviceId, void *TgtPtr, int32_t Kind) {
-  auto Err =
-      Plugin::get().getDevice(DeviceId).dataDelete(TgtPtr, (TargetAllocTy)Kind);
+  auto Err = PluginTy::get().getDevice(DeviceId).dataDelete(
+      TgtPtr, (TargetAllocTy)Kind);
   if (Err) {
     REPORT("Failure to deallocate device pointer %p: %s\n", TgtPtr,
            toString(std::move(Err)).data());
@@ -1722,7 +1724,7 @@ int32_t __tgt_rtl_data_delete(int32_t DeviceId, void *TgtPtr, int32_t Kind) {
 
 int32_t __tgt_rtl_data_lock(int32_t DeviceId, void *Ptr, int64_t Size,
                             void **LockedPtr) {
-  auto LockedPtrOrErr = Plugin::get().getDevice(DeviceId).dataLock(Ptr, Size);
+  auto LockedPtrOrErr = PluginTy::get().getDevice(DeviceId).dataLock(Ptr, Size);
   if (!LockedPtrOrErr) {
     auto Err = LockedPtrOrErr.takeError();
     REPORT("Failure to lock memory %p: %s\n", Ptr,
@@ -1740,7 +1742,7 @@ int32_t __tgt_rtl_data_lock(int32_t DeviceId, void *Ptr, int64_t Size,
 }
 
 int32_t __tgt_rtl_data_unlock(int32_t DeviceId, void *Ptr) {
-  auto Err = Plugin::get().getDevice(DeviceId).dataUnlock(Ptr);
+  auto Err = PluginTy::get().getDevice(DeviceId).dataUnlock(Ptr);
   if (Err) {
     REPORT("Failure to unlock memory %p: %s\n", Ptr,
            toString(std::move(Err)).data());
@@ -1752,7 +1754,7 @@ int32_t __tgt_rtl_data_unlock(int32_t DeviceId, void *Ptr) {
 
 int32_t __tgt_rtl_data_notify_mapped(int32_t DeviceId, void *HstPtr,
                                      int64_t Size) {
-  auto Err = Plugin::get().getDevice(DeviceId).notifyDataMapped(HstPtr, Size);
+  auto Err = PluginTy::get().getDevice(DeviceId).notifyDataMapped(HstPtr, Size);
   if (Err) {
     REPORT("Failure to notify data mapped %p: %s\n", HstPtr,
            toString(std::move(Err)).data());
@@ -1763,7 +1765,7 @@ int32_t __tgt_rtl_data_notify_mapped(int32_t DeviceId, void *HstPtr,
 }
 
 int32_t __tgt_rtl_data_notify_unmapped(int32_t DeviceId, void *HstPtr) {
-  auto Err = Plugin::get().getDevice(DeviceId).notifyDataUnmapped(HstPtr);
+  auto Err = PluginTy::get().getDevice(DeviceId).notifyDataUnmapped(HstPtr);
   if (Err) {
     REPORT("Failure to notify data unmapped %p: %s\n", HstPtr,
            toString(std::move(Err)).data());
@@ -1782,8 +1784,8 @@ int32_t __tgt_rtl_data_submit(int32_t DeviceId, void *TgtPtr, void *HstPtr,
 int32_t __tgt_rtl_data_submit_async(int32_t DeviceId, void *TgtPtr,
                                     void *HstPtr, int64_t Size,
                                     __tgt_async_info *AsyncInfoPtr) {
-  auto Err = Plugin::get().getDevice(DeviceId).dataSubmit(TgtPtr, HstPtr, Size,
-                                                          AsyncInfoPtr);
+  auto Err = PluginTy::get().getDevice(DeviceId).dataSubmit(TgtPtr, HstPtr,
+                                                            Size, AsyncInfoPtr);
   if (Err) {
     REPORT("Failure to copy data from host to device. Pointers: host "
            "= " DPxMOD ", device = " DPxMOD ", size = %" PRId64 ": %s\n",
@@ -1804,8 +1806,8 @@ int32_t __tgt_rtl_data_retrieve(int32_t DeviceId, void *HstPtr, void *TgtPtr,
 int32_t __tgt_rtl_data_retrieve_async(int32_t DeviceId, void *HstPtr,
                                       void *TgtPtr, int64_t Size,
                                       __tgt_async_info *AsyncInfoPtr) {
-  auto Err = Plugin::get().getDevice(DeviceId).dataRetrieve(HstPtr, TgtPtr,
-                                                            Size, AsyncInfoPtr);
+  auto Err = PluginTy::get().getDevice(DeviceId).dataRetrieve(
+      HstPtr, TgtPtr, Size, AsyncInfoPtr);
   if (Err) {
     REPORT("Faliure to copy data from device to host. Pointers: host "
            "= " DPxMOD ", device = " DPxMOD ", size = %" PRId64 ": %s\n",
@@ -1829,8 +1831,8 @@ int32_t __tgt_rtl_data_exchange_async(int32_t SrcDeviceId, void *SrcPtr,
                                       int DstDeviceId, void *DstPtr,
                                       int64_t Size,
                                       __tgt_async_info *AsyncInfo) {
-  GenericDeviceTy &SrcDevice = Plugin::get().getDevice(SrcDeviceId);
-  GenericDeviceTy &DstDevice = Plugin::get().getDevice(DstDeviceId);
+  GenericDeviceTy &SrcDevice = PluginTy::get().getDevice(SrcDeviceId);
+  GenericDeviceTy &DstDevice = PluginTy::get().getDevice(DstDeviceId);
   auto Err = SrcDevice.dataExchange(SrcPtr, DstDevice, DstPtr, Size, AsyncInfo);
   if (Err) {
     REPORT("Failure to copy data from device (%d) to device (%d). Pointers: "
@@ -1847,7 +1849,7 @@ int32_t __tgt_rtl_launch_kernel(int32_t DeviceId, void *TgtEntryPtr,
                                 void **TgtArgs, ptrdiff_t *TgtOffsets,
                                 KernelArgsTy *KernelArgs,
                                 __tgt_async_info *AsyncInfoPtr) {
-  auto Err = Plugin::get().getDevice(DeviceId).launchKernel(
+  auto Err = PluginTy::get().getDevice(DeviceId).launchKernel(
       TgtEntryPtr, TgtArgs, TgtOffsets, *KernelArgs, AsyncInfoPtr);
   if (Err) {
     REPORT("Failure to run target region " DPxMOD " in device %d: %s\n",
@@ -1860,7 +1862,7 @@ int32_t __tgt_rtl_launch_kernel(int32_t DeviceId, void *TgtEntryPtr,
 
 int32_t __tgt_rtl_synchronize(int32_t DeviceId,
                               __tgt_async_info *AsyncInfoPtr) {
-  auto Err = Plugin::get().getDevice(DeviceId).synchronize(AsyncInfoPtr);
+  auto Err = PluginTy::get().getDevice(DeviceId).synchronize(AsyncInfoPtr);
   if (Err) {
     REPORT("Failure to synchronize stream %p: %s\n", AsyncInfoPtr->Queue,
            toString(std::move(Err)).data());
@@ -1872,7 +1874,7 @@ int32_t __tgt_rtl_synchronize(int32_t DeviceId,
 
 int32_t __tgt_rtl_query_async(int32_t DeviceId,
                               __tgt_async_info *AsyncInfoPtr) {
-  auto Err = Plugin::get().getDevice(DeviceId).queryAsync(AsyncInfoPtr);
+  auto Err = PluginTy::get().getDevice(DeviceId).queryAsync(AsyncInfoPtr);
   if (Err) {
     REPORT("Failure to query stream %p: %s\n", AsyncInfoPtr->Queue,
            toString(std::move(Err)).data());
@@ -1883,13 +1885,13 @@ int32_t __tgt_rtl_query_async(int32_t DeviceId,
 }
 
 void __tgt_rtl_print_device_info(int32_t DeviceId) {
-  if (auto Err = Plugin::get().getDevice(DeviceId).printInfo())
+  if (auto Err = PluginTy::get().getDevice(DeviceId).printInfo())
     REPORT("Failure to print device %d info: %s\n", DeviceId,
            toString(std::move(Err)).data());
 }
 
 int32_t __tgt_rtl_create_event(int32_t DeviceId, void **EventPtr) {
-  auto Err = Plugin::get().getDevice(DeviceId).createEvent(EventPtr);
+  auto Err = PluginTy::get().getDevice(DeviceId).createEvent(EventPtr);
   if (Err) {
     REPORT("Failure to create event: %s\n", toString(std::move(Err)).data());
     return OFFLOAD_FAIL;
@@ -1901,7 +1903,7 @@ int32_t __tgt_rtl_create_event(int32_t DeviceId, void **EventPtr) {
 int32_t __tgt_rtl_record_event(int32_t DeviceId, void *EventPtr,
                                __tgt_async_info *AsyncInfoPtr) {
   auto Err =
-      Plugin::get().getDevice(DeviceId).recordEvent(EventPtr, AsyncInfoPtr);
+      PluginTy::get().getDevice(DeviceId).recordEvent(EventPtr, AsyncInfoPtr);
   if (Err) {
     REPORT("Failure to record event %p: %s\n", EventPtr,
            toString(std::move(Err)).data());
@@ -1914,7 +1916,7 @@ int32_t __tgt_rtl_record_event(int32_t DeviceId, void *EventPtr,
 int32_t __tgt_rtl_wait_event(int32_t DeviceId, void *EventPtr,
                              __tgt_async_info *AsyncInfoPtr) {
   auto Err =
-      Plugin::get().getDevice(DeviceId).waitEvent(EventPtr, AsyncInfoPtr);
+      PluginTy::get().getDevice(DeviceId).waitEvent(EventPtr, AsyncInfoPtr);
   if (Err) {
     REPORT("Failure to wait event %p: %s\n", EventPtr,
            toString(std::move(Err)).data());
@@ -1925,7 +1927,7 @@ int32_t __tgt_rtl_wait_event(int32_t DeviceId, void *EventPtr,
 }
 
 int32_t __tgt_rtl_sync_event(int32_t DeviceId, void *EventPtr) {
-  auto Err = Plugin::get().getDevice(DeviceId).syncEvent(EventPtr);
+  auto Err = PluginTy::get().getDevice(DeviceId).syncEvent(EventPtr);
   if (Err) {
     REPORT("Failure to synchronize event %p: %s\n", EventPtr,
            toString(std::move(Err)).data());
@@ -1936,7 +1938,7 @@ int32_t __tgt_rtl_sync_event(int32_t DeviceId, void *EventPtr) {
 }
 
 int32_t __tgt_rtl_destroy_event(int32_t DeviceId, void *EventPtr) {
-  auto Err = Plugin::get().getDevice(DeviceId).destroyEvent(EventPtr);
+  auto Err = PluginTy::get().getDevice(DeviceId).destroyEvent(EventPtr);
   if (Err) {
     REPORT("Failure to destroy event %p: %s\n", EventPtr,
            toString(std::move(Err)).data());
@@ -1955,7 +1957,7 @@ int32_t __tgt_rtl_init_async_info(int32_t DeviceId,
                                   __tgt_async_info **AsyncInfoPtr) {
   assert(AsyncInfoPtr && "Invalid async info");
 
-  auto Err = Plugin::get().getDevice(DeviceId).initAsyncInfo(AsyncInfoPtr);
+  auto Err = PluginTy::get().getDevice(DeviceId).initAsyncInfo(AsyncInfoPtr);
   if (Err) {
     REPORT("Failure to initialize async info at " DPxMOD " on device %d: %s\n",
            DPxPTR(*AsyncInfoPtr), DeviceId, toString(std::move(Err)).data());
@@ -1970,7 +1972,7 @@ int32_t __tgt_rtl_init_device_info(int32_t DeviceId,
                                    const char **ErrStr) {
   *ErrStr = "";
 
-  auto Err = Plugin::get().getDevice(DeviceId).initDeviceInfo(DeviceInfo);
+  auto Err = PluginTy::get().getDevice(DeviceId).initDeviceInfo(DeviceInfo);
   if (Err) {
     REPORT("Failure to initialize device info at " DPxMOD " on device %d: %s\n",
            DPxPTR(DeviceInfo), DeviceId, toString(std::move(Err)).data());
@@ -1981,7 +1983,7 @@ int32_t __tgt_rtl_init_device_info(int32_t DeviceId,
 }
 
 int32_t __tgt_rtl_set_device_offset(int32_t DeviceIdOffset) {
-  Plugin::get().setDeviceIdStartIndex(DeviceIdOffset);
+  PluginTy::get().setDeviceIdStartIndex(DeviceIdOffset);
 
   return OFFLOAD_SUCCESS;
 }
@@ -1990,9 +1992,9 @@ int32_t __tgt_rtl_use_auto_zero_copy(int32_t DeviceId) {
   // Automatic zero-copy only applies to programs that did
   // not request unified_shared_memory and are deployed on an
   // APU with XNACK enabled.
-  if (Plugin::get().getRequiresFlags() & OMP_REQ_UNIFIED_SHARED_MEMORY)
+  if (PluginTy::get().getRequiresFlags() & OMP_REQ_UNIFIED_SHARED_MEMORY)
     return false;
-  return Plugin::get().getDevice(DeviceId).useAutoZeroCopy();
+  return PluginTy::get().getDevice(DeviceId).useAutoZeroCopy();
 }
 
 int32_t __tgt_rtl_get_global(__tgt_device_binary Binary, uint64_t Size,
@@ -2000,7 +2002,7 @@ int32_t __tgt_rtl_get_global(__tgt_device_binary Binary, uint64_t Size,
   assert(Binary.handle && "Invalid device binary handle");
   DeviceImageTy &Image = *reinterpret_cast<DeviceImageTy *>(Binary.handle);
 
-  GenericPluginTy &Plugin = Plugin::get();
+  GenericPluginTy &Plugin = PluginTy::get();
   GenericDeviceTy &Device = Image.getDevice();
 
   GlobalTy DeviceGlobal(Name, Size);
diff --git a/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp b/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp
index b862bc749092..d43c723c3502 100644
--- a/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp
+++ b/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp
@@ -471,7 +471,7 @@ struct CUDADeviceTy : public GenericDeviceTy {
   /// Allocate and construct a CUDA kernel.
   Expected<GenericKernelTy &> constructKernel(const char *Name) override {
     // Allocate and construct the CUDA kernel.
-    CUDAKernelTy *CUDAKernel = Plugin::get().allocate<CUDAKernelTy>();
+    CUDAKernelTy *CUDAKernel = PluginTy::get().allocate<CUDAKernelTy>();
     if (!CUDAKernel)
       return Plugin::error("Failed to allocate memory for CUDA kernel");
 
@@ -529,7 +529,8 @@ struct CUDADeviceTy : public GenericDeviceTy {
       return std::move(Err);
 
     // Allocate and initialize the image object.
-    CUDADeviceImageTy *CUDAImage = Plugin::get().allocate<CUDADeviceImageTy>();
+    CUDADeviceImageTy *CUDAImage =
+        PluginTy::get().allocate<CUDADeviceImageTy>();
     new (CUDAImage) CUDADeviceImageTy(ImageId, *this, TgtImage);
 
     // Load the CUDA module.
@@ -1371,6 +1372,16 @@ struct CUDAPluginTy final : public GenericPluginTy {
   /// Deinitialize the plugin.
   Error deinitImpl() override { return Plugin::success(); }
 
+  /// Creates a CUDA device to use for offloading.
+  GenericDeviceTy *createDevice(int32_t DeviceId, int32_t NumDevices) override {
+    return new CUDADeviceTy(DeviceId, NumDevices);
+  }
+
+  /// Creates a CUDA global handler.
+  GenericGlobalHandlerTy *createGlobalHandler() override {
+    return new CUDAGlobalHandlerTy();
+  }
+
   /// Get the ELF code for recognizing the compatible image binary.
   uint16_t getMagicElfBits() const override { return ELF::EM_CUDA; }
 
@@ -1484,18 +1495,10 @@ Error CUDADeviceTy::dataExchangeImpl(const void *SrcPtr,
   return Plugin::check(Res, "Error in cuMemcpyDtoDAsync: %s");
 }
 
-GenericPluginTy *Plugin::createPlugin() { return new CUDAPluginTy(); }
-
-GenericDeviceTy *Plugin::createDevice(int32_t DeviceId, int32_t NumDevices) {
-  return new CUDADeviceTy(DeviceId, NumDevices);
-}
-
-GenericGlobalHandlerTy *Plugin::createGlobalHandler() {
-  return new CUDAGlobalHandlerTy();
-}
+GenericPluginTy *PluginTy::createPlugin() { return new CUDAPluginTy(); }
 
 template <typename... ArgsTy>
-Error Plugin::check(int32_t Code, const char *ErrFmt, ArgsTy... Args) {
+static Error Plugin::check(int32_t Code, const char *ErrFmt, ArgsTy... Args) {
   CUresult ResultCode = static_cast<CUresult>(Code);
   if (ResultCode == CUDA_SUCCESS)
     return Error::success();
diff --git a/openmp/libomptarget/plugins-nextgen/host/src/rtl.cpp b/openmp/libomptarget/plugins-nextgen/host/src/rtl.cpp
index 1ef18814a26a..ea8ed8d6a856 100644
--- a/openmp/libomptarget/plugins-nextgen/host/src/rtl.cpp
+++ b/openmp/libomptarget/plugins-nextgen/host/src/rtl.cpp
@@ -66,7 +66,7 @@ struct GenELF64KernelTy : public GenericKernelTy {
     GlobalTy Global(getName(), 0);
 
     // Get the metadata (address) of the kernel function.
-    GenericGlobalHandlerTy &GHandler = Plugin::get().getGlobalHandler();
+    GenericGlobalHandlerTy &GHandler = PluginTy::get().getGlobalHandler();
     if (auto Err = GHandler.getGlobalMetadataFromDevice(Device, Image, Global))
       return Err;
 
@@ -150,7 +150,7 @@ struct GenELF64DeviceTy : public GenericDeviceTy {
   Expected<GenericKernelTy &> constructKernel(const char *Name) override {
     // Allocate and construct the kernel.
     GenELF64KernelTy *GenELF64Kernel =
-        Plugin::get().allocate<GenELF64KernelTy>();
+        PluginTy::get().allocate<GenELF64KernelTy>();
     if (!GenELF64Kernel)
       return Plugin::error("Failed to allocate memory for GenELF64 kernel");
 
@@ -167,7 +167,7 @@ struct GenELF64DeviceTy : public GenericDeviceTy {
                                            int32_t ImageId) override {
     // Allocate and initialize the image object.
     GenELF64DeviceImageTy *Image =
-        Plugin::get().allocate<GenELF64DeviceImageTy>();
+        PluginTy::get().allocate<GenELF64DeviceImageTy>();
     new (Image) GenELF64DeviceImageTy(ImageId, *this, TgtImage);
 
     // Create a temporary file.
@@ -399,6 +399,16 @@ struct GenELF64PluginTy final : public GenericPluginTy {
   /// Deinitialize the plugin.
   Error deinitImpl() override { return Plugin::success(); }
 
+  /// Creates a generic ELF device.
+  GenericDeviceTy *createDevice(int32_t DeviceId, int32_t NumDevices) override {
+    return new GenELF64DeviceTy(DeviceId, NumDevices);
+  }
+
+  /// Creates a generic global handler.
+  GenericGlobalHandlerTy *createGlobalHandler() override {
+    return new GenELF64GlobalHandlerTy();
+  }
+
   /// Get the ELF code to recognize the compatible binary images.
   uint16_t getMagicElfBits() const override { return ELF::TARGET_ELF_ID; }
 
@@ -415,18 +425,10 @@ struct GenELF64PluginTy final : public GenericPluginTy {
   }
 };
 
-GenericPluginTy *Plugin::createPlugin() { return new GenELF64PluginTy(); }
-
-GenericDeviceTy *Plugin::createDevice(int32_t DeviceId, int32_t NumDevices) {
-  return new GenELF64DeviceTy(DeviceId, NumDevices);
-}
-
-GenericGlobalHandlerTy *Plugin::createGlobalHandler() {
-  return new GenELF64GlobalHandlerTy();
-}
+GenericPluginTy *PluginTy::createPlugin() { return new GenELF64PluginTy(); }
 
 template <typename... ArgsTy>
-Error Plugin::check(int32_t Code, const char *ErrMsg, ArgsTy... Args) {
+static Error Plugin::check(int32_t Code, const char *ErrMsg, ArgsTy... Args) {
   if (Code == 0)
     return Error::success();
 
-- 
GitLab


From 4b941ff4b4d4c2522a12298624e2847e62a3fd7e Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Mon, 25 Mar 2024 12:59:56 -0700
Subject: [PATCH 188/404] [RISCV] Add coverage for abdu and abds (absolute
 difference)

Test copied from aarch64 with minimal adaption.  We likely need addition
coverage, but this is a reasonable starting point.
---
 llvm/test/CodeGen/RISCV/rvv/abd.ll            | 357 +++++++
 .../CodeGen/RISCV/rvv/fixed-vectors-abd.ll    | 948 ++++++++++++++++++
 2 files changed, 1305 insertions(+)
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/abd.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/fixed-vectors-abd.ll

diff --git a/llvm/test/CodeGen/RISCV/rvv/abd.ll b/llvm/test/CodeGen/RISCV/rvv/abd.ll
new file mode 100644
index 000000000000..7c0dc8688602
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/abd.ll
@@ -0,0 +1,357 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+v,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+v,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
+
+;
+; SABD
+;
+
+define <vscale x 16 x i8> @sabd_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: sabd_b:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vwsub.vv v12, v8, v10
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v12, 0
+; CHECK-NEXT:    vmax.vv v12, v12, v8
+; CHECK-NEXT:    vsetvli zero, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v12, 0
+; CHECK-NEXT:    ret
+  %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i16>
+  %b.sext = sext <vscale x 16 x i8> %b to <vscale x 16 x i16>
+  %sub = sub <vscale x 16 x i16> %a.sext, %b.sext
+  %abs = call <vscale x 16 x i16> @llvm.abs.nxv16i16(<vscale x 16 x i16> %sub, i1 true)
+  %trunc = trunc <vscale x 16 x i16> %abs to <vscale x 16 x i8>
+  ret <vscale x 16 x i8> %trunc
+}
+
+define <vscale x 16 x i8> @sabd_b_promoted_ops(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: sabd_b_promoted_ops:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmv.v.i v10, 0
+; CHECK-NEXT:    vmerge.vim v12, v10, -1, v0
+; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmerge.vim v8, v10, -1, v0
+; CHECK-NEXT:    vsub.vv v8, v12, v8
+; CHECK-NEXT:    vrsub.vi v10, v8, 0
+; CHECK-NEXT:    vmax.vv v8, v8, v10
+; CHECK-NEXT:    ret
+  %a.sext = sext <vscale x 16 x i1> %a to <vscale x 16 x i8>
+  %b.sext = sext <vscale x 16 x i1> %b to <vscale x 16 x i8>
+  %sub = sub <vscale x 16 x i8> %a.sext, %b.sext
+  %abs = call <vscale x 16 x i8> @llvm.abs.nxv16i8(<vscale x 16 x i8> %sub, i1 true)
+  ret <vscale x 16 x i8> %abs
+}
+
+define <vscale x 8 x i16> @sabd_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: sabd_h:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vwsub.vv v12, v8, v10
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v12, 0
+; CHECK-NEXT:    vmax.vv v12, v12, v8
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v12, 0
+; CHECK-NEXT:    ret
+  %a.sext = sext <vscale x 8 x i16> %a to <vscale x 8 x i32>
+  %b.sext = sext <vscale x 8 x i16> %b to <vscale x 8 x i32>
+  %sub = sub <vscale x 8 x i32> %a.sext, %b.sext
+  %abs = call <vscale x 8 x i32> @llvm.abs.nxv8i32(<vscale x 8 x i32> %sub, i1 true)
+  %trunc = trunc <vscale x 8 x i32> %abs to <vscale x 8 x i16>
+  ret <vscale x 8 x i16> %trunc
+}
+
+define <vscale x 8 x i16> @sabd_h_promoted_ops(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
+; CHECK-LABEL: sabd_h_promoted_ops:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a.sext = sext <vscale x 8 x i8> %a to <vscale x 8 x i16>
+  %b.sext = sext <vscale x 8 x i8> %b to <vscale x 8 x i16>
+  %sub = sub <vscale x 8 x i16> %a.sext, %b.sext
+  %abs = call <vscale x 8 x i16> @llvm.abs.nxv8i16(<vscale x 8 x i16> %sub, i1 true)
+  ret <vscale x 8 x i16> %abs
+}
+
+define <vscale x 4 x i32> @sabd_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sabd_s:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vwsub.vv v12, v8, v10
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v12, 0
+; CHECK-NEXT:    vmax.vv v12, v12, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v12, 0
+; CHECK-NEXT:    ret
+  %a.sext = sext <vscale x 4 x i32> %a to <vscale x 4 x i64>
+  %b.sext = sext <vscale x 4 x i32> %b to <vscale x 4 x i64>
+  %sub = sub <vscale x 4 x i64> %a.sext, %b.sext
+  %abs = call <vscale x 4 x i64> @llvm.abs.nxv4i64(<vscale x 4 x i64> %sub, i1 true)
+  %trunc = trunc <vscale x 4 x i64> %abs to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %trunc
+}
+
+define <vscale x 4 x i32> @sabd_s_promoted_ops(<vscale x 4 x i16> %a, <vscale x 4 x i16> %b) {
+; CHECK-LABEL: sabd_s_promoted_ops:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a.sext = sext <vscale x 4 x i16> %a to <vscale x 4 x i32>
+  %b.sext = sext <vscale x 4 x i16> %b to <vscale x 4 x i32>
+  %sub = sub <vscale x 4 x i32> %a.sext, %b.sext
+  %abs = call <vscale x 4 x i32> @llvm.abs.nxv4i32(<vscale x 4 x i32> %sub, i1 true)
+  ret <vscale x 4 x i32> %abs
+}
+
+; FIXME: Crashes legalization if enabled
+;; define <vscale x 2 x i64> @sabd_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+;;   %a.sext = sext <vscale x 2 x i64> %a to <vscale x 2 x i128>
+;;   %b.sext = sext <vscale x 2 x i64> %b to <vscale x 2 x i128>
+;;   %sub = sub <vscale x 2 x i128> %a.sext, %b.sext
+;;   %abs = call <vscale x 2 x i128> @llvm.abs.nxv2i128(<vscale x 2 x i128> %sub, i1 true)
+;;   %trunc = trunc <vscale x 2 x i128> %abs to <vscale x 2 x i64>
+;;   ret <vscale x 2 x i64> %trunc
+;; }
+
+define <vscale x 2 x i64> @sabd_d_promoted_ops(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: sabd_d_promoted_ops:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a.sext = sext <vscale x 2 x i32> %a to <vscale x 2 x i64>
+  %b.sext = sext <vscale x 2 x i32> %b to <vscale x 2 x i64>
+  %sub = sub <vscale x 2 x i64> %a.sext, %b.sext
+  %abs = call <vscale x 2 x i64> @llvm.abs.nxv2i64(<vscale x 2 x i64> %sub, i1 true)
+  ret <vscale x 2 x i64> %abs
+}
+
+;
+; UABD
+;
+
+define <vscale x 16 x i8> @uabd_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: uabd_b:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vwsubu.vv v12, v8, v10
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v12, 0
+; CHECK-NEXT:    vmax.vv v12, v12, v8
+; CHECK-NEXT:    vsetvli zero, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v12, 0
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 16 x i8> %a to <vscale x 16 x i16>
+  %b.zext = zext <vscale x 16 x i8> %b to <vscale x 16 x i16>
+  %sub = sub <vscale x 16 x i16> %a.zext, %b.zext
+  %abs = call <vscale x 16 x i16> @llvm.abs.nxv16i16(<vscale x 16 x i16> %sub, i1 true)
+  %trunc = trunc <vscale x 16 x i16> %abs to <vscale x 16 x i8>
+  ret <vscale x 16 x i8> %trunc
+}
+
+define <vscale x 16 x i8> @uabd_b_promoted_ops(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b) {
+; CHECK-LABEL: uabd_b_promoted_ops:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmv.v.i v10, 0
+; CHECK-NEXT:    vmerge.vim v12, v10, 1, v0
+; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmerge.vim v8, v10, 1, v0
+; CHECK-NEXT:    vsub.vv v8, v12, v8
+; CHECK-NEXT:    vrsub.vi v10, v8, 0
+; CHECK-NEXT:    vmax.vv v8, v8, v10
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 16 x i1> %a to <vscale x 16 x i8>
+  %b.zext = zext <vscale x 16 x i1> %b to <vscale x 16 x i8>
+  %sub = sub <vscale x 16 x i8> %a.zext, %b.zext
+  %abs = call <vscale x 16 x i8> @llvm.abs.nxv16i8(<vscale x 16 x i8> %sub, i1 true)
+  ret <vscale x 16 x i8> %abs
+}
+
+define <vscale x 8 x i16> @uabd_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: uabd_h:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vwsubu.vv v12, v8, v10
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v12, 0
+; CHECK-NEXT:    vmax.vv v12, v12, v8
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v12, 0
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 8 x i16> %a to <vscale x 8 x i32>
+  %b.zext = zext <vscale x 8 x i16> %b to <vscale x 8 x i32>
+  %sub = sub <vscale x 8 x i32> %a.zext, %b.zext
+  %abs = call <vscale x 8 x i32> @llvm.abs.nxv8i32(<vscale x 8 x i32> %sub, i1 true)
+  %trunc = trunc <vscale x 8 x i32> %abs to <vscale x 8 x i16>
+  ret <vscale x 8 x i16> %trunc
+}
+
+define <vscale x 8 x i16> @uabd_h_promoted_ops(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
+; CHECK-LABEL: uabd_h_promoted_ops:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 8 x i8> %a to <vscale x 8 x i16>
+  %b.zext = zext <vscale x 8 x i8> %b to <vscale x 8 x i16>
+  %sub = sub <vscale x 8 x i16> %a.zext, %b.zext
+  %abs = call <vscale x 8 x i16> @llvm.abs.nxv8i16(<vscale x 8 x i16> %sub, i1 true)
+  ret <vscale x 8 x i16> %abs
+}
+
+define <vscale x 4 x i32> @uabd_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: uabd_s:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vwsubu.vv v12, v8, v10
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v12, 0
+; CHECK-NEXT:    vmax.vv v12, v12, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v12, 0
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 4 x i32> %a to <vscale x 4 x i64>
+  %b.zext = zext <vscale x 4 x i32> %b to <vscale x 4 x i64>
+  %sub = sub <vscale x 4 x i64> %a.zext, %b.zext
+  %abs = call <vscale x 4 x i64> @llvm.abs.nxv4i64(<vscale x 4 x i64> %sub, i1 true)
+  %trunc = trunc <vscale x 4 x i64> %abs to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %trunc
+}
+
+define <vscale x 4 x i32> @uabd_s_promoted_ops(<vscale x 4 x i16> %a, <vscale x 4 x i16> %b) {
+; CHECK-LABEL: uabd_s_promoted_ops:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 4 x i16> %a to <vscale x 4 x i32>
+  %b.zext = zext <vscale x 4 x i16> %b to <vscale x 4 x i32>
+  %sub = sub <vscale x 4 x i32> %a.zext, %b.zext
+  %abs = call <vscale x 4 x i32> @llvm.abs.nxv4i32(<vscale x 4 x i32> %sub, i1 true)
+  ret <vscale x 4 x i32> %abs
+}
+
+; FIXME: Crashes legalization if enabled
+;; define <vscale x 2 x i64> @uabd_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+;;   %a.zext = zext <vscale x 2 x i64> %a to <vscale x 2 x i128>
+;;   %b.zext = zext <vscale x 2 x i64> %b to <vscale x 2 x i128>
+;;   %sub = sub <vscale x 2 x i128> %a.zext, %b.zext
+;;   %abs = call <vscale x 2 x i128> @llvm.abs.nxv2i128(<vscale x 2 x i128> %sub, i1 true)
+;;   %trunc = trunc <vscale x 2 x i128> %abs to <vscale x 2 x i64>
+;;   ret <vscale x 2 x i64> %trunc
+;; }
+
+define <vscale x 2 x i64> @uabd_d_promoted_ops(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: uabd_d_promoted_ops:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 2 x i32> %a to <vscale x 2 x i64>
+  %b.zext = zext <vscale x 2 x i32> %b to <vscale x 2 x i64>
+  %sub = sub <vscale x 2 x i64> %a.zext, %b.zext
+  %abs = call <vscale x 2 x i64> @llvm.abs.nxv2i64(<vscale x 2 x i64> %sub, i1 true)
+  ret <vscale x 2 x i64> %abs
+}
+
+; Test the situation where isLegal(ISD::ABD, typeof(%a)) returns true but %a and
+; %b have differing types.
+define <vscale x 4 x i32> @uabd_non_matching_extension(<vscale x 4 x i32> %a, <vscale x 4 x i8> %b) {
+; CHECK-LABEL: uabd_non_matching_extension:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vzext.vf4 v12, v10
+; CHECK-NEXT:    vwsubu.vv v16, v8, v12
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v16, 0
+; CHECK-NEXT:    vmax.vv v12, v16, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v12, 0
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 4 x i32> %a to <vscale x 4 x i64>
+  %b.zext = zext <vscale x 4 x i8> %b to <vscale x 4 x i64>
+  %sub = sub <vscale x 4 x i64> %a.zext, %b.zext
+  %abs = call <vscale x 4 x i64> @llvm.abs.nxv4i64(<vscale x 4 x i64> %sub, i1 true)
+  %trunc = trunc <vscale x 4 x i64> %abs to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %trunc
+}
+
+; Test the situation where isLegal(ISD::ABD, typeof(%a.zext)) returns true but
+; %a and %b have differing types.
+define <vscale x 4 x i32> @uabd_non_matching_promoted_ops(<vscale x 4 x i8> %a, <vscale x 4 x i16> %b) {
+; CHECK-LABEL: uabd_non_matching_promoted_ops:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vzext.vf2 v10, v8
+; CHECK-NEXT:    vwsubu.vv v12, v10, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v12, 0
+; CHECK-NEXT:    vmax.vv v8, v12, v8
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 4 x i8> %a to <vscale x 4 x i32>
+  %b.zext = zext <vscale x 4 x i16> %b to <vscale x 4 x i32>
+  %sub = sub <vscale x 4 x i32> %a.zext, %b.zext
+  %abs = call <vscale x 4 x i32> @llvm.abs.nxv4i32(<vscale x 4 x i32> %sub, i1 true)
+  ret <vscale x 4 x i32> %abs
+}
+
+; Test the situation where isLegal(ISD::ABD, typeof(%a)) returns true but %a and
+; %b are promoted differently.
+define <vscale x 4 x i32> @uabd_non_matching_promotion(<vscale x 4 x i8> %a, <vscale x 4 x i8> %b) {
+; CHECK-LABEL: uabd_non_matching_promotion:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vzext.vf4 v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vsext.vf2 v8, v9
+; CHECK-NEXT:    vwsub.wv v10, v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 4 x i8> %a to <vscale x 4 x i32>
+  %b.zext = sext <vscale x 4 x i8> %b to <vscale x 4 x i32>
+  %sub = sub <vscale x 4 x i32> %a.zext, %b.zext
+  %abs = call <vscale x 4 x i32> @llvm.abs.nxv4i32(<vscale x 4 x i32> %sub, i1 true)
+  ret <vscale x 4 x i32> %abs
+}
+
+declare <vscale x 16 x i8> @llvm.abs.nxv16i8(<vscale x 16 x i8>, i1)
+
+declare <vscale x 8 x i16> @llvm.abs.nxv8i16(<vscale x 8 x i16>, i1)
+declare <vscale x 16 x i16> @llvm.abs.nxv16i16(<vscale x 16 x i16>, i1)
+
+declare <vscale x 4 x i32> @llvm.abs.nxv4i32(<vscale x 4 x i32>, i1)
+declare <vscale x 8 x i32> @llvm.abs.nxv8i32(<vscale x 8 x i32>, i1)
+
+declare <vscale x 2 x i64> @llvm.abs.nxv2i64(<vscale x 2 x i64>, i1)
+declare <vscale x 4 x i64> @llvm.abs.nxv4i64(<vscale x 4 x i64>, i1)
+
+declare <vscale x 2 x i128> @llvm.abs.nxv2i128(<vscale x 2 x i128>, i1)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; RV32: {{.*}}
+; RV64: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-abd.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-abd.ll
new file mode 100644
index 000000000000..79c0857f90ec
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-abd.ll
@@ -0,0 +1,948 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+v,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+v,+d -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
+;
+; SABD
+;
+
+define <8 x i8> @sabd_8b_as_16b(<8 x i8> %a, <8 x i8> %b) {
+;
+; CHECK-LABEL: sabd_8b_as_16b:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    ret
+  %a.sext = sext <8 x i8> %a to <8 x i16>
+  %b.sext = sext <8 x i8> %b to <8 x i16>
+  %sub = sub <8 x i16> %a.sext, %b.sext
+  %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true)
+  %trunc = trunc <8 x i16> %abs to <8 x i8>
+  ret <8 x i8> %trunc
+}
+
+define <8 x i8> @sabd_8b_as_32b(<8 x i8> %a, <8 x i8> %b) {
+;
+; CHECK-LABEL: sabd_8b_as_32b:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vsext.vf2 v10, v8
+; CHECK-NEXT:    vsext.vf2 v8, v9
+; CHECK-NEXT:    vwsub.vv v12, v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v12, 0
+; CHECK-NEXT:    vmax.vv v8, v12, v8
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vnsrl.wi v10, v8, 0
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v10, 0
+; CHECK-NEXT:    ret
+  %a.sext = sext <8 x i8> %a to <8 x i32>
+  %b.sext = sext <8 x i8> %b to <8 x i32>
+  %sub = sub <8 x i32> %a.sext, %b.sext
+  %abs = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %sub, i1 true)
+  %trunc = trunc <8 x i32> %abs to <8 x i8>
+  ret <8 x i8> %trunc
+}
+
+define <16 x i8> @sabd_16b(<16 x i8> %a, <16 x i8> %b) {
+;
+; CHECK-LABEL: sabd_16b:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v10, v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v10, 0
+; CHECK-NEXT:    ret
+  %a.sext = sext <16 x i8> %a to <16 x i16>
+  %b.sext = sext <16 x i8> %b to <16 x i16>
+  %sub = sub <16 x i16> %a.sext, %b.sext
+  %abs = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %sub, i1 true)
+  %trunc = trunc <16 x i16> %abs to <16 x i8>
+  ret <16 x i8> %trunc
+}
+
+define <4 x i16> @sabd_4h(<4 x i16> %a, <4 x i16> %b) {
+;
+; CHECK-LABEL: sabd_4h:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    ret
+  %a.sext = sext <4 x i16> %a to <4 x i32>
+  %b.sext = sext <4 x i16> %b to <4 x i32>
+  %sub = sub <4 x i32> %a.sext, %b.sext
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true)
+  %trunc = trunc <4 x i32> %abs to <4 x i16>
+  ret <4 x i16> %trunc
+}
+
+define <4 x i16> @sabd_4h_promoted_ops(<4 x i8> %a, <4 x i8> %b) {
+;
+; CHECK-LABEL: sabd_4h_promoted_ops:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a.sext = sext <4 x i8> %a to <4 x i16>
+  %b.sext = sext <4 x i8> %b to <4 x i16>
+  %sub = sub <4 x i16> %a.sext, %b.sext
+  %abs = call <4 x i16> @llvm.abs.v4i16(<4 x i16> %sub, i1 true)
+  ret <4 x i16> %abs
+}
+
+define <8 x i16> @sabd_8h(<8 x i16> %a, <8 x i16> %b) {
+;
+; CHECK-LABEL: sabd_8h:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v10, v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v10, 0
+; CHECK-NEXT:    ret
+  %a.sext = sext <8 x i16> %a to <8 x i32>
+  %b.sext = sext <8 x i16> %b to <8 x i32>
+  %sub = sub <8 x i32> %a.sext, %b.sext
+  %abs = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %sub, i1 true)
+  %trunc = trunc <8 x i32> %abs to <8 x i16>
+  ret <8 x i16> %trunc
+}
+
+define <8 x i16> @sabd_8h_promoted_ops(<8 x i8> %a, <8 x i8> %b) {
+;
+; CHECK-LABEL: sabd_8h_promoted_ops:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a.sext = sext <8 x i8> %a to <8 x i16>
+  %b.sext = sext <8 x i8> %b to <8 x i16>
+  %sub = sub <8 x i16> %a.sext, %b.sext
+  %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true)
+  ret <8 x i16> %abs
+}
+
+define <2 x i32> @sabd_2s(<2 x i32> %a, <2 x i32> %b) {
+;
+; CHECK-LABEL: sabd_2s:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    ret
+  %a.sext = sext <2 x i32> %a to <2 x i64>
+  %b.sext = sext <2 x i32> %b to <2 x i64>
+  %sub = sub <2 x i64> %a.sext, %b.sext
+  %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true)
+  %trunc = trunc <2 x i64> %abs to <2 x i32>
+  ret <2 x i32> %trunc
+}
+
+define <2 x i32> @sabd_2s_promoted_ops(<2 x i16> %a, <2 x i16> %b) {
+;
+; CHECK-LABEL: sabd_2s_promoted_ops:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a.sext = sext <2 x i16> %a to <2 x i32>
+  %b.sext = sext <2 x i16> %b to <2 x i32>
+  %sub = sub <2 x i32> %a.sext, %b.sext
+  %abs = call <2 x i32> @llvm.abs.v2i32(<2 x i32> %sub, i1 true)
+  ret <2 x i32> %abs
+}
+
+define <4 x i32> @sabd_4s(<4 x i32> %a, <4 x i32> %b) {
+;
+; CHECK-LABEL: sabd_4s:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v10, v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v10, 0
+; CHECK-NEXT:    ret
+  %a.sext = sext <4 x i32> %a to <4 x i64>
+  %b.sext = sext <4 x i32> %b to <4 x i64>
+  %sub = sub <4 x i64> %a.sext, %b.sext
+  %abs = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %sub, i1 true)
+  %trunc = trunc <4 x i64> %abs to <4 x i32>
+  ret <4 x i32> %trunc
+}
+
+define <4 x i32> @sabd_4s_promoted_ops(<4 x i16> %a, <4 x i16> %b) {
+;
+; CHECK-LABEL: sabd_4s_promoted_ops:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a.sext = sext <4 x i16> %a to <4 x i32>
+  %b.sext = sext <4 x i16> %b to <4 x i32>
+  %sub = sub <4 x i32> %a.sext, %b.sext
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true)
+  ret <4 x i32> %abs
+}
+
+define <2 x i64> @sabd_2d(<2 x i64> %a, <2 x i64> %b) {
+; RV32-LABEL: sabd_2d:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v8, 1
+; RV32-NEXT:    li a1, 32
+; RV32-NEXT:    vsrl.vx v11, v10, a1
+; RV32-NEXT:    vmv.x.s a3, v11
+; RV32-NEXT:    vsrl.vx v11, v8, a1
+; RV32-NEXT:    vmv.x.s a5, v11
+; RV32-NEXT:    srai t1, a5, 31
+; RV32-NEXT:    vmv.x.s a4, v10
+; RV32-NEXT:    vmv.x.s a0, v8
+; RV32-NEXT:    vslidedown.vi v8, v9, 1
+; RV32-NEXT:    vsrl.vx v10, v8, a1
+; RV32-NEXT:    vmv.x.s a6, v10
+; RV32-NEXT:    vsrl.vx v10, v9, a1
+; RV32-NEXT:    vmv.x.s a7, v10
+; RV32-NEXT:    srai t4, a7, 31
+; RV32-NEXT:    vmv.x.s a1, v9
+; RV32-NEXT:    sltu a2, a0, a1
+; RV32-NEXT:    vmv.x.s t0, v8
+; RV32-NEXT:    mv t5, a2
+; RV32-NEXT:    beq a5, a7, .LBB11_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    sltu t5, a5, a7
+; RV32-NEXT:  .LBB11_2:
+; RV32-NEXT:    srai t2, a3, 31
+; RV32-NEXT:    srai t3, a6, 31
+; RV32-NEXT:    sub t6, t1, t4
+; RV32-NEXT:    sltu t5, t6, t5
+; RV32-NEXT:    sltu t1, t1, t4
+; RV32-NEXT:    sltu t4, a4, t0
+; RV32-NEXT:    sub t1, t6, t1
+; RV32-NEXT:    mv t6, t4
+; RV32-NEXT:    beq a3, a6, .LBB11_4
+; RV32-NEXT:  # %bb.3:
+; RV32-NEXT:    sltu t6, a3, a6
+; RV32-NEXT:  .LBB11_4:
+; RV32-NEXT:    sub t1, t1, t5
+; RV32-NEXT:    sub t5, t2, t3
+; RV32-NEXT:    sltu t6, t5, t6
+; RV32-NEXT:    sltu t2, t2, t3
+; RV32-NEXT:    sub t2, t5, t2
+; RV32-NEXT:    sub t2, t2, t6
+; RV32-NEXT:    sub a5, a5, a7
+; RV32-NEXT:    sub a3, a3, a6
+; RV32-NEXT:    sub a3, a3, t4
+; RV32-NEXT:    sub a4, a4, t0
+; RV32-NEXT:    bgez t2, .LBB11_6
+; RV32-NEXT:  # %bb.5:
+; RV32-NEXT:    snez a6, a4
+; RV32-NEXT:    neg a6, a6
+; RV32-NEXT:    sub a3, a6, a3
+; RV32-NEXT:    neg a4, a4
+; RV32-NEXT:  .LBB11_6:
+; RV32-NEXT:    sub a5, a5, a2
+; RV32-NEXT:    sub a0, a0, a1
+; RV32-NEXT:    bgez t1, .LBB11_8
+; RV32-NEXT:  # %bb.7:
+; RV32-NEXT:    snez a1, a0
+; RV32-NEXT:    neg a1, a1
+; RV32-NEXT:    sub a5, a1, a5
+; RV32-NEXT:    neg a0, a0
+; RV32-NEXT:  .LBB11_8:
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a5
+; RV32-NEXT:    vmv.v.x v9, a4
+; RV32-NEXT:    vslide1down.vx v9, v9, a3
+; RV32-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; RV32-NEXT:    vslideup.vi v8, v9, 1
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: sabd_2d:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v8, 1
+; RV64-NEXT:    vmv.x.s a1, v10
+; RV64-NEXT:    srai a4, a1, 63
+; RV64-NEXT:    vmv.x.s a0, v8
+; RV64-NEXT:    srai a3, a0, 63
+; RV64-NEXT:    vslidedown.vi v8, v9, 1
+; RV64-NEXT:    vmv.x.s a5, v8
+; RV64-NEXT:    srai a6, a5, 63
+; RV64-NEXT:    vmv.x.s a2, v9
+; RV64-NEXT:    srai a7, a2, 63
+; RV64-NEXT:    sltu t0, a0, a2
+; RV64-NEXT:    sub a3, a3, a7
+; RV64-NEXT:    sub a3, a3, t0
+; RV64-NEXT:    sltu a7, a1, a5
+; RV64-NEXT:    sub a4, a4, a6
+; RV64-NEXT:    sub a4, a4, a7
+; RV64-NEXT:    sub a1, a1, a5
+; RV64-NEXT:    bgez a4, .LBB11_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    neg a1, a1
+; RV64-NEXT:  .LBB11_2:
+; RV64-NEXT:    sub a0, a0, a2
+; RV64-NEXT:    bgez a3, .LBB11_4
+; RV64-NEXT:  # %bb.3:
+; RV64-NEXT:    neg a0, a0
+; RV64-NEXT:  .LBB11_4:
+; RV64-NEXT:    vmv.s.x v8, a0
+; RV64-NEXT:    vmv.s.x v9, a1
+; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; RV64-NEXT:    vslideup.vi v8, v9, 1
+; RV64-NEXT:    ret
+  %a.sext = sext <2 x i64> %a to <2 x i128>
+  %b.sext = sext <2 x i64> %b to <2 x i128>
+  %sub = sub <2 x i128> %a.sext, %b.sext
+  %abs = call <2 x i128> @llvm.abs.v2i128(<2 x i128> %sub, i1 true)
+  %trunc = trunc <2 x i128> %abs to <2 x i64>
+  ret <2 x i64> %trunc
+}
+
+define <2 x i64> @sabd_2d_promoted_ops(<2 x i32> %a, <2 x i32> %b) {
+;
+; CHECK-LABEL: sabd_2d_promoted_ops:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a.sext = sext <2 x i32> %a to <2 x i64>
+  %b.sext = sext <2 x i32> %b to <2 x i64>
+  %sub = sub <2 x i64> %a.sext, %b.sext
+  %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true)
+  ret <2 x i64> %abs
+}
+
+;
+; UABD
+;
+
+define <8 x i8> @uabd_8b(<8 x i8> %a, <8 x i8> %b) {
+;
+; CHECK-LABEL: uabd_8b:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    ret
+  %a.zext = zext <8 x i8> %a to <8 x i16>
+  %b.zext = zext <8 x i8> %b to <8 x i16>
+  %sub = sub <8 x i16> %a.zext, %b.zext
+  %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true)
+  %trunc = trunc <8 x i16> %abs to <8 x i8>
+  ret <8 x i8> %trunc
+}
+
+define <16 x i8> @uabd_16b(<16 x i8> %a, <16 x i8> %b) {
+;
+; CHECK-LABEL: uabd_16b:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v10, v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v10, 0
+; CHECK-NEXT:    ret
+  %a.zext = zext <16 x i8> %a to <16 x i16>
+  %b.zext = zext <16 x i8> %b to <16 x i16>
+  %sub = sub <16 x i16> %a.zext, %b.zext
+  %abs = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %sub, i1 true)
+  %trunc = trunc <16 x i16> %abs to <16 x i8>
+  ret <16 x i8> %trunc
+}
+
+define <4 x i16> @uabd_4h(<4 x i16> %a, <4 x i16> %b) {
+;
+; CHECK-LABEL: uabd_4h:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    ret
+  %a.zext = zext <4 x i16> %a to <4 x i32>
+  %b.zext = zext <4 x i16> %b to <4 x i32>
+  %sub = sub <4 x i32> %a.zext, %b.zext
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true)
+  %trunc = trunc <4 x i32> %abs to <4 x i16>
+  ret <4 x i16> %trunc
+}
+
+define <4 x i16> @uabd_4h_promoted_ops(<4 x i8> %a, <4 x i8> %b) {
+;
+; CHECK-LABEL: uabd_4h_promoted_ops:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a.zext = zext <4 x i8> %a to <4 x i16>
+  %b.zext = zext <4 x i8> %b to <4 x i16>
+  %sub = sub <4 x i16> %a.zext, %b.zext
+  %abs = call <4 x i16> @llvm.abs.v4i16(<4 x i16> %sub, i1 true)
+  ret <4 x i16> %abs
+}
+
+define <8 x i16> @uabd_8h(<8 x i16> %a, <8 x i16> %b) {
+;
+; CHECK-LABEL: uabd_8h:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v10, v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v10, 0
+; CHECK-NEXT:    ret
+  %a.zext = zext <8 x i16> %a to <8 x i32>
+  %b.zext = zext <8 x i16> %b to <8 x i32>
+  %sub = sub <8 x i32> %a.zext, %b.zext
+  %abs = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %sub, i1 true)
+  %trunc = trunc <8 x i32> %abs to <8 x i16>
+  ret <8 x i16> %trunc
+}
+
+define <8 x i16> @uabd_8h_promoted_ops(<8 x i8> %a, <8 x i8> %b) {
+;
+; CHECK-LABEL: uabd_8h_promoted_ops:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a.zext = zext <8 x i8> %a to <8 x i16>
+  %b.zext = zext <8 x i8> %b to <8 x i16>
+  %sub = sub <8 x i16> %a.zext, %b.zext
+  %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true)
+  ret <8 x i16> %abs
+}
+
+define <2 x i32> @uabd_2s(<2 x i32> %a, <2 x i32> %b) {
+;
+; CHECK-LABEL: uabd_2s:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    ret
+  %a.zext = zext <2 x i32> %a to <2 x i64>
+  %b.zext = zext <2 x i32> %b to <2 x i64>
+  %sub = sub <2 x i64> %a.zext, %b.zext
+  %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true)
+  %trunc = trunc <2 x i64> %abs to <2 x i32>
+  ret <2 x i32> %trunc
+}
+
+define <2 x i32> @uabd_2s_promoted_ops(<2 x i16> %a, <2 x i16> %b) {
+;
+; CHECK-LABEL: uabd_2s_promoted_ops:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a.zext = zext <2 x i16> %a to <2 x i32>
+  %b.zext = zext <2 x i16> %b to <2 x i32>
+  %sub = sub <2 x i32> %a.zext, %b.zext
+  %abs = call <2 x i32> @llvm.abs.v2i32(<2 x i32> %sub, i1 true)
+  ret <2 x i32> %abs
+}
+
+define <4 x i32> @uabd_4s(<4 x i32> %a, <4 x i32> %b) {
+;
+; CHECK-LABEL: uabd_4s:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v10, v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vnsrl.wi v8, v10, 0
+; CHECK-NEXT:    ret
+  %a.zext = zext <4 x i32> %a to <4 x i64>
+  %b.zext = zext <4 x i32> %b to <4 x i64>
+  %sub = sub <4 x i64> %a.zext, %b.zext
+  %abs = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %sub, i1 true)
+  %trunc = trunc <4 x i64> %abs to <4 x i32>
+  ret <4 x i32> %trunc
+}
+
+define <4 x i32> @uabd_4s_promoted_ops(<4 x i16> %a, <4 x i16> %b) {
+;
+; CHECK-LABEL: uabd_4s_promoted_ops:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a.zext = zext <4 x i16> %a to <4 x i32>
+  %b.zext = zext <4 x i16> %b to <4 x i32>
+  %sub = sub <4 x i32> %a.zext, %b.zext
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true)
+  ret <4 x i32> %abs
+}
+
+define <2 x i64> @uabd_2d(<2 x i64> %a, <2 x i64> %b) {
+; RV32-LABEL: uabd_2d:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v8, 1
+; RV32-NEXT:    li a4, 32
+; RV32-NEXT:    vsrl.vx v11, v10, a4
+; RV32-NEXT:    vmv.x.s a0, v11
+; RV32-NEXT:    vsrl.vx v11, v8, a4
+; RV32-NEXT:    vmv.x.s a3, v11
+; RV32-NEXT:    vmv.x.s a1, v10
+; RV32-NEXT:    vmv.x.s a2, v8
+; RV32-NEXT:    vslidedown.vi v8, v9, 1
+; RV32-NEXT:    vsrl.vx v10, v8, a4
+; RV32-NEXT:    vmv.x.s a7, v10
+; RV32-NEXT:    vsrl.vx v10, v9, a4
+; RV32-NEXT:    vmv.x.s a6, v10
+; RV32-NEXT:    vmv.x.s a4, v9
+; RV32-NEXT:    sltu a5, a2, a4
+; RV32-NEXT:    vmv.x.s t0, v8
+; RV32-NEXT:    mv t2, a5
+; RV32-NEXT:    beq a3, a6, .LBB23_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    sltu t2, a3, a6
+; RV32-NEXT:  .LBB23_2:
+; RV32-NEXT:    sltu t3, a1, t0
+; RV32-NEXT:    mv t4, t3
+; RV32-NEXT:    beq a0, a7, .LBB23_4
+; RV32-NEXT:  # %bb.3:
+; RV32-NEXT:    sltu t4, a0, a7
+; RV32-NEXT:  .LBB23_4:
+; RV32-NEXT:    sub t1, a3, a6
+; RV32-NEXT:    neg a3, t2
+; RV32-NEXT:    neg a6, t4
+; RV32-NEXT:    sub a0, a0, a7
+; RV32-NEXT:    sub a0, a0, t3
+; RV32-NEXT:    sub a1, a1, t0
+; RV32-NEXT:    bgez a6, .LBB23_6
+; RV32-NEXT:  # %bb.5:
+; RV32-NEXT:    snez a7, a1
+; RV32-NEXT:    neg a7, a7
+; RV32-NEXT:    sub a0, a7, a0
+; RV32-NEXT:  .LBB23_6:
+; RV32-NEXT:    sub a5, t1, a5
+; RV32-NEXT:    sub a2, a2, a4
+; RV32-NEXT:    bltz a3, .LBB23_11
+; RV32-NEXT:  # %bb.7:
+; RV32-NEXT:    bltz a6, .LBB23_12
+; RV32-NEXT:  .LBB23_8:
+; RV32-NEXT:    bgez a3, .LBB23_10
+; RV32-NEXT:  .LBB23_9:
+; RV32-NEXT:    neg a2, a2
+; RV32-NEXT:  .LBB23_10:
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a2
+; RV32-NEXT:    vslide1down.vx v8, v8, a5
+; RV32-NEXT:    vmv.v.x v9, a1
+; RV32-NEXT:    vslide1down.vx v9, v9, a0
+; RV32-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; RV32-NEXT:    vslideup.vi v8, v9, 1
+; RV32-NEXT:    ret
+; RV32-NEXT:  .LBB23_11:
+; RV32-NEXT:    snez a4, a2
+; RV32-NEXT:    neg a4, a4
+; RV32-NEXT:    sub a5, a4, a5
+; RV32-NEXT:    bgez a6, .LBB23_8
+; RV32-NEXT:  .LBB23_12:
+; RV32-NEXT:    neg a1, a1
+; RV32-NEXT:    bltz a3, .LBB23_9
+; RV32-NEXT:    j .LBB23_10
+;
+; RV64-LABEL: uabd_2d:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v8, 1
+; RV64-NEXT:    vmv.x.s a0, v10
+; RV64-NEXT:    vmv.x.s a1, v8
+; RV64-NEXT:    vslidedown.vi v8, v9, 1
+; RV64-NEXT:    vmv.x.s a4, v8
+; RV64-NEXT:    vmv.x.s a2, v9
+; RV64-NEXT:    sltu a3, a1, a2
+; RV64-NEXT:    neg a3, a3
+; RV64-NEXT:    sltu a5, a0, a4
+; RV64-NEXT:    neg a5, a5
+; RV64-NEXT:    sub a0, a0, a4
+; RV64-NEXT:    bgez a5, .LBB23_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    neg a0, a0
+; RV64-NEXT:  .LBB23_2:
+; RV64-NEXT:    sub a1, a1, a2
+; RV64-NEXT:    bgez a3, .LBB23_4
+; RV64-NEXT:  # %bb.3:
+; RV64-NEXT:    neg a1, a1
+; RV64-NEXT:  .LBB23_4:
+; RV64-NEXT:    vmv.s.x v8, a1
+; RV64-NEXT:    vmv.s.x v9, a0
+; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; RV64-NEXT:    vslideup.vi v8, v9, 1
+; RV64-NEXT:    ret
+  %a.zext = zext <2 x i64> %a to <2 x i128>
+  %b.zext = zext <2 x i64> %b to <2 x i128>
+  %sub = sub <2 x i128> %a.zext, %b.zext
+  %abs = call <2 x i128> @llvm.abs.v2i128(<2 x i128> %sub, i1 true)
+  %trunc = trunc <2 x i128> %abs to <2 x i64>
+  ret <2 x i64> %trunc
+}
+
+define <2 x i64> @uabd_2d_promoted_ops(<2 x i32> %a, <2 x i32> %b) {
+;
+; CHECK-LABEL: uabd_2d_promoted_ops:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a.zext = zext <2 x i32> %a to <2 x i64>
+  %b.zext = zext <2 x i32> %b to <2 x i64>
+  %sub = sub <2 x i64> %a.zext, %b.zext
+  %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true)
+  ret <2 x i64> %abs
+}
+
+define <16 x i8> @uabd_v16i8_nuw(<16 x i8> %a, <16 x i8> %b) {
+;
+; CHECK-LABEL: uabd_v16i8_nuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vsub.vv v8, v8, v9
+; CHECK-NEXT:    vrsub.vi v9, v8, 0
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    ret
+  %sub = sub nuw <16 x i8> %a, %b
+  %abs = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %sub, i1 true)
+  ret <16 x i8> %abs
+}
+
+define <8 x i16> @uabd_v8i16_nuw(<8 x i16> %a, <8 x i16> %b) {
+;
+; CHECK-LABEL: uabd_v8i16_nuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vsub.vv v8, v8, v9
+; CHECK-NEXT:    vrsub.vi v9, v8, 0
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    ret
+  %sub = sub nuw <8 x i16> %a, %b
+  %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true)
+  ret <8 x i16> %abs
+}
+
+define <4 x i32> @uabd_v4i32_nuw(<4 x i32> %a, <4 x i32> %b) {
+;
+; CHECK-LABEL: uabd_v4i32_nuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vsub.vv v8, v8, v9
+; CHECK-NEXT:    vrsub.vi v9, v8, 0
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    ret
+  %sub = sub nuw <4 x i32> %a, %b
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true)
+  ret <4 x i32> %abs
+}
+
+define <2 x i64> @uabd_v2i64_nuw(<2 x i64> %a, <2 x i64> %b) {
+;
+; CHECK-LABEL: uabd_v2i64_nuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT:    vsub.vv v8, v8, v9
+; CHECK-NEXT:    vrsub.vi v9, v8, 0
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    ret
+  %sub = sub nuw <2 x i64> %a, %b
+  %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true)
+  ret <2 x i64> %abs
+}
+
+define <16 x i8> @sabd_v16i8_nsw(<16 x i8> %a, <16 x i8> %b) {
+;
+; CHECK-LABEL: sabd_v16i8_nsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vsub.vv v8, v8, v9
+; CHECK-NEXT:    vrsub.vi v9, v8, 0
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    ret
+  %sub = sub nsw <16 x i8> %a, %b
+  %abs = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %sub, i1 true)
+  ret <16 x i8> %abs
+}
+
+define <8 x i16> @sabd_v8i16_nsw(<8 x i16> %a, <8 x i16> %b) {
+;
+; CHECK-LABEL: sabd_v8i16_nsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vsub.vv v8, v8, v9
+; CHECK-NEXT:    vrsub.vi v9, v8, 0
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    ret
+  %sub = sub nsw <8 x i16> %a, %b
+  %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true)
+  ret <8 x i16> %abs
+}
+
+define <4 x i32> @sabd_v4i32_nsw(<4 x i32> %a, <4 x i32> %b) {
+;
+; CHECK-LABEL: sabd_v4i32_nsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vsub.vv v8, v8, v9
+; CHECK-NEXT:    vrsub.vi v9, v8, 0
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    ret
+  %sub = sub nsw <4 x i32> %a, %b
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true)
+  ret <4 x i32> %abs
+}
+
+define <2 x i64> @sabd_v2i64_nsw(<2 x i64> %a, <2 x i64> %b) {
+;
+; CHECK-LABEL: sabd_v2i64_nsw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT:    vsub.vv v8, v8, v9
+; CHECK-NEXT:    vrsub.vi v9, v8, 0
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    ret
+  %sub = sub nsw <2 x i64> %a, %b
+  %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true)
+  ret <2 x i64> %abs
+}
+
+define <16 x i8> @smaxmin_v16i8(<16 x i8> %0, <16 x i8> %1) {
+;
+; CHECK-LABEL: smaxmin_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vmax.vv v10, v8, v9
+; CHECK-NEXT:    vmin.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a = tail call <16 x i8> @llvm.smax.v16i8(<16 x i8> %0, <16 x i8> %1)
+  %b = tail call <16 x i8> @llvm.smin.v16i8(<16 x i8> %0, <16 x i8> %1)
+  %sub = sub <16 x i8> %a, %b
+  ret <16 x i8> %sub
+}
+
+define <8 x i16> @smaxmin_v8i16(<8 x i16> %0, <8 x i16> %1) {
+;
+; CHECK-LABEL: smaxmin_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vmax.vv v10, v8, v9
+; CHECK-NEXT:    vmin.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a = tail call <8 x i16> @llvm.smax.v8i16(<8 x i16> %0, <8 x i16> %1)
+  %b = tail call <8 x i16> @llvm.smin.v8i16(<8 x i16> %0, <8 x i16> %1)
+  %sub = sub <8 x i16> %a, %b
+  ret <8 x i16> %sub
+}
+
+define <4 x i32> @smaxmin_v4i32(<4 x i32> %0, <4 x i32> %1) {
+;
+; CHECK-LABEL: smaxmin_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmax.vv v10, v8, v9
+; CHECK-NEXT:    vmin.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a = tail call <4 x i32> @llvm.smax.v4i32(<4 x i32> %0, <4 x i32> %1)
+  %b = tail call <4 x i32> @llvm.smin.v4i32(<4 x i32> %0, <4 x i32> %1)
+  %sub = sub <4 x i32> %a, %b
+  ret <4 x i32> %sub
+}
+
+define <2 x i64> @smaxmin_v2i64(<2 x i64> %0, <2 x i64> %1) {
+;
+; CHECK-LABEL: smaxmin_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT:    vmax.vv v10, v8, v9
+; CHECK-NEXT:    vmin.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a = tail call <2 x i64> @llvm.smax.v2i64(<2 x i64> %0, <2 x i64> %1)
+  %b = tail call <2 x i64> @llvm.smin.v2i64(<2 x i64> %0, <2 x i64> %1)
+  %sub = sub <2 x i64> %a, %b
+  ret <2 x i64> %sub
+}
+
+define <16 x i8> @umaxmin_v16i8(<16 x i8> %0, <16 x i8> %1) {
+;
+; CHECK-LABEL: umaxmin_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vmaxu.vv v10, v8, v9
+; CHECK-NEXT:    vminu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a = tail call <16 x i8> @llvm.umax.v16i8(<16 x i8> %0, <16 x i8> %1)
+  %b = tail call <16 x i8> @llvm.umin.v16i8(<16 x i8> %0, <16 x i8> %1)
+  %sub = sub <16 x i8> %a, %b
+  ret <16 x i8> %sub
+}
+
+define <8 x i16> @umaxmin_v8i16(<8 x i16> %0, <8 x i16> %1) {
+;
+; CHECK-LABEL: umaxmin_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vmaxu.vv v10, v8, v9
+; CHECK-NEXT:    vminu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a = tail call <8 x i16> @llvm.umax.v8i16(<8 x i16> %0, <8 x i16> %1)
+  %b = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %0, <8 x i16> %1)
+  %sub = sub <8 x i16> %a, %b
+  ret <8 x i16> %sub
+}
+
+define <4 x i32> @umaxmin_v4i32(<4 x i32> %0, <4 x i32> %1) {
+;
+; CHECK-LABEL: umaxmin_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmaxu.vv v10, v8, v9
+; CHECK-NEXT:    vminu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a = tail call <4 x i32> @llvm.umax.v4i32(<4 x i32> %0, <4 x i32> %1)
+  %b = tail call <4 x i32> @llvm.umin.v4i32(<4 x i32> %0, <4 x i32> %1)
+  %sub = sub <4 x i32> %a, %b
+  ret <4 x i32> %sub
+}
+
+define <2 x i64> @umaxmin_v2i64(<2 x i64> %0, <2 x i64> %1) {
+;
+; CHECK-LABEL: umaxmin_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT:    vmaxu.vv v10, v8, v9
+; CHECK-NEXT:    vminu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a = tail call <2 x i64> @llvm.umax.v2i64(<2 x i64> %0, <2 x i64> %1)
+  %b = tail call <2 x i64> @llvm.umin.v2i64(<2 x i64> %0, <2 x i64> %1)
+  %sub = sub <2 x i64> %a, %b
+  ret <2 x i64> %sub
+}
+
+define <16 x i8> @umaxmin_v16i8_com1(<16 x i8> %0, <16 x i8> %1) {
+;
+; CHECK-LABEL: umaxmin_v16i8_com1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vmaxu.vv v10, v8, v9
+; CHECK-NEXT:    vminu.vv v8, v9, v8
+; CHECK-NEXT:    vsub.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %a = tail call <16 x i8> @llvm.umax.v16i8(<16 x i8> %0, <16 x i8> %1)
+  %b = tail call <16 x i8> @llvm.umin.v16i8(<16 x i8> %1, <16 x i8> %0)
+  %sub = sub <16 x i8> %a, %b
+  ret <16 x i8> %sub
+}
+
+declare <8 x i8> @llvm.abs.v8i8(<8 x i8>, i1)
+declare <16 x i8> @llvm.abs.v16i8(<16 x i8>, i1)
+
+declare <4 x i16> @llvm.abs.v4i16(<4 x i16>, i1)
+declare <8 x i16> @llvm.abs.v8i16(<8 x i16>, i1)
+declare <16 x i16> @llvm.abs.v16i16(<16 x i16>, i1)
+
+declare <2 x i32> @llvm.abs.v2i32(<2 x i32>, i1)
+declare <4 x i32> @llvm.abs.v4i32(<4 x i32>, i1)
+declare <8 x i32> @llvm.abs.v8i32(<8 x i32>, i1)
+
+declare <2 x i64> @llvm.abs.v2i64(<2 x i64>, i1)
+declare <4 x i64> @llvm.abs.v4i64(<4 x i64>, i1)
+
+declare <2 x i128> @llvm.abs.v2i128(<2 x i128>, i1)
+
+declare <16 x i8> @llvm.smax.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.smax.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.smax.v4i32(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.smax.v2i64(<2 x i64>, <2 x i64>)
+declare <16 x i8> @llvm.smin.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.smin.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.smin.v4i32(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.smin.v2i64(<2 x i64>, <2 x i64>)
+declare <16 x i8> @llvm.umax.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.umax.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.umax.v4i32(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.umax.v2i64(<2 x i64>, <2 x i64>)
+declare <16 x i8> @llvm.umin.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.umin.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.umin.v4i32(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.umin.v2i64(<2 x i64>, <2 x i64>)
+
-- 
GitLab


From 07ee9bd215d35260c6a56b83ea95b6543db0dd64 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Mon, 25 Mar 2024 13:08:29 -0700
Subject: [PATCH 189/404] [RISCV] Add fixed vector coverage for
 sum-absolute-difference (sad) pattern

This builds on the previously added absolute difference cases, and adds
the reduction at the end.  This is mostly interesting for examining
impact of extend placement when changing the abdu lowering.
---
 .../CodeGen/RISCV/rvv/fixed-vectors-sad.ll    | 191 ++++++++++++++++++
 1 file changed, 191 insertions(+)
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sad.ll

diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sad.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sad.ll
new file mode 100644
index 000000000000..cf177a03a3b4
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sad.ll
@@ -0,0 +1,191 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v | FileCheck %s
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v | FileCheck %s
+
+define signext i16 @sad_4x8_as_i16(<4 x i8> %a, <4 x i8> %b) {
+; CHECK-LABEL: sad_4x8_as_i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vmv.s.x v9, zero
+; CHECK-NEXT:    vredsum.vs v8, v8, v9
+; CHECK-NEXT:    vmv.x.s a0, v8
+; CHECK-NEXT:    ret
+entry:
+  %1 = zext <4 x i8> %a to <4 x i16>
+  %3 = zext <4 x i8> %b to <4 x i16>
+  %4 = sub nsw <4 x i16> %1, %3
+  %5 = tail call <4 x i16> @llvm.abs.v4i16(<4 x i16> %4, i1 true)
+  %6 = tail call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %5)
+  ret i16 %6
+}
+
+define signext i32 @sad_4x8_as_i32(<4 x i8> %a, <4 x i8> %b) {
+; CHECK-LABEL: sad_4x8_as_i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vmv.s.x v9, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vwredsumu.vs v8, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vmv.x.s a0, v8
+; CHECK-NEXT:    ret
+entry:
+  %1 = zext <4 x i8> %a to <4 x i32>
+  %3 = zext <4 x i8> %b to <4 x i32>
+  %4 = sub nsw <4 x i32> %1, %3
+  %5 = tail call <4 x i32> @llvm.abs.v4i32(<4 x i32> %4, i1 true)
+  %6 = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %5)
+  ret i32 %6
+}
+
+define signext i16 @sad_16x8_as_i16(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: sad_16x8_as_i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vmv.s.x v10, zero
+; CHECK-NEXT:    vredsum.vs v8, v8, v10
+; CHECK-NEXT:    vmv.x.s a0, v8
+; CHECK-NEXT:    ret
+entry:
+  %1 = zext <16 x i8> %a to <16 x i16>
+  %3 = zext <16 x i8> %b to <16 x i16>
+  %4 = sub nsw <16 x i16> %1, %3
+  %5 = tail call <16 x i16> @llvm.abs.v16i16(<16 x i16> %4, i1 true)
+  %6 = tail call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %5)
+  ret i16 %6
+}
+
+define signext i32 @sad_16x8_as_i32(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: sad_16x8_as_i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vmv.s.x v10, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vwredsumu.vs v8, v8, v10
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vmv.x.s a0, v8
+; CHECK-NEXT:    ret
+entry:
+  %1 = zext <16 x i8> %a to <16 x i32>
+  %3 = zext <16 x i8> %b to <16 x i32>
+  %4 = sub nsw <16 x i32> %1, %3
+  %5 = tail call <16 x i32> @llvm.abs.v16i32(<16 x i32> %4, i1 true)
+  %6 = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %5)
+  ret i32 %6
+}
+
+define signext i32 @sad_2block_16xi8_as_i32(ptr %a, ptr %b, i32 signext %stridea, i32 signext %strideb) {
+; CHECK-LABEL: sad_2block_16xi8_as_i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vle8.v v9, (a1)
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    add a0, a0, a2
+; CHECK-NEXT:    add a1, a1, a3
+; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vle8.v v9, (a1)
+; CHECK-NEXT:    vrsub.vi v12, v10, 0
+; CHECK-NEXT:    vmax.vv v12, v10, v12
+; CHECK-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    add a0, a0, a2
+; CHECK-NEXT:    add a1, a1, a3
+; CHECK-NEXT:    vle8.v v14, (a0)
+; CHECK-NEXT:    vle8.v v15, (a1)
+; CHECK-NEXT:    vmax.vv v16, v10, v8
+; CHECK-NEXT:    vwaddu.vv v8, v16, v12
+; CHECK-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwsubu.vv v12, v14, v15
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v14, v12, 0
+; CHECK-NEXT:    add a0, a0, a2
+; CHECK-NEXT:    add a1, a1, a3
+; CHECK-NEXT:    vle8.v v16, (a0)
+; CHECK-NEXT:    vle8.v v17, (a1)
+; CHECK-NEXT:    vmax.vv v12, v12, v14
+; CHECK-NEXT:    vwaddu.wv v8, v8, v12
+; CHECK-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwsubu.vv v12, v16, v17
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v14, v12, 0
+; CHECK-NEXT:    vmax.vv v12, v12, v14
+; CHECK-NEXT:    vwaddu.wv v8, v8, v12
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vmv.s.x v12, zero
+; CHECK-NEXT:    vredsum.vs v8, v8, v12
+; CHECK-NEXT:    vmv.x.s a0, v8
+; CHECK-NEXT:    ret
+entry:
+  %idx.ext8 = sext i32 %strideb to i64
+  %idx.ext = sext i32 %stridea to i64
+  %0 = load <16 x i8>, ptr %a, align 1
+  %1 = zext <16 x i8> %0 to <16 x i32>
+  %2 = load <16 x i8>, ptr %b, align 1
+  %3 = zext <16 x i8> %2 to <16 x i32>
+  %4 = sub nsw <16 x i32> %1, %3
+  %5 = tail call <16 x i32> @llvm.abs.v16i32(<16 x i32> %4, i1 true)
+  %6 = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %5)
+  %add.ptr = getelementptr inbounds i8, ptr %a, i64 %idx.ext
+  %add.ptr9 = getelementptr inbounds i8, ptr %b, i64 %idx.ext8
+  %7 = load <16 x i8>, ptr %add.ptr, align 1
+  %8 = zext <16 x i8> %7 to <16 x i32>
+  %9 = load <16 x i8>, ptr %add.ptr9, align 1
+  %10 = zext <16 x i8> %9 to <16 x i32>
+  %11 = sub nsw <16 x i32> %8, %10
+  %12 = tail call <16 x i32> @llvm.abs.v16i32(<16 x i32> %11, i1 true)
+  %13 = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %12)
+  %op.rdx.1 = add i32 %13, %6
+  %add.ptr.1 = getelementptr inbounds i8, ptr %add.ptr, i64 %idx.ext
+  %add.ptr9.1 = getelementptr inbounds i8, ptr %add.ptr9, i64 %idx.ext8
+  %14 = load <16 x i8>, ptr %add.ptr.1, align 1
+  %15 = zext <16 x i8> %14 to <16 x i32>
+  %16 = load <16 x i8>, ptr %add.ptr9.1, align 1
+  %17 = zext <16 x i8> %16 to <16 x i32>
+  %18 = sub nsw <16 x i32> %15, %17
+  %19 = tail call <16 x i32> @llvm.abs.v16i32(<16 x i32> %18, i1 true)
+  %20 = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %19)
+  %op.rdx.2 = add i32 %20, %op.rdx.1
+  %add.ptr.2 = getelementptr inbounds i8, ptr %add.ptr.1, i64 %idx.ext
+  %add.ptr9.2 = getelementptr inbounds i8, ptr %add.ptr9.1, i64 %idx.ext8
+  %21 = load <16 x i8>, ptr %add.ptr.2, align 1
+  %22 = zext <16 x i8> %21 to <16 x i32>
+  %23 = load <16 x i8>, ptr %add.ptr9.2, align 1
+  %24 = zext <16 x i8> %23 to <16 x i32>
+  %25 = sub nsw <16 x i32> %22, %24
+  %26 = tail call <16 x i32> @llvm.abs.v16i32(<16 x i32> %25, i1 true)
+  %27 = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %26)
+  %op.rdx.3 = add i32 %27, %op.rdx.2
+  ret i32 %op.rdx.3
+}
+
+declare <4 x i32> @llvm.abs.v4i32(<4 x i32>, i1)
+declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
+declare <4 x i16> @llvm.abs.v4i16(<4 x i16>, i1)
+declare i16 @llvm.vector.reduce.add.v4i16(<4 x i16>)
+
+declare <16 x i32> @llvm.abs.v16i32(<16 x i32>, i1)
+declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)
+declare <16 x i16> @llvm.abs.v16i16(<16 x i16>, i1)
+declare i16 @llvm.vector.reduce.add.v16i16(<16 x i16>)
-- 
GitLab


From 0fe0ef4d7f33d327b64b08a250be3713d89fa416 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 25 Mar 2024 13:39:42 -0700
Subject: [PATCH 190/404] [RISCV] Remove unused tablegen class. NFC

---
 llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 8be4c7741ca1..e42ac68a8b67 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -3372,22 +3372,6 @@ multiclass VPseudoVMAC_VV_VX_AAXA<string Constraint = ""> {
   }
 }
 
-multiclass VPseudoVMAC_VV_VF_AAXA<string Constraint = ""> {
-  foreach m = MxListF in {
-    defm "" : VPseudoTernaryV_VV_AAXA<m, Constraint>,
-              SchedTernary<"WriteVFMulAddV", "ReadVFMulAddV", "ReadVFMulAddV",
-                           "ReadVFMulAddV", m.MX>;
-  }
-
-  foreach f = FPList in {
-    foreach m = f.MxList in {
-      defm "" : VPseudoTernaryV_VF_AAXA<m, f, Constraint>,
-                SchedTernary<"WriteVFMulAddF", "ReadVFMulAddV", "ReadVFMulAddF",
-                             "ReadVFMulAddV", m.MX>;
-    }
-  }
-}
-
 multiclass VPseudoVMAC_VV_VF_AAXA_RM<string Constraint = ""> {
   foreach m = MxListF in {
     defm "" : VPseudoTernaryV_VV_AAXA_RM<m, Constraint>,
-- 
GitLab


From 0a4299403e66871f88518f7399d511b72c634c1d Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Mon, 25 Mar 2024 16:55:37 -0400
Subject: [PATCH 191/404] [GlobalISel] Fold G_CTTZ if possible (#86224)

This patch tries to fold `G_CTTZ` if possible.
---
 llvm/include/llvm/CodeGen/GlobalISel/Utils.h  |  8 ++--
 llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp | 10 ++++-
 llvm/lib/CodeGen/GlobalISel/Utils.cpp         |  5 ++-
 llvm/unittests/CodeGen/GlobalISel/CSETest.cpp | 42 +++++++++++++++++++
 4 files changed, 58 insertions(+), 7 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index f8900f3434cc..0241ec4f2111 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -308,10 +308,12 @@ std::optional<APFloat> ConstantFoldIntToFloat(unsigned Opcode, LLT DstTy,
                                               Register Src,
                                               const MachineRegisterInfo &MRI);
 
-/// Tries to constant fold a G_CTLZ operation on \p Src. If \p Src is a vector
-/// then it tries to do an element-wise constant fold.
+/// Tries to constant fold a counting-zero operation (G_CTLZ or G_CTTZ) on \p
+/// Src. If \p Src is a vector then it tries to do an element-wise constant
+/// fold.
 std::optional<SmallVector<unsigned>>
-ConstantFoldCTLZ(Register Src, const MachineRegisterInfo &MRI);
+ConstantFoldCountZeros(Register Src, const MachineRegisterInfo &MRI,
+                       std::function<unsigned(APInt)> CB);
 
 /// Test if the given value is known to have exactly one bit set. This differs
 /// from computeKnownBits in that it doesn't necessarily determine which bit is
diff --git a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
index 1869e0d41a51..a0bc325c6cda 100644
--- a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
@@ -256,10 +256,16 @@ MachineInstrBuilder CSEMIRBuilder::buildInstr(unsigned Opc,
       return buildFConstant(DstOps[0], *Cst);
     break;
   }
-  case TargetOpcode::G_CTLZ: {
+  case TargetOpcode::G_CTLZ:
+  case TargetOpcode::G_CTTZ: {
     assert(SrcOps.size() == 1 && "Expected one source");
     assert(DstOps.size() == 1 && "Expected one dest");
-    auto MaybeCsts = ConstantFoldCTLZ(SrcOps[0].getReg(), *getMRI());
+    std::function<unsigned(APInt)> CB;
+    if (Opc == TargetOpcode::G_CTLZ)
+      CB = [](APInt V) -> unsigned { return V.countl_zero(); };
+    else
+      CB = [](APInt V) -> unsigned { return V.countTrailingZeros(); };
+    auto MaybeCsts = ConstantFoldCountZeros(SrcOps[0].getReg(), *getMRI(), CB);
     if (!MaybeCsts)
       break;
     if (MaybeCsts->size() == 1)
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index a9fa73b60a09..8c41f8b1bdcd 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -966,14 +966,15 @@ llvm::ConstantFoldIntToFloat(unsigned Opcode, LLT DstTy, Register Src,
 }
 
 std::optional<SmallVector<unsigned>>
-llvm::ConstantFoldCTLZ(Register Src, const MachineRegisterInfo &MRI) {
+llvm::ConstantFoldCountZeros(Register Src, const MachineRegisterInfo &MRI,
+                             std::function<unsigned(APInt)> CB) {
   LLT Ty = MRI.getType(Src);
   SmallVector<unsigned> FoldedCTLZs;
   auto tryFoldScalar = [&](Register R) -> std::optional<unsigned> {
     auto MaybeCst = getIConstantVRegVal(R, MRI);
     if (!MaybeCst)
       return std::nullopt;
-    return MaybeCst->countl_zero();
+    return CB(*MaybeCst);
   };
   if (Ty.isVector()) {
     // Try to constant fold each element.
diff --git a/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp b/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp
index 116099eff14a..08857de3cf4e 100644
--- a/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/CSETest.cpp
@@ -233,4 +233,46 @@ TEST_F(AArch64GISelMITest, TestConstantFoldCTL) {
   EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 }
 
+TEST_F(AArch64GISelMITest, TestConstantFoldCTT) {
+  setUp();
+  if (!TM)
+    GTEST_SKIP();
+
+  LLT s32 = LLT::scalar(32);
+
+  GISelCSEInfo CSEInfo;
+  CSEInfo.setCSEConfig(std::make_unique<CSEConfigConstantOnly>());
+  CSEInfo.analyze(*MF);
+  B.setCSEInfo(&CSEInfo);
+  CSEMIRBuilder CSEB(B.getState());
+  auto Cst8 = CSEB.buildConstant(s32, 8);
+  auto *CttzDef = &*CSEB.buildCTTZ(s32, Cst8);
+  EXPECT_TRUE(CttzDef->getOpcode() == TargetOpcode::G_CONSTANT);
+  EXPECT_TRUE(CttzDef->getOperand(1).getCImm()->getZExtValue() == 3);
+
+  // Test vector.
+  auto Cst16 = CSEB.buildConstant(s32, 16);
+  auto Cst32 = CSEB.buildConstant(s32, 32);
+  auto Cst64 = CSEB.buildConstant(s32, 64);
+  LLT VecTy = LLT::fixed_vector(4, s32);
+  auto BV = CSEB.buildBuildVector(VecTy, {Cst8.getReg(0), Cst16.getReg(0),
+                                          Cst32.getReg(0), Cst64.getReg(0)});
+  CSEB.buildCTTZ(VecTy, BV);
+
+  auto CheckStr = R"(
+  ; CHECK: [[CST8:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; CHECK: [[CST3:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+  ; CHECK: [[CST16:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; CHECK: [[CST32:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+  ; CHECK: [[CST64:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+  ; CHECK: [[BV1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[CST8]]:_(s32), [[CST16]]:_(s32), [[CST32]]:_(s32), [[CST64]]:_(s32)
+  ; CHECK: [[CST27:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; CHECK: [[CST26:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+  ; CHECK: [[CST25:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+  ; CHECK: [[BV2:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[CST3]]:_(s32), [[CST27]]:_(s32), [[CST26]]:_(s32), [[CST25]]:_(s32)
+  )";
+
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
+}
+
 } // namespace
-- 
GitLab


From 5f1f9cfaa42a8dee59c9f3a354f973fd8cb003d7 Mon Sep 17 00:00:00 2001
From: "Balaji V. Iyer" <43187390+bviyer@users.noreply.github.com>
Date: Mon, 25 Mar 2024 16:05:09 -0500
Subject: [PATCH 192/404] [mlir][Vector] Fix an assertion on failing cast in
 vector-transfer-flatten-patterns (#86030)

When the result is not a vectorType, there is an assert. This patch will
do the check and bail when the result is not a VectorType.
---
 .../Dialect/Vector/Transforms/VectorLinearize.cpp   |  4 ++--
 mlir/test/Dialect/Vector/linearize.mlir             | 13 +++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
index 7ca035370498..38536de43f13 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp
@@ -22,9 +22,9 @@ using namespace mlir;
 static bool isLessThanTargetBitWidth(Operation *op, unsigned targetBitWidth) {
   auto resultTypes = op->getResultTypes();
   for (auto resType : resultTypes) {
-    VectorType vecType = cast<VectorType>(resType);
+    VectorType vecType = dyn_cast<VectorType>(resType);
     // Reject index since getElementTypeBitWidth will abort for Index types.
-    if (vecType.getElementType().isIndex())
+    if (!vecType || vecType.getElementType().isIndex())
       return false;
     unsigned trailingVecDimBitWidth =
         vecType.getShape().back() * vecType.getElementTypeBitWidth();
diff --git a/mlir/test/Dialect/Vector/linearize.mlir b/mlir/test/Dialect/Vector/linearize.mlir
index 2cbf9bec7a41..e865fcb7a419 100644
--- a/mlir/test/Dialect/Vector/linearize.mlir
+++ b/mlir/test/Dialect/Vector/linearize.mlir
@@ -90,3 +90,16 @@ func.func @test_index_no_linearize(%arg0: vector<2x2xindex>, %arg1: vector<2x2xi
     %0 = arith.addi %arg0, %arg1 : vector<2x2xindex>
     return %0 : vector<2x2xindex>
 }
+
+// -----
+
+// vectorizable operation (arith.mulf) with tensor result types.
+
+func.func @nonvec_result(%arg0: tensor<2x2xf32>, %arg1: tensor<2x2xf32>) -> (tensor<2x2xf32>, tensor<2x2xf32>) {
+    // CHECK: %[[MULF:.*]] = arith.mulf %arg0, %arg1 : tensor<2x2xf32>
+    // CHECK128: %[[MULF:.*]] = arith.mulf %arg0, %arg1 : tensor<2x2xf32>
+    // CHECK0: %[[MULF:.*]] = arith.mulf %arg0, %arg1 : tensor<2x2xf32>
+    %0 = arith.mulf %arg0, %arg1 : tensor<2x2xf32>
+
+    return %0, %arg0 : tensor<2x2xf32>, tensor<2x2xf32>
+}
-- 
GitLab


From 36713a47c8fea0732bcb798ffba4e7c5d22d19f3 Mon Sep 17 00:00:00 2001
From: Justin Fargnoli <justinfargnoli@gmail.com>
Date: Mon, 25 Mar 2024 14:09:25 -0700
Subject: [PATCH 193/404] [utils] Use Python3 for `bisect-skip-count` (#84979)

---
 llvm/utils/bisect-skip-count | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/utils/bisect-skip-count b/llvm/utils/bisect-skip-count
index efdd2c937e15..1a64eba381dc 100755
--- a/llvm/utils/bisect-skip-count
+++ b/llvm/utils/bisect-skip-count
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # This script is used to bisect skip and count arguments for --debug-counter.
 # It is similar to bisect, except it understands how to increase skip and decrease count
 #
-- 
GitLab


From a9d8bf41bf9538154bcc3afcef55bdf309890c6f Mon Sep 17 00:00:00 2001
From: Cyndy Ishida <cyndy_ishida@apple.com>
Date: Mon, 25 Mar 2024 17:18:23 -0400
Subject: [PATCH 194/404] [InstallAPI] Silence unused variable warning, NFC

---
 clang/lib/InstallAPI/Visitor.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/clang/lib/InstallAPI/Visitor.cpp b/clang/lib/InstallAPI/Visitor.cpp
index 452c8f2fb1e4..f8f5d8d53d56 100644
--- a/clang/lib/InstallAPI/Visitor.cpp
+++ b/clang/lib/InstallAPI/Visitor.cpp
@@ -205,9 +205,10 @@ bool InstallAPIVisitor::VisitObjCCategoryDecl(const ObjCCategoryDecl *D) {
   const ObjCInterfaceDecl *InterfaceD = D->getClassInterface();
   const StringRef InterfaceName = InterfaceD->getName();
 
-  auto [Category, FA] = Ctx.Slice->addObjCCategory(InterfaceName, CategoryName,
-                                                   Avail, D, *Access);
-  recordObjCInstanceVariables(D->getASTContext(), Category, InterfaceName,
+  std::pair<ObjCCategoryRecord *, FrontendAttrs *> Category =
+      Ctx.Slice->addObjCCategory(InterfaceName, CategoryName, Avail, D,
+                                 *Access);
+  recordObjCInstanceVariables(D->getASTContext(), Category.first, InterfaceName,
                               D->ivars());
   return true;
 }
-- 
GitLab


From 6646fe884c0bba531893549c5bc1b17103faef64 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Mon, 25 Mar 2024 14:21:49 -0700
Subject: [PATCH 195/404] [memprof] Compute CallStackId when deserializing
 IndexedAllocationInfo (#86421)

There are two ways to create in-memory instances of
IndexedAllocationInfo -- deserialization of the raw MemProf data and
that of the indexed MemProf data.

With:

  commit 74799f424063a2d751e0f9ea698db1f4efd0d8b2
  Author: Kazu Hirata <kazu@google.com>
  Date:   Sat Mar 23 19:50:15 2024 -0700

we compute CallStackId for each call stack in IndexedAllocationInfo
while deserializing the raw MemProf data.

This patch does the same while deserilizing the indexed MemProf data.

As with the patch above, this patch does not add any use of
CallStackId yet.
---
 llvm/include/llvm/ProfileData/MemProf.h  |  5 +++++
 llvm/lib/ProfileData/InstrProfReader.cpp |  8 ++++++++
 llvm/lib/ProfileData/MemProf.cpp         | 13 +++++++++----
 3 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h
index 75ea0e49d453..cd0c148b4f2f 100644
--- a/llvm/include/llvm/ProfileData/MemProf.h
+++ b/llvm/include/llvm/ProfileData/MemProf.h
@@ -635,6 +635,11 @@ public:
 // Compute a CallStackId for a given call stack.
 CallStackId hashCallStack(ArrayRef<FrameId> CS);
 
+// Verify that each CallStackId is computed with hashCallStack.  This function
+// is intended to help transition from CallStack to CSId in
+// IndexedAllocationInfo.
+void verifyIndexedMemProfRecord(const IndexedMemProfRecord &Record);
+
 // Verify that each CallStackId is computed with hashCallStack.  This function
 // is intended to help transition from CallStack to CSId in
 // IndexedAllocationInfo.
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index 31b742bca14d..efe135a0b409 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -1261,6 +1261,14 @@ Error IndexedInstrProfReader::readHeader() {
         /*Buckets=*/Start + FrameTableOffset,
         /*Payload=*/Start + FramePayloadOffset,
         /*Base=*/Start, memprof::FrameLookupTrait()));
+
+#if EXPENSIVE_CHECKS
+    // Go through all the records and verify that CSId has been correctly
+    // populated.  Do this only under EXPENSIVE_CHECKS.  Otherwise, we
+    // would defeat the purpose of OnDiskIterableChainedHashTable.
+    for (const auto &Record : MemProfRecordTable->data())
+      verifyIndexedMemProfRecord(Record);
+#endif
   }
 
   // BinaryIdOffset field in the header is only valid when the format version
diff --git a/llvm/lib/ProfileData/MemProf.cpp b/llvm/lib/ProfileData/MemProf.cpp
index bffa4ed19be8..6c419811d59e 100644
--- a/llvm/lib/ProfileData/MemProf.cpp
+++ b/llvm/lib/ProfileData/MemProf.cpp
@@ -53,6 +53,7 @@ IndexedMemProfRecord::deserialize(const MemProfSchema &Schema,
           endian::readNext<FrameId, llvm::endianness::little, unaligned>(Ptr);
       Node.CallStack.push_back(Id);
     }
+    Node.CSId = hashCallStack(Node.CallStack);
     Node.Info.deserialize(Schema, Ptr);
     Ptr += PortableMemInfoBlock::serializedSize();
     Record.AllocSites.push_back(Node);
@@ -130,15 +131,19 @@ CallStackId hashCallStack(ArrayRef<FrameId> CS) {
   return CSId;
 }
 
+void verifyIndexedMemProfRecord(const IndexedMemProfRecord &Record) {
+  for (const auto &AS : Record.AllocSites) {
+    assert(AS.CSId == hashCallStack(AS.CallStack));
+    (void)AS;
+  }
+}
+
 void verifyFunctionProfileData(
     const llvm::MapVector<GlobalValue::GUID, IndexedMemProfRecord>
         &FunctionProfileData) {
   for (const auto &[GUID, Record] : FunctionProfileData) {
     (void)GUID;
-    for (const auto &AS : Record.AllocSites) {
-      assert(AS.CSId == hashCallStack(AS.CallStack));
-      (void)AS;
-    }
+    verifyIndexedMemProfRecord(Record);
   }
 }
 
-- 
GitLab


From 7c8f7544560879dd7879657b0c0b245c00ed0158 Mon Sep 17 00:00:00 2001
From: Chris B <chris.bieneman@me.com>
Date: Mon, 25 Mar 2024 16:23:23 -0500
Subject: [PATCH 196/404] Fix failing tests after #86242 (#86572)

PR #86242 introduced a copy-paste error that caused some tests to fail
by mapping the wrong pass name to a pass. This resolves the issue and
fixes the failing tests.
---
 llvm/lib/Target/DirectX/DirectXPassRegistry.def | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/DirectX/DirectXPassRegistry.def b/llvm/lib/Target/DirectX/DirectXPassRegistry.def
index bbf0c254bb69..1b326d020d51 100644
--- a/llvm/lib/Target/DirectX/DirectXPassRegistry.def
+++ b/llvm/lib/Target/DirectX/DirectXPassRegistry.def
@@ -24,6 +24,6 @@ MODULE_ANALYSIS("dxil-resource", DXILResourceAnalysis())
 #define MODULE_PASS(NAME, CREATE_PASS)
 #endif
 // TODO: rename to print<foo> after NPM switch
-MODULE_PASS("print-dx-shader-flags", DXILResourcePrinterPass(dbgs()))
+MODULE_PASS("print-dx-shader-flags", dxil::ShaderFlagsAnalysisPrinter(dbgs()))
 MODULE_PASS("print-dxil-resource", DXILResourcePrinterPass(dbgs()))
 #undef MODULE_PASS
-- 
GitLab


From 487720fc137ac7e97b15ca50ca2fc4a42430a6be Mon Sep 17 00:00:00 2001
From: Cyndy Ishida <cyndy_ishida@apple.com>
Date: Mon, 25 Mar 2024 14:32:21 -0700
Subject: [PATCH 197/404] Reapply "[InstallAPI] Add --extra* and --exclude* cli
 options for header input (#86522)" (#86574)

---
 .../clang/Basic/DiagnosticInstallAPIKinds.td  |    3 +
 clang/include/clang/InstallAPI/HeaderFile.h   |   54 +-
 clang/include/clang/InstallAPI/MachO.h        |    1 +
 clang/lib/InstallAPI/Frontend.cpp             |    2 +
 clang/lib/InstallAPI/HeaderFile.cpp           |   51 +
 .../Inputs/Simple/Extra/SimpleExtraAPI1.h     |    1 +
 .../Inputs/Simple/Extra/SimpleExtraAPI2.h     |    1 +
 .../Simple/Simple.framework/Headers/Basic.h   |  103 +
 .../Simple.framework/Headers/External.h       |   19 +
 .../Simple/Simple.framework/Headers/Simple.h  |   45 +
 .../Simple.framework/Headers/SimpleAPI.h      |    1 +
 .../PrivateHeaders/SimplePrivate.h            |    5 +
 .../PrivateHeaders/SimplePrivateSPI.h         |    2 +
 .../test/InstallAPI/Inputs/Simple/Simple.yaml | 3196 +++++++++++++++++
 .../Inputs/Simple/SimpleInternalAPI.h         |    3 +
 .../Inputs/Simple/SimpleInternalAPI2.h        |    7 +
 .../Inputs/Simple/SimpleInternalSPI.h         |    5 +
 .../InstallAPI/extra-exclude-headers.test     |  207 ++
 .../tools/clang-installapi/InstallAPIOpts.td  |   32 +
 clang/tools/clang-installapi/Options.cpp      |  121 +
 clang/tools/clang-installapi/Options.h        |   21 +
 llvm/include/llvm/TextAPI/Utils.h             |    7 +
 llvm/lib/TextAPI/Utils.cpp                    |   46 +
 23 files changed, 3931 insertions(+), 2 deletions(-)
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI1.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI2.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Basic.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/External.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Simple.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/SimpleAPI.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivate.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivateSPI.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/Simple.yaml
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI2.h
 create mode 100644 clang/test/InstallAPI/Inputs/Simple/SimpleInternalSPI.h
 create mode 100644 clang/test/InstallAPI/extra-exclude-headers.test

diff --git a/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td b/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td
index a4c6e630ac5f..27df731fa286 100644
--- a/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td
+++ b/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td
@@ -15,6 +15,9 @@ let CategoryName = "Command line" in {
 def err_cannot_write_file : Error<"cannot write file '%0': %1">;
 def err_no_install_name : Error<"no install name specified: add -install_name <path>">;
 def err_no_output_file: Error<"no output file specified">;
+def err_no_such_header_file : Error<"no such %select{public|private|project}1 header file: '%0'">;
+def warn_no_such_excluded_header_file : Warning<"no such excluded %select{public|private}0 header file: '%1'">, InGroup<InstallAPIViolation>;
+def warn_glob_did_not_match: Warning<"glob '%0' did not match any header file">, InGroup<InstallAPIViolation>;
 } // end of command line category.
 
 let CategoryName = "Verification" in {
diff --git a/clang/include/clang/InstallAPI/HeaderFile.h b/clang/include/clang/InstallAPI/HeaderFile.h
index 70e83bbb3e76..235b4da3add8 100644
--- a/clang/include/clang/InstallAPI/HeaderFile.h
+++ b/clang/include/clang/InstallAPI/HeaderFile.h
@@ -13,7 +13,9 @@
 #ifndef LLVM_CLANG_INSTALLAPI_HEADERFILE_H
 #define LLVM_CLANG_INSTALLAPI_HEADERFILE_H
 
+#include "clang/Basic/FileManager.h"
 #include "clang/Basic/LangStandard.h"
+#include "clang/InstallAPI/MachO.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Regex.h"
@@ -56,6 +58,10 @@ class HeaderFile {
   std::string IncludeName;
   /// Supported language mode for header.
   std::optional<clang::Language> Language;
+  /// Exclude header file from processing.
+  bool Excluded{false};
+  /// Add header file to processing.
+  bool Extra{false};
 
 public:
   HeaderFile() = delete;
@@ -71,17 +77,48 @@ public:
   StringRef getIncludeName() const { return IncludeName; }
   StringRef getPath() const { return FullPath; }
 
+  void setExtra(bool V = true) { Extra = V; }
+  void setExcluded(bool V = true) { Excluded = V; }
+  bool isExtra() const { return Extra; }
+  bool isExcluded() const { return Excluded; }
+
   bool useIncludeName() const {
     return Type != HeaderType::Project && !IncludeName.empty();
   }
 
   bool operator==(const HeaderFile &Other) const {
-    return std::tie(Type, FullPath, IncludeName, Language) ==
+    return std::tie(Type, FullPath, IncludeName, Language, Excluded, Extra) ==
            std::tie(Other.Type, Other.FullPath, Other.IncludeName,
-                    Other.Language);
+                    Other.Language, Other.Excluded, Other.Extra);
   }
 };
 
+/// Glob that represents a pattern of header files to retreive.
+class HeaderGlob {
+private:
+  std::string GlobString;
+  llvm::Regex Rule;
+  HeaderType Type;
+  bool FoundMatch{false};
+
+public:
+  HeaderGlob(StringRef GlobString, llvm::Regex &&, HeaderType Type);
+
+  /// Create a header glob from string for the header access level.
+  static llvm::Expected<std::unique_ptr<HeaderGlob>>
+  create(StringRef GlobString, HeaderType Type);
+
+  /// Query if provided header matches glob.
+  bool match(const HeaderFile &Header);
+
+  /// Query if a header was matched in the glob, used primarily for error
+  /// reporting.
+  bool didMatch() { return FoundMatch; }
+
+  /// Provide back input glob string.
+  StringRef str() { return GlobString; }
+};
+
 /// Assemble expected way header will be included by clients.
 /// As in what maps inside the brackets of `#include <IncludeName.h>`
 /// For example,
@@ -93,6 +130,19 @@ public:
 std::optional<std::string> createIncludeHeaderName(const StringRef FullPath);
 using HeaderSeq = std::vector<HeaderFile>;
 
+/// Determine if Path is a header file.
+/// It does not touch the file system.
+///
+/// \param  Path File path to file.
+bool isHeaderFile(StringRef Path);
+
+/// Given input directory, collect all header files.
+///
+/// \param FM FileManager for finding input files.
+/// \param Directory Path to directory file.
+llvm::Expected<PathSeq> enumerateFiles(clang::FileManager &FM,
+                                       StringRef Directory);
+
 } // namespace clang::installapi
 
 #endif // LLVM_CLANG_INSTALLAPI_HEADERFILE_H
diff --git a/clang/include/clang/InstallAPI/MachO.h b/clang/include/clang/InstallAPI/MachO.h
index f0dea8bbd24c..4961c596fd68 100644
--- a/clang/include/clang/InstallAPI/MachO.h
+++ b/clang/include/clang/InstallAPI/MachO.h
@@ -40,6 +40,7 @@ using SymbolSet = llvm::MachO::SymbolSet;
 using SimpleSymbol = llvm::MachO::SimpleSymbol;
 using FileType = llvm::MachO::FileType;
 using PackedVersion = llvm::MachO::PackedVersion;
+using PathSeq = llvm::MachO::PathSeq;
 using Target = llvm::MachO::Target;
 using TargetList = llvm::MachO::TargetList;
 
diff --git a/clang/lib/InstallAPI/Frontend.cpp b/clang/lib/InstallAPI/Frontend.cpp
index 12cd5fcbc22b..e07ccb14e0b8 100644
--- a/clang/lib/InstallAPI/Frontend.cpp
+++ b/clang/lib/InstallAPI/Frontend.cpp
@@ -138,6 +138,8 @@ std::unique_ptr<MemoryBuffer> createInputBuffer(InstallAPIContext &Ctx) {
   SmallString<4096> Contents;
   raw_svector_ostream OS(Contents);
   for (const HeaderFile &H : Ctx.InputHeaders) {
+    if (H.isExcluded())
+      continue;
     if (H.getType() != Ctx.Type)
       continue;
     if (Ctx.LangMode == Language::C || Ctx.LangMode == Language::CXX)
diff --git a/clang/lib/InstallAPI/HeaderFile.cpp b/clang/lib/InstallAPI/HeaderFile.cpp
index c2d8372741ee..0b7041ec8147 100644
--- a/clang/lib/InstallAPI/HeaderFile.cpp
+++ b/clang/lib/InstallAPI/HeaderFile.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/InstallAPI/HeaderFile.h"
+#include "llvm/TextAPI/Utils.h"
 
 using namespace llvm;
 namespace clang::installapi {
@@ -34,4 +35,54 @@ std::optional<std::string> createIncludeHeaderName(const StringRef FullPath) {
   return Matches[1].drop_front(Matches[1].rfind('/') + 1).str() + "/" +
          Matches[3].str();
 }
+
+bool isHeaderFile(StringRef Path) {
+  return StringSwitch<bool>(sys::path::extension(Path))
+      .Cases(".h", ".H", ".hh", ".hpp", ".hxx", true)
+      .Default(false);
+}
+
+llvm::Expected<PathSeq> enumerateFiles(FileManager &FM, StringRef Directory) {
+  PathSeq Files;
+  std::error_code EC;
+  auto &FS = FM.getVirtualFileSystem();
+  for (llvm::vfs::recursive_directory_iterator i(FS, Directory, EC), ie;
+       i != ie; i.increment(EC)) {
+    if (EC)
+      return errorCodeToError(EC);
+
+    // Skip files that do not exist. This usually happens for broken symlinks.
+    if (FS.status(i->path()) == std::errc::no_such_file_or_directory)
+      continue;
+
+    StringRef Path = i->path();
+    if (isHeaderFile(Path))
+      Files.emplace_back(Path);
+  }
+
+  return Files;
+}
+
+HeaderGlob::HeaderGlob(StringRef GlobString, Regex &&Rule, HeaderType Type)
+    : GlobString(GlobString), Rule(std::move(Rule)), Type(Type) {}
+
+bool HeaderGlob::match(const HeaderFile &Header) {
+  if (Header.getType() != Type)
+    return false;
+
+  bool Match = Rule.match(Header.getPath());
+  if (Match)
+    FoundMatch = true;
+  return Match;
+}
+
+Expected<std::unique_ptr<HeaderGlob>> HeaderGlob::create(StringRef GlobString,
+                                                         HeaderType Type) {
+  auto Rule = MachO::createRegexFromGlob(GlobString);
+  if (!Rule)
+    return Rule.takeError();
+
+  return std::make_unique<HeaderGlob>(GlobString, std::move(*Rule), Type);
+}
+
 } // namespace clang::installapi
diff --git a/clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI1.h b/clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI1.h
new file mode 100644
index 000000000000..83a5b9507de3
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI1.h
@@ -0,0 +1 @@
+extern int extraGlobalAPI1;
diff --git a/clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI2.h b/clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI2.h
new file mode 100644
index 000000000000..34fe3364bba8
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/Extra/SimpleExtraAPI2.h
@@ -0,0 +1 @@
+extern int extraGlobalAPI2;
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Basic.h b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Basic.h
new file mode 100644
index 000000000000..08412bb2de28
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Basic.h
@@ -0,0 +1,103 @@
+#import <Foundation/Foundation.h>
+
+// Basic class with no super class
+@interface Basic1
+@end
+
+@interface Basic2 : NSObject
+@end
+
+@interface Basic3 : NSObject
+@property BOOL property1;
+@property(readonly) BOOL property2;
+@property(getter=isProperty3) BOOL property3;
+@property BOOL dynamicProp;
+@end
+
+@interface Basic4 : NSObject {
+@public
+  BOOL ivar1;
+@protected
+  BOOL ivar2;
+@package
+  BOOL ivar3;
+@private
+  BOOL ivar4;
+}
+@end
+
+__attribute__((visibility("hidden"))) @interface Basic4_1 : NSObject {
+@public
+  BOOL ivar1;
+@protected
+  BOOL ivar2;
+@package
+  BOOL ivar3;
+@private
+  BOOL ivar4;
+}
+@end
+
+@interface Basic4_2 : NSObject {
+@private
+  BOOL ivar4;
+@package
+  BOOL ivar3;
+@protected
+  BOOL ivar2;
+@public
+  BOOL ivar1;
+}
+@end
+
+@interface Basic5 : NSObject
++ (void)aClassMethod;
+- (void)anInstanceMethod;
+@end
+
+@interface Basic6 : NSObject
+@end
+
+@interface Basic6 () {
+@public
+  BOOL ivar1;
+}
+@property BOOL property1;
+- (void)anInstanceMethodFromAnExtension;
+@end
+
+@interface Basic6 (Foo)
+@property BOOL property2;
+- (void)anInstanceMethodFromACategory;
+@end
+
+__attribute__((visibility("hidden")))
+@interface Basic7 : NSObject
+@end
+
+@interface Basic7 ()
+- (void) anInstanceMethodFromAnHiddenExtension;
+@end
+
+@interface Basic8 : NSObject
++ (void)useSameName;
+@end
+
+// Classes and protocols can have the same name. For now they would only clash
+// in the selector map if the protocl starts with '_'.
+@protocol _A
+- (void)aMethod;
+@end
+
+@interface A : NSObject
+- (void)aMethod NS_AVAILABLE(10_11, 9_0);
+- (void)bMethod NS_UNAVAILABLE;
+@end
+
+@interface Basic9 : NSObject
+@property(readonly) BOOL aProperty NS_AVAILABLE(10_10, 8_0);
+@end
+
+@interface Basic9 (deprecated)
+@property(readwrite) BOOL aProperty NS_DEPRECATED_MAC(10_8, 10_10);
+@end
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/External.h b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/External.h
new file mode 100644
index 000000000000..5dc3c92f34c2
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/External.h
@@ -0,0 +1,19 @@
+#import <Foundation/Foundation.h>
+
+// Sub-class an external defined ObjC Class.
+@interface ExternalManagedObject : NSManagedObject
+- (void)foo;
+@end
+
+// Add category to external defined ObjC Class.
+@interface NSManagedObject (Simple)
+- (int)supportsSimple;
+@end
+
+// CoreData Accessors are dynamically generated and have no implementation.
+@interface ExternalManagedObject (CoreDataGeneratedAccessors)
+- (void)addChildObject:(ExternalManagedObject *)value;
+- (void)removeChildObject:(ExternalManagedObject *)value;
+- (void)addChild:(NSSet *)values;
+- (void)removeChild:(NSSet *)values;
+@end
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Simple.h b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Simple.h
new file mode 100644
index 000000000000..12c77098a8d9
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/Simple.h
@@ -0,0 +1,45 @@
+#import <Foundation/Foundation.h>
+
+// Useless forward declaration. This is used for testing.
+@class FooBar;
+@protocol FooProtocol;
+
+@protocol ForwardProcotol;
+
+// Test public global.
+extern int publicGlobalVariable;
+
+// Test weak public global.
+extern int weakPublicGlobalVariable __attribute__((weak));
+
+// Test public ObjC class
+@interface Simple : NSObject
+@end
+
+__attribute__((objc_exception))
+@interface Base : NSObject
+@end
+
+@interface SubClass : Base
+@end
+
+@protocol BaseProtocol
+- (void) baseMethod;
+@end
+
+NS_AVAILABLE(10_11, 9_0)
+@protocol FooProtocol <BaseProtocol>
+- (void) protocolMethod;
+@end
+
+@protocol BarProtocol
+- (void) barMethod;
+@end
+
+@interface FooClass <FooProtocol, BarProtocol>
+@end
+
+// Create an empty category conforms to a forward declared protocol.
+// <rdar://problem/35605892>
+@interface FooClass (Test) <ForwardProcotol>
+@end
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/SimpleAPI.h b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/SimpleAPI.h
new file mode 100644
index 000000000000..d953fac966da
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/Headers/SimpleAPI.h
@@ -0,0 +1 @@
+extern int otherFrameworkAPI;
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivate.h b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivate.h
new file mode 100644
index 000000000000..5a28cda3928e
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivate.h
@@ -0,0 +1,5 @@
+// Test private global variable.
+extern int privateGlobalVariable;
+
+// Test weak private global.
+extern int weakPrivateGlobalVariable __attribute__((weak));
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivateSPI.h b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivateSPI.h
new file mode 100644
index 000000000000..c9aca30fa82f
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/Simple.framework/PrivateHeaders/SimplePrivateSPI.h
@@ -0,0 +1,2 @@
+// Test private global variable.
+extern int otherFrameworkSPI;
diff --git a/clang/test/InstallAPI/Inputs/Simple/Simple.yaml b/clang/test/InstallAPI/Inputs/Simple/Simple.yaml
new file mode 100644
index 000000000000..998e51f1a67d
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/Simple.yaml
@@ -0,0 +1,3196 @@
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x1000007
+  cpusubtype:      0x3
+  filetype:        0x6
+  ncmds:           15
+  sizeofcmds:      1952
+  flags:           0x118085
+  reserved:        0x0
+LoadCommands:
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         472
+    segname:         __TEXT
+    vmaddr:          0
+    vmsize:          12288
+    fileoff:         0
+    filesize:        12288
+    maxprot:         5
+    initprot:        5
+    nsects:          5
+    flags:           0
+    Sections:
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x1BC0
+        size:            180
+        offset:          0x1BC0
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         554889E50FBE47085DC3554889E58857085DC3554889E50FBE47095DC3554889E50FBE470A5DC3554889E588570A5DC3554889E55DC3554889E55DC3554889E55DC3554889E50FBE47095DC3554889E58857095DC3554889E5B8010000005DC3554889E55DC3554889E55DC3554889E55DC3554889E55DC3554889E5B0015DC3554889E55DC3554889E55DC3554889E55DC3554889E50FBE47085DC3554889E55DC3554889E55DC3554889E55DC3554889E55DC3
+      - sectname:        __cstring
+        segname:         __TEXT
+        addr:            0x1C74
+        size:            296
+        offset:          0x1C74
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         53696D706C65004261736500537562436C6173730053696D706C65496E7465726E616C4150490053696D706C65496E7465726E616C53504900426173696331004261736963320042617369633300426173696334004261736963345F31004261736963345F32004261736963350042617369633600466F6F004261736963370045787465726E616C4D616E616765644F626A6563740048696464656E436C61737300426173696338004100426173696339006465707265636174656400466F6F436C61737300466F6F50726F746F636F6C004261736550726F746F636F6C0042617250726F746F636F6C0050726976617465005072697661746550726F746F636F6C0063313640303A380076323040303A3863313600630076313640303A380042313640303A3800
+      - sectname:        __objc_methname
+        segname:         __TEXT
+        addr:            0x1D9C
+        size:            450
+        offset:          0x1D9C
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x2
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         70726F7065727479310073657450726F7065727479313A0070726F70657274793200697350726F7065727479330073657450726F7065727479333A0070726F7065727479330054632C5670726F7065727479310054632C522C5670726F7065727479320054632C47697350726F7065727479332C5670726F7065727479330064796E616D696350726F700054632C440069766172310069766172320069766172330069766172340061436C6173734D6574686F6400616E496E7374616E63654D6574686F6400616E496E7374616E63654D6574686F6446726F6D416E457874656E73696F6E0073657450726F7065727479323A00616E496E7374616E63654D6574686F6446726F6D4143617465676F727900546300616E496E7374616E63654D6574686F6446726F6D416E48696464656E457874656E73696F6E00666F6F00737570706F72747353696D706C650075736553616D654E616D6500614D6574686F64006150726F7065727479005F6150726F70657274790054632C522C565F6150726F706572747900626173654D6574686F640070726F746F636F6C4D6574686F64006261724D6574686F64007072697661746550726F636F746F6C4D6574686F6400
+      - sectname:        __unwind_info
+        segname:         __TEXT
+        addr:            0x1F60
+        size:            4152
+        offset:          0x1F60
+        align:           2
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         010000001C000000010000002000000000000000200000000200000000000001C01B00003800000038000000741C00000000000038000000030000000C0001001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+      - sectname:        __eh_frame
+        segname:         __TEXT
+        addr:            0x2F98
+        size:            24
+        offset:          0x2F98
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x6000000B
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         1400000000000000017A520001781001100C070890010000
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         792
+    segname:         __DATA
+    vmaddr:          12288
+    vmsize:          8192
+    fileoff:         12288
+    filesize:        8192
+    maxprot:         3
+    initprot:        3
+    nsects:          9
+    flags:           0
+    Sections:
+      - sectname:        __objc_const
+        segname:         __DATA
+        addr:            0x3000
+        size:            4952
+        offset:          0x3000
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         010000002800000028000000000000000000000000000000741C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000008000000000000000000000000000000741C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000028000000280000000000000000000000000000007B1C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000008000000080000000000000000000000000000007B1C0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000007B1C000000000000D043000000000000010000002800000028000000000000000000000000000000801C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000800000008000000000000000000000000000000801C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000801C0000000000002044000000000000010000002800000028000000000000000000000000000000891C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000800000008000000000000000000000000000000891C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000891C00000000000070440000000000000100000028000000280000000000000000000000000000009B1C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000008000000080000000000000000000000000000009B1C0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000009B1C000000000000C044000000000000030000002800000028000000000000000000000000000000AD1C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000AD1C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000002800000028000000000000000000000000000000B41C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000008000000000000000000000000000000B41C00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000002800000028000000000000000000000000000000BB1C0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000018000000050000009C1D000000000000771D000000000000C01B000000000000A61D0000000000007F1D000000000000CA1B000000000000B41D000000000000771D000000000000D31B000000000000BE1D000000000000771D000000000000DD1B000000000000CA1D0000000000007F1D000000000000E71B000000000000200000000300000098490000000000009C1D0000000000008A1D0000000000000000000001000000A049000000000000B41D0000000000008A1D0000000000000000000001000000A849000000000000D81D0000000000008A1D000000000000000000000100000010000000040000009C1D000000000000E21D000000000000B41D000000000000F01D000000000000D81D000000000000001E0000000000001B1E000000000000271E00000000000000000000080000000B000000000000000000000000000000BB1C00000000000098340000000000000000000000000000183500000000000000000000000000008035000000000000010000002800000028000000000000000000000000000000C21C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000004000000B0490000000000002C1E0000000000008A1D0000000000000000000001000000B849000000000000321E0000000000008A1D0000000000000000000001000000C049000000000000381E0000000000008A1D0000000000000000000001000000C8490000000000003E1E0000000000008A1D000000000000000000000100000000000000080000000C000000000000000000000000000000C21C00000000000000000000000000000000000000000000583600000000000000000000000000000000000000000000110000002800000028000000000000000000000000000000C91C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000004000000D0490000000000002C1E0000000000008A1D0000000000000000000001000000D849000000000000321E0000000000008A1D0000000000000000000001000000E049000000000000381E0000000000008A1D0000000000000000000001000000E8490000000000003E1E0000000000008A1D000000000000000000000100000010000000080000000C000000000000000000000000000000C91C00000000000000000000000000000000000000000000703700000000000000000000000000000000000000000000010000002800000028000000000000000000000000000000D21C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000004000000F0490000000000003E1E0000000000008A1D0000000000000000000001000000F849000000000000381E0000000000008A1D0000000000000000000001000000004A000000000000321E0000000000008A1D0000000000000000000001000000084A0000000000002C1E0000000000008A1D000000000000000000000100000000000000080000000C000000000000000000000000000000D21C000000000000000000000000000000000000000000008838000000000000000000000000000000000000000000001800000001000000441E0000000000008C1D000000000000F01B000000000000010000002800000028000000000000000000000000000000DB1C000000000000583900000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000511E0000000000008C1D000000000000F61B000000000000000000000800000008000000000000000000000000000000DB1C000000000000C0390000000000000000000000000000000000000000000000000000000000000000000000000000010000002800000028000000000000000000000000000000E21C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001800000003000000621E0000000000008C1D000000000000FC1B0000000000009C1D000000000000771D000000000000021C000000000000A61D0000000000007F1D0000000000000C1C0000000000002000000002000000104A0000000000002C1E0000000000008A1D0000000000000000000001000000184A0000000000009C1D0000000000008A1D000000000000000000000100000010000000010000009C1D000000000000E21D00000000000000000000080000000A000000000000000000000000000000E21C000000000000703A0000000000000000000000000000C03A0000000000000000000000000000083B0000000000001800000003000000B41D000000000000771D000000000000151C000000000000821E0000000000007F1D000000000000201C000000000000901E0000000000008C1D000000000000261C0000000000001000000001000000B41D000000000000AE1E000000000000E91C0000000000004047000000000000683B00000000000000000000000000000000000000000000B83B00000000000000000000000000004000000000000000110000002800000028000000000000000000000000000000ED1C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000B11E0000000000008C1D0000000000002C1C000000000000100000000800000008000000000000000000000000000000ED1C000000000000583C0000000000000000000000000000000000000000000000000000000000000000000000000000010000002800000028000000000000000000000000000000F41C000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000D71E0000000000008C1D000000000000321C000000000000000000000800000008000000000000000000000000000000F41C000000000000083D00000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000DB1E000000000000941D000000000000381C000000000000741C0000000000000000000000000000703D000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000001100000028000000280000000000000000000000000000000A1D000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000008000000080000000000000000000000000000000A1D000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000EA1E0000000000008C1D000000000000401C000000000000010000002800000028000000000000000000000000000000161D000000000000603E00000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000EA1E0000000000008C1D000000000000461C000000000000000000000800000008000000000000000000000000000000161D000000000000C83E00000000000000000000000000000000000000000000000000000000000000000000000000000100000028000000280000000000000000000000000000001D1D000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000F61E0000000000008C1D0000000000004C1C0000000000000000000008000000080000000000000000000000000000001D1D000000000000783F00000000000000000000000000000000000000000000000000000000000000000000000000000100000028000000280000000000000000000000000000001F1D000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001800000001000000FE1E000000000000771D000000000000521C0000000000002000000001000000204A000000000000081F0000000000008A1D00000000000000000000010000001000000001000000FE1E000000000000131F0000000000000000000008000000090000000000000000000000000000001F1D000000000000284000000000000000000000000000004840000000000000000000000000000070400000000000001000000001000000FE1E000000000000AE1E000000000000261D0000000000002049000000000000000000000000000000000000000000000000000000000000D040000000000000000000000000000040000000000000001800000001000000241F0000000000008C1D00000000000000000000000000008C1D0000000000000100000000000000284A000000000000000000000000000018000000010000002F1F0000000000008C1D00000000000000000000000000008C1D00000000000018000000010000003E1F0000000000008C1D00000000000000000000000000008C1D0000000000000200000000000000884A000000000000E84A0000000000000000000000000000030000002800000028000000000000000000000000000000311D0000000000000000000000000000B8410000000000000000000000000000000000000000000000000000000000001800000003000000241F0000000000008C1D0000000000005C1C0000000000002F1F0000000000008C1D000000000000621C0000000000003E1F0000000000008C1D000000000000681C000000000000020000000000000000000000000000000000000000000000311D0000000000002042000000000000B8410000000000000000000000000000000000000000000000000000000000001800000001000000481F0000000000008C1D0000000000006E1C0000000000001800000001000000481F0000000000008C1D00000000000000000000000000008C1D0000000000000100000000000000484B00000000000000000000000000005F1D0000000000004849000000000000B84200000000000000000000000000000043000000000000000000000000000000000000000000004000000000000000
+      - sectname:        __objc_data
+        segname:         __DATA
+        addr:            0x4358
+        size:            1600
+        offset:          0x4358
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         000000000000000000000000000000000000000000000000000000000000000000300000000000005843000000000000000000000000000000000000000000000000000000000000483000000000000000000000000000000000000000000000000000000000000000000000000000009030000000000000A843000000000000000000000000000000000000000000000000000000000000D8300000000000000000000000000000A843000000000000000000000000000000000000000000003831000000000000F843000000000000D0430000000000000000000000000000000000000000000080310000000000000000000000000000000000000000000000000000000000000000000000000000E03100000000000048440000000000000000000000000000000000000000000000000000000000002832000000000000000000000000000000000000000000000000000000000000000000000000000088320000000000009844000000000000000000000000000000000000000000000000000000000000D032000000000000104500000000000000000000000000000000000000000000000000000000000078330000000000001045000000000000E8440000000000000000000000000000000000000000000030330000000000000000000000000000000000000000000000000000000000000000000000000000C03300000000000038450000000000000000000000000000000000000000000000000000000000000834000000000000000000000000000000000000000000000000000000000000000000000000000050340000000000008845000000000000000000000000000000000000000000000000000000000000C83500000000000000000000000000000000000000000000000000000000000000000000000000001036000000000000D845000000000000000000000000000000000000000000000000000000000000E036000000000000000000000000000000000000000000000000000000000000000000000000000028370000000000002846000000000000000000000000000000000000000000000000000000000000F837000000000000000000000000000000000000000000000000000000000000000000000000000040380000000000007846000000000000000000000000000000000000000000000000000000000000103900000000000000000000000000000000000000000000000000000000000000000000000000007839000000000000C846000000000000000000000000000000000000000000000000000000000000E0390000000000000000000000000000000000000000000000000000000000000000000000000000283A0000000000001847000000000000000000000000000000000000000000000000000000000000203B0000000000000000000000000000000000000000000000000000000000000000000000000000103C0000000000006847000000000000000000000000000000000000000000000000000000000000783C0000000000000000000000000000000000000000000000000000000000000000000000000000C03C000000000000B847000000000000000000000000000000000000000000000000000000000000283D0000000000000000000000000000000000000000000000000000000000000000000000000000D03D0000000000000848000000000000000000000000000000000000000000000000000000000000183E0000000000000000000000000000000000000000000000000000000000000000000000000000803E0000000000005848000000000000000000000000000000000000000000000000000000000000E83E0000000000000000000000000000000000000000000000000000000000000000000000000000303F000000000000A848000000000000000000000000000000000000000000000000000000000000983F0000000000000000000000000000000000000000000000000000000000000000000000000000E03F000000000000F8480000000000000000000000000000000000000000000000000000000000008840000000000000704900000000000000000000000000000000000000000000000000000000000070420000000000007049000000000000484900000000000000000000000000000000000000000000D841000000000000
+      - sectname:        __objc_ivar
+        segname:         __DATA
+        addr:            0x4998
+        size:            144
+        offset:          0x4998
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         080000000000000009000000000000000A00000000000000080000000000000009000000000000000A000000000000000B00000000000000080000000000000009000000000000000A000000000000000B00000000000000080000000000000009000000000000000A000000000000000B00000000000000080000000000000009000000000000000800000000000000
+      - sectname:        __data
+        segname:         __DATA
+        addr:            0x4A28
+        size:            392
+        offset:          0x4A28
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         0000000000000000461D000000000000000000000000000028410000000000000000000000000000000000000000000000000000000000000000000000000000600000000000000048410000000000000000000000000000000000000000000000000000000000003A1D00000000000050410000000000006841000000000000000000000000000000000000000000000000000000000000000000000000000060000000000000008841000000000000000000000000000000000000000000000000000000000000531D0000000000000000000000000000904100000000000000000000000000000000000000000000000000000000000000000000000000006000000000000000B041000000000000000000000000000000000000000000000000000000000000671D0000000000000000000000000000D84200000000000000000000000000000000000000000000000000000000000000000000000000006000000000000000F842000000000000000000000000000000000000000000000000000000000000
+      - sectname:        __objc_protolist
+        segname:         __DATA
+        addr:            0x4BB0
+        size:            32
+        offset:          0x4BB0
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x1000000B
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         284A000000000000884A000000000000E84A000000000000484B000000000000
+      - sectname:        __objc_classlist
+        segname:         __DATA
+        addr:            0x4BD0
+        size:            160
+        offset:          0x4BD0
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x10000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         8043000000000000D04300000000000020440000000000007044000000000000C044000000000000E8440000000000006045000000000000B04500000000000000460000000000005046000000000000A046000000000000F04600000000000040470000000000009047000000000000E04700000000000030480000000000008048000000000000D04800000000000020490000000000004849000000000000
+      - sectname:        __objc_catlist
+        segname:         __DATA
+        addr:            0x4C70
+        size:            32
+        offset:          0x4C70
+        align:           3
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x10000000
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         D03B000000000000903D000000000000E8400000000000001843000000000000
+      - sectname:        __objc_imageinfo
+        segname:         __DATA
+        addr:            0x4C90
+        size:            8
+        offset:          0x4C90
+        align:           0
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         '0000000040000000'
+      - sectname:        __common
+        segname:         __DATA
+        addr:            0x4C98
+        size:            16
+        offset:          0x0
+        align:           2
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x1
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __LINKEDIT
+    vmaddr:          20480
+    vmsize:          10272
+    fileoff:         20480
+    filesize:        10272
+    maxprot:         1
+    initprot:        1
+    nsects:          0
+    flags:           0
+  - cmd:             LC_DYLD_INFO_ONLY
+    cmdsize:         48
+    rebase_off:      20480
+    rebase_size:     320
+    bind_off:        20800
+    bind_size:       480
+    weak_bind_off:   0
+    weak_bind_size:  0
+    lazy_bind_off:   0
+    lazy_bind_size:  0
+    export_off:      21280
+    export_size:     896
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          22208
+    nsyms:           187
+    stroff:          25200
+    strsize:         5552
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       131
+    iextdefsym:      131
+    nextdefsym:      49
+    iundefsym:       180
+    nundefsym:       7
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+  - cmd:             LC_ID_DYLIB
+    cmdsize:         88
+    dylib:
+      name:            24
+      timestamp:       0
+      current_version: 66051
+      compatibility_version: 65536
+    Content:         '/System/Library/Frameworks/Simple.framework/Versions/A/Simple'
+    ZeroPadBytes:    3
+  - cmd:             LC_UUID
+    cmdsize:         24
+    uuid:            4C4C441D-5555-3144-A104-DD1AF4EF8FE7
+  - cmd:             LC_VERSION_MIN_MACOSX
+    cmdsize:         16
+    version:         658432
+    sdk:             983040
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         96
+    dylib:
+      name:            24
+      timestamp:       0
+      current_version: 197722368
+      compatibility_version: 19660800
+    Content:         '/System/Library/Frameworks/Foundation.framework/Versions/C/Foundation'
+    ZeroPadBytes:    3
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         56
+    dylib:
+      name:            24
+      timestamp:       0
+      current_version: 14942208
+      compatibility_version: 65536
+    Content:         '/usr/lib/libobjc.A.dylib'
+    ZeroPadBytes:    8
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         96
+    dylib:
+      name:            24
+      timestamp:       0
+      current_version: 91750400
+      compatibility_version: 65536
+    Content:         '/System/Library/Frameworks/CoreData.framework/Versions/A/CoreData'
+    ZeroPadBytes:    7
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         56
+    dylib:
+      name:            24
+      timestamp:       0
+      current_version: 88539136
+      compatibility_version: 65536
+    Content:         '/usr/lib/libSystem.B.dylib'
+    ZeroPadBytes:    6
+  - cmd:             LC_FUNCTION_STARTS
+    cmdsize:         16
+    dataoff:         22176
+    datasize:        32
+  - cmd:             LC_DATA_IN_CODE
+    cmdsize:         16
+    dataoff:         22208
+    datasize:        0
+LinkEditData:
+  RebaseOpcodes:
+    - Opcode:          REBASE_OPCODE_SET_TYPE_IMM
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB
+      Imm:             1
+      ExtraData:       [ 0x18 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x3, 0x40 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x40 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x40 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x40 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x4, 0x40 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             15
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             8
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x10 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x10 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x10 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             5
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             7
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             9
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             9
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             5
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x28 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             7
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             5
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x40 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             5
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             7
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             7
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x30 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x18 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x8 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x8 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             9
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x38 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             1
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x20 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             2
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x98 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x18 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_IMM_TIMES
+      Imm:             3
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             5
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x18 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x18 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES_SKIPPING_ULEB
+      Imm:             0
+      ExtraData:       [ 0x2, 0x8 ]
+    - Opcode:          REBASE_OPCODE_ADD_ADDR_IMM_SCALED
+      Imm:             4
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ADD_ADDR_ULEB
+      Imm:             0
+      ExtraData:       [ 0x18 ]
+    - Opcode:          REBASE_OPCODE_DO_REBASE_ULEB_TIMES
+      Imm:             0
+      ExtraData:       [ 0x1C ]
+    - Opcode:          REBASE_OPCODE_DONE
+      Imm:             0
+  BindOpcodes:
+    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
+      Imm:             0
+      Symbol:          _objc_ehtype_vtable
+    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
+      Imm:             1
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_DYLIB_ORDINAL_IMM
+      Imm:             2
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB
+      Imm:             1
+      ULEBExtraData:   [ 0x120 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_ADDEND_SLEB
+      Imm:             0
+      SLEBExtraData:   [ 16 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0xA0 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0xA0 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0xA0 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
+      Imm:             0
+      Symbol:          '_OBJC_CLASS_$_NSManagedObject'
+    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
+      Imm:             1
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_DYLIB_ORDINAL_IMM
+      Imm:             3
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0xA78 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_ADDEND_SLEB
+      Imm:             0
+      SLEBExtraData:   [ 0 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0xA48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
+      Imm:             0
+      Symbol:          '_OBJC_METACLASS_$_NSObject'
+    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
+      Imm:             1
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_DYLIB_ORDINAL_IMM
+      Imm:             2
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0xFFFFFFFFFFFFFB68 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x90 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x40 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
+      Imm:             0
+      Symbol:          __objc_empty_cache
+    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
+      Imm:             1
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0xFFFFFFFFFFFFFA60 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x20 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
+      Imm:             0
+      Symbol:          '_OBJC_CLASS_$_NSObject'
+    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
+      Imm:             1
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0xFFFFFFFFFFFFFA00 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x98 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x98 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x98 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0x48 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM
+      Imm:             0
+      Symbol:          '_OBJC_METACLASS_$_NSManagedObject'
+    - Opcode:          BIND_OPCODE_SET_TYPE_IMM
+      Imm:             1
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_SET_DYLIB_ORDINAL_IMM
+      Imm:             3
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_ADD_ADDR_ULEB
+      Imm:             0
+      ULEBExtraData:   [ 0xFFFFFFFFFFFFFE90 ]
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DO_BIND
+      Imm:             0
+      Symbol:          ''
+    - Opcode:          BIND_OPCODE_DONE
+      Imm:             0
+      Symbol:          ''
+  ExportTrie:
+    TerminalSize:    0
+    NodeOffset:      0
+    Name:            ''
+    Flags:           0x0
+    Address:         0x0
+    Other:           0x0
+    ImportName:      ''
+    Children:
+      - TerminalSize:    0
+        NodeOffset:      5
+        Name:            _
+        Flags:           0x0
+        Address:         0x0
+        Other:           0x0
+        ImportName:      ''
+        Children:
+          - TerminalSize:    0
+            NodeOffset:      41
+            Name:            p
+            Flags:           0x0
+            Address:         0x0
+            Other:           0x0
+            ImportName:      ''
+            Children:
+              - TerminalSize:    4
+                NodeOffset:      86
+                Name:            rivateGlobalVariable
+                Flags:           0x0
+                Address:         0x4CA0
+                Other:           0x0
+                ImportName:      ''
+              - TerminalSize:    4
+                NodeOffset:      92
+                Name:            ublicGlobalVariable
+                Flags:           0x0
+                Address:         0x4CA4
+                Other:           0x0
+                ImportName:      ''
+          - TerminalSize:    0
+            NodeOffset:      98
+            Name:            extraGlobalAPI
+            Flags:           0x0
+            Address:         0x0
+            Other:           0x0
+            ImportName:      ''
+            Children:
+              - TerminalSize:    4
+                NodeOffset:      106
+                Name:            '1'
+                Flags:           0x0
+                Address:         0x4C98
+                Other:           0x0
+                ImportName:      ''
+              - TerminalSize:    4
+                NodeOffset:      112
+                Name:            '2'
+                Flags:           0x0
+                Address:         0x4C9C
+                Other:           0x0
+                ImportName:      ''
+          - TerminalSize:    0
+            NodeOffset:      118
+            Name:            weakP
+            Flags:           0x0
+            Address:         0x0
+            Other:           0x0
+            ImportName:      ''
+            Children:
+              - TerminalSize:    4
+                NodeOffset:      165
+                Name:            rivateGlobalVariable
+                Flags:           0x4
+                Address:         0x4BAC
+                Other:           0x0
+                ImportName:      ''
+              - TerminalSize:    4
+                NodeOffset:      171
+                Name:            ublicGlobalVariable
+                Flags:           0x4
+                Address:         0x4BA8
+                Other:           0x0
+                ImportName:      ''
+          - TerminalSize:    0
+            NodeOffset:      177
+            Name:            OBJC_
+            Flags:           0x0
+            Address:         0x0
+            Other:           0x0
+            ImportName:      ''
+            Children:
+              - TerminalSize:    0
+                NodeOffset:      232
+                Name:            'IVAR_$_Basic'
+                Flags:           0x0
+                Address:         0x0
+                Other:           0x0
+                ImportName:      ''
+                Children:
+                  - TerminalSize:    4
+                    NodeOffset:      248
+                    Name:            6.ivar1
+                    Flags:           0x0
+                    Address:         0x4A10
+                    Other:           0x0
+                    ImportName:      ''
+                  - TerminalSize:    0
+                    NodeOffset:      254
+                    Name:            '4'
+                    Flags:           0x0
+                    Address:         0x0
+                    Other:           0x0
+                    ImportName:      ''
+                    Children:
+                      - TerminalSize:    0
+                        NodeOffset:      274
+                        Name:            .ivar
+                        Flags:           0x0
+                        Address:         0x0
+                        Other:           0x0
+                        ImportName:      ''
+                        Children:
+                          - TerminalSize:    4
+                            NodeOffset:      284
+                            Name:            '2'
+                            Flags:           0x0
+                            Address:         0x49B8
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      290
+                            Name:            '1'
+                            Flags:           0x0
+                            Address:         0x49B0
+                            Other:           0x0
+                            ImportName:      ''
+                      - TerminalSize:    0
+                        NodeOffset:      296
+                        Name:            _2.ivar
+                        Flags:           0x0
+                        Address:         0x0
+                        Other:           0x0
+                        ImportName:      ''
+                        Children:
+                          - TerminalSize:    4
+                            NodeOffset:      306
+                            Name:            '1'
+                            Flags:           0x0
+                            Address:         0x4A08
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      312
+                            Name:            '2'
+                            Flags:           0x0
+                            Address:         0x4A00
+                            Other:           0x0
+                            ImportName:      ''
+              - TerminalSize:    0
+                NodeOffset:      318
+                Name:            'METACLASS_$_'
+                Flags:           0x0
+                Address:         0x0
+                Other:           0x0
+                ImportName:      ''
+                Children:
+                  - TerminalSize:    4
+                    NodeOffset:      369
+                    Name:            FooClass
+                    Flags:           0x0
+                    Address:         0x4970
+                    Other:           0x0
+                    ImportName:      ''
+                  - TerminalSize:    4
+                    NodeOffset:      375
+                    Name:            ExternalManagedObject
+                    Flags:           0x0
+                    Address:         0x47B8
+                    Other:           0x0
+                    ImportName:      ''
+                  - TerminalSize:    0
+                    NodeOffset:      381
+                    Name:            S
+                    Flags:           0x0
+                    Address:         0x0
+                    Other:           0x0
+                    ImportName:      ''
+                    Children:
+                      - TerminalSize:    4
+                        NodeOffset:      401
+                        Name:            imple
+                        Flags:           0x0
+                        Address:         0x4358
+                        Other:           0x0
+                        ImportName:      ''
+                        Children:
+                          - TerminalSize:    0
+                            NodeOffset:      418
+                            Name:            Internal
+                            Flags:           0x0
+                            Address:         0x0
+                            Other:           0x0
+                            ImportName:      ''
+                            Children:
+                              - TerminalSize:    4
+                                NodeOffset:      432
+                                Name:            SPI
+                                Flags:           0x0
+                                Address:         0x4498
+                                Other:           0x0
+                                ImportName:      ''
+                              - TerminalSize:    4
+                                NodeOffset:      438
+                                Name:            API
+                                Flags:           0x0
+                                Address:         0x4448
+                                Other:           0x0
+                                ImportName:      ''
+                      - TerminalSize:    4
+                        NodeOffset:      444
+                        Name:            ubClass
+                        Flags:           0x0
+                        Address:         0x43F8
+                        Other:           0x0
+                        ImportName:      ''
+                  - TerminalSize:    0
+                    NodeOffset:      450
+                    Name:            Bas
+                    Flags:           0x0
+                    Address:         0x0
+                    Other:           0x0
+                    ImportName:      ''
+                    Children:
+                      - TerminalSize:    4
+                        NodeOffset:      461
+                        Name:            e
+                        Flags:           0x0
+                        Address:         0x43A8
+                        Other:           0x0
+                        ImportName:      ''
+                      - TerminalSize:    0
+                        NodeOffset:      467
+                        Name:            ic
+                        Flags:           0x0
+                        Address:         0x0
+                        Other:           0x0
+                        ImportName:      ''
+                        Children:
+                          - TerminalSize:    4
+                            NodeOffset:      501
+                            Name:            '2'
+                            Flags:           0x0
+                            Address:         0x4538
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      507
+                            Name:            '3'
+                            Flags:           0x0
+                            Address:         0x4588
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      513
+                            Name:            '5'
+                            Flags:           0x0
+                            Address:         0x46C8
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      519
+                            Name:            '4'
+                            Flags:           0x0
+                            Address:         0x45D8
+                            Other:           0x0
+                            ImportName:      ''
+                            Children:
+                              - TerminalSize:    4
+                                NodeOffset:      530
+                                Name:            _2
+                                Flags:           0x0
+                                Address:         0x4678
+                                Other:           0x0
+                                ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      536
+                            Name:            '9'
+                            Flags:           0x0
+                            Address:         0x48F8
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      542
+                            Name:            '8'
+                            Flags:           0x0
+                            Address:         0x4858
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      548
+                            Name:            '6'
+                            Flags:           0x0
+                            Address:         0x4718
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      554
+                            Name:            '1'
+                            Flags:           0x0
+                            Address:         0x4510
+                            Other:           0x0
+                            ImportName:      ''
+                  - TerminalSize:    4
+                    NodeOffset:      560
+                    Name:            A
+                    Flags:           0x0
+                    Address:         0x48A8
+                    Other:           0x0
+                    ImportName:      ''
+              - TerminalSize:    0
+                NodeOffset:      566
+                Name:            'EHTYPE_$_'
+                Flags:           0x0
+                Address:         0x0
+                Other:           0x0
+                ImportName:      ''
+                Children:
+                  - TerminalSize:    3
+                    NodeOffset:      579
+                    Name:            Base
+                    Flags:           0x0
+                    Address:         0x3120
+                    Other:           0x0
+                    ImportName:      ''
+                  - TerminalSize:    0
+                    NodeOffset:      584
+                    Name:            S
+                    Flags:           0x0
+                    Address:         0x0
+                    Other:           0x0
+                    ImportName:      ''
+                    Children:
+                      - TerminalSize:    3
+                        NodeOffset:      612
+                        Name:            ubClass
+                        Flags:           0x0
+                        Address:         0x31C8
+                        Other:           0x0
+                        ImportName:      ''
+                      - TerminalSize:    0
+                        NodeOffset:      617
+                        Name:            impleInternal
+                        Flags:           0x0
+                        Address:         0x0
+                        Other:           0x0
+                        ImportName:      ''
+                        Children:
+                          - TerminalSize:    3
+                            NodeOffset:      631
+                            Name:            SPI
+                            Flags:           0x0
+                            Address:         0x3318
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    3
+                            NodeOffset:      636
+                            Name:            API
+                            Flags:           0x0
+                            Address:         0x3270
+                            Other:           0x0
+                            ImportName:      ''
+              - TerminalSize:    0
+                NodeOffset:      641
+                Name:            'CLASS_$_'
+                Flags:           0x0
+                Address:         0x0
+                Other:           0x0
+                ImportName:      ''
+                Children:
+                  - TerminalSize:    4
+                    NodeOffset:      692
+                    Name:            A
+                    Flags:           0x0
+                    Address:         0x48D0
+                    Other:           0x0
+                    ImportName:      ''
+                  - TerminalSize:    4
+                    NodeOffset:      698
+                    Name:            ExternalManagedObject
+                    Flags:           0x0
+                    Address:         0x47E0
+                    Other:           0x0
+                    ImportName:      ''
+                  - TerminalSize:    4
+                    NodeOffset:      704
+                    Name:            FooClass
+                    Flags:           0x0
+                    Address:         0x4948
+                    Other:           0x0
+                    ImportName:      ''
+                  - TerminalSize:    0
+                    NodeOffset:      710
+                    Name:            S
+                    Flags:           0x0
+                    Address:         0x0
+                    Other:           0x0
+                    ImportName:      ''
+                    Children:
+                      - TerminalSize:    4
+                        NodeOffset:      730
+                        Name:            ubClass
+                        Flags:           0x0
+                        Address:         0x4420
+                        Other:           0x0
+                        ImportName:      ''
+                      - TerminalSize:    4
+                        NodeOffset:      736
+                        Name:            imple
+                        Flags:           0x0
+                        Address:         0x4380
+                        Other:           0x0
+                        ImportName:      ''
+                        Children:
+                          - TerminalSize:    0
+                            NodeOffset:      753
+                            Name:            Internal
+                            Flags:           0x0
+                            Address:         0x0
+                            Other:           0x0
+                            ImportName:      ''
+                            Children:
+                              - TerminalSize:    4
+                                NodeOffset:      767
+                                Name:            API
+                                Flags:           0x0
+                                Address:         0x4470
+                                Other:           0x0
+                                ImportName:      ''
+                              - TerminalSize:    4
+                                NodeOffset:      773
+                                Name:            SPI
+                                Flags:           0x0
+                                Address:         0x44C0
+                                Other:           0x0
+                                ImportName:      ''
+                  - TerminalSize:    0
+                    NodeOffset:      779
+                    Name:            Bas
+                    Flags:           0x0
+                    Address:         0x0
+                    Other:           0x0
+                    ImportName:      ''
+                    Children:
+                      - TerminalSize:    4
+                        NodeOffset:      790
+                        Name:            e
+                        Flags:           0x0
+                        Address:         0x43D0
+                        Other:           0x0
+                        ImportName:      ''
+                      - TerminalSize:    0
+                        NodeOffset:      796
+                        Name:            ic
+                        Flags:           0x0
+                        Address:         0x0
+                        Other:           0x0
+                        ImportName:      ''
+                        Children:
+                          - TerminalSize:    4
+                            NodeOffset:      830
+                            Name:            '1'
+                            Flags:           0x0
+                            Address:         0x44E8
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      836
+                            Name:            '3'
+                            Flags:           0x0
+                            Address:         0x45B0
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      842
+                            Name:            '4'
+                            Flags:           0x0
+                            Address:         0x4600
+                            Other:           0x0
+                            ImportName:      ''
+                            Children:
+                              - TerminalSize:    4
+                                NodeOffset:      853
+                                Name:            _2
+                                Flags:           0x0
+                                Address:         0x46A0
+                                Other:           0x0
+                                ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      859
+                            Name:            '2'
+                            Flags:           0x0
+                            Address:         0x4560
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      865
+                            Name:            '8'
+                            Flags:           0x0
+                            Address:         0x4880
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      871
+                            Name:            '9'
+                            Flags:           0x0
+                            Address:         0x4920
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      877
+                            Name:            '6'
+                            Flags:           0x0
+                            Address:         0x4740
+                            Other:           0x0
+                            ImportName:      ''
+                          - TerminalSize:    4
+                            NodeOffset:      883
+                            Name:            '5'
+                            Flags:           0x0
+                            Address:         0x46F0
+                            Other:           0x0
+                            ImportName:      ''
+  NameList:
+    - n_strx:          2
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7104
+    - n_strx:          22
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7114
+    - n_strx:          46
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7123
+    - n_strx:          66
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7133
+    - n_strx:          88
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7143
+    - n_strx:          112
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7152
+    - n_strx:          135
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7158
+    - n_strx:          162
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7164
+    - n_strx:          204
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7170
+    - n_strx:          224
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7180
+    - n_strx:          248
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7189
+    - n_strx:          273
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7200
+    - n_strx:          302
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7206
+    - n_strx:          347
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7212
+    - n_strx:          395
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7218
+    - n_strx:          424
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7224
+    - n_strx:          466
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7232
+    - n_strx:          488
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7238
+    - n_strx:          510
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7244
+    - n_strx:          523
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7250
+    - n_strx:          543
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7260
+    - n_strx:          566
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7266
+    - n_strx:          593
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7272
+    - n_strx:          615
+      n_type:          0xE
+      n_sect:          1
+      n_desc:          0
+      n_value:         7278
+    - n_strx:          658
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         12288
+    - n_strx:          687
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         12360
+    - n_strx:          712
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         12432
+    - n_strx:          739
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         12504
+    - n_strx:          762
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         12600
+    - n_strx:          793
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         12672
+    - n_strx:          820
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         12768
+    - n_strx:          860
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         12840
+    - n_strx:          896
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         12936
+    - n_strx:          936
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13008
+    - n_strx:          972
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13176
+    - n_strx:          997
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13104
+    - n_strx:          1026
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13248
+    - n_strx:          1055
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13320
+    - n_strx:          1080
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13392
+    - n_strx:          1109
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13464
+    - n_strx:          1142
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13592
+    - n_strx:          1177
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13696
+    - n_strx:          1203
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13768
+    - n_strx:          1228
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13840
+    - n_strx:          1257
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         13912
+    - n_strx:          1292
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14048
+    - n_strx:          1317
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14120
+    - n_strx:          1348
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14192
+    - n_strx:          1385
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14328
+    - n_strx:          1412
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14400
+    - n_strx:          1443
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14472
+    - n_strx:          1480
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14608
+    - n_strx:          1507
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14680
+    - n_strx:          1537
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14712
+    - n_strx:          1566
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14784
+    - n_strx:          1599
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14816
+    - n_strx:          1624
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14888
+    - n_strx:          1653
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         14960
+    - n_strx:          1686
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15040
+    - n_strx:          1721
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15112
+    - n_strx:          1747
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15136
+    - n_strx:          1772
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15208
+    - n_strx:          1820
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15288
+    - n_strx:          1852
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15312
+    - n_strx:          1883
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15376
+    - n_strx:          1912
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15448
+    - n_strx:          1945
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15480
+    - n_strx:          1970
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15552
+    - n_strx:          2014
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15624
+    - n_strx:          2062
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15656
+    - n_strx:          2102
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15728
+    - n_strx:          2162
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15760
+    - n_strx:          2205
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15824
+    - n_strx:          2239
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15896
+    - n_strx:          2269
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         15968
+    - n_strx:          2299
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16000
+    - n_strx:          2328
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16072
+    - n_strx:          2361
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16104
+    - n_strx:          2386
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16176
+    - n_strx:          2410
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16248
+    - n_strx:          2438
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16280
+    - n_strx:          2458
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16352
+    - n_strx:          2487
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16424
+    - n_strx:          2520
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16456
+    - n_strx:          2555
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16496
+    - n_strx:          2581
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16520
+    - n_strx:          2606
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16592
+    - n_strx:          2645
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16616
+    - n_strx:          2683
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         17008
+    - n_strx:          2710
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16856
+    - n_strx:          2741
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16680
+    - n_strx:          2789
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16712
+    - n_strx:          2833
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16720
+    - n_strx:          2868
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16744
+    - n_strx:          2915
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16776
+    - n_strx:          2958
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16784
+    - n_strx:          3005
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16816
+    - n_strx:          3048
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16824
+    - n_strx:          3082
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         16928
+    - n_strx:          3117
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         17080
+    - n_strx:          3171
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         17112
+    - n_strx:          3222
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         17144
+    - n_strx:          3269
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         17152
+    - n_strx:          3316
+      n_type:          0xE
+      n_sect:          6
+      n_desc:          0
+      n_value:         17176
+    - n_strx:          4000
+      n_type:          0x1E
+      n_sect:          7
+      n_desc:          0
+      n_value:         17960
+    - n_strx:          4027
+      n_type:          0x1E
+      n_sect:          7
+      n_desc:          0
+      n_value:         18000
+    - n_strx:          4192
+      n_type:          0x1E
+      n_sect:          7
+      n_desc:          0
+      n_value:         18280
+    - n_strx:          4217
+      n_type:          0x1E
+      n_sect:          7
+      n_desc:          0
+      n_value:         18320
+    - n_strx:          4314
+      n_type:          0x1E
+      n_sect:          7
+      n_desc:          0
+      n_value:         18440
+    - n_strx:          4344
+      n_type:          0x1E
+      n_sect:          7
+      n_desc:          0
+      n_value:         18480
+    - n_strx:          4548
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18840
+    - n_strx:          4578
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18848
+    - n_strx:          4608
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18856
+    - n_strx:          4690
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18880
+    - n_strx:          4716
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18888
+    - n_strx:          4742
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18896
+    - n_strx:          4770
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18904
+    - n_strx:          4798
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18912
+    - n_strx:          4826
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18920
+    - n_strx:          4854
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18928
+    - n_strx:          4882
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18936
+    - n_strx:          4992
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18968
+    - n_strx:          5022
+      n_type:          0x1E
+      n_sect:          8
+      n_desc:          0
+      n_value:         18976
+    - n_strx:          5053
+      n_type:          0x1E
+      n_sect:          9
+      n_desc:          0
+      n_value:         18984
+    - n_strx:          5084
+      n_type:          0x1E
+      n_sect:          9
+      n_desc:          0
+      n_value:         19080
+    - n_strx:          5114
+      n_type:          0x1E
+      n_sect:          9
+      n_desc:          0
+      n_value:         19176
+    - n_strx:          5144
+      n_type:          0x1E
+      n_sect:          9
+      n_desc:          0
+      n_value:         19272
+    - n_strx:          5231
+      n_type:          0x1E
+      n_sect:          10
+      n_desc:          0
+      n_value:         19376
+    - n_strx:          5268
+      n_type:          0x1E
+      n_sect:          10
+      n_desc:          0
+      n_value:         19384
+    - n_strx:          5304
+      n_type:          0x1E
+      n_sect:          10
+      n_desc:          0
+      n_value:         19392
+    - n_strx:          5340
+      n_type:          0x1E
+      n_sect:          10
+      n_desc:          0
+      n_value:         19400
+    - n_strx:          3353
+      n_type:          0xF
+      n_sect:          14
+      n_desc:          0
+      n_value:         19608
+    - n_strx:          3370
+      n_type:          0xF
+      n_sect:          14
+      n_desc:          0
+      n_value:         19612
+    - n_strx:          3387
+      n_type:          0xF
+      n_sect:          14
+      n_desc:          0
+      n_value:         19616
+    - n_strx:          3410
+      n_type:          0xF
+      n_sect:          14
+      n_desc:          0
+      n_value:         19620
+    - n_strx:          3432
+      n_type:          0xF
+      n_sect:          6
+      n_desc:          0
+      n_value:         12576
+    - n_strx:          3452
+      n_type:          0xF
+      n_sect:          6
+      n_desc:          0
+      n_value:         12744
+    - n_strx:          3476
+      n_type:          0xF
+      n_sect:          6
+      n_desc:          0
+      n_value:         12912
+    - n_strx:          3509
+      n_type:          0xF
+      n_sect:          6
+      n_desc:          0
+      n_value:         13080
+    - n_strx:          3542
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17240
+    - n_strx:          3567
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17280
+    - n_strx:          3588
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17320
+    - n_strx:          3611
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17360
+    - n_strx:          3630
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17400
+    - n_strx:          3657
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17440
+    - n_strx:          3680
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17480
+    - n_strx:          3716
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17520
+    - n_strx:          3748
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17560
+    - n_strx:          3784
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17600
+    - n_strx:          3816
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17640
+    - n_strx:          3837
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17680
+    - n_strx:          3862
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17720
+    - n_strx:          3887
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17760
+    - n_strx:          3908
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17800
+    - n_strx:          3933
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17840
+    - n_strx:          3954
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17880
+    - n_strx:          3979
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         17920
+    - n_strx:          4050
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18040
+    - n_strx:          4077
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18080
+    - n_strx:          4100
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18120
+    - n_strx:          4125
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18160
+    - n_strx:          4146
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18200
+    - n_strx:          4171
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18240
+    - n_strx:          4238
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18360
+    - n_strx:          4278
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18400
+    - n_strx:          4370
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18520
+    - n_strx:          4395
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18560
+    - n_strx:          4416
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18600
+    - n_strx:          4436
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18640
+    - n_strx:          4452
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18680
+    - n_strx:          4477
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18720
+    - n_strx:          4498
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18760
+    - n_strx:          4521
+      n_type:          0xF
+      n_sect:          7
+      n_desc:          0
+      n_value:         18800
+    - n_strx:          4638
+      n_type:          0xF
+      n_sect:          8
+      n_desc:          0
+      n_value:         18864
+    - n_strx:          4664
+      n_type:          0xF
+      n_sect:          8
+      n_desc:          0
+      n_value:         18872
+    - n_strx:          4910
+      n_type:          0xF
+      n_sect:          8
+      n_desc:          0
+      n_value:         18944
+    - n_strx:          4938
+      n_type:          0xF
+      n_sect:          8
+      n_desc:          0
+      n_value:         18952
+    - n_strx:          4966
+      n_type:          0xF
+      n_sect:          8
+      n_desc:          0
+      n_value:         18960
+    - n_strx:          5178
+      n_type:          0xF
+      n_sect:          9
+      n_desc:          128
+      n_value:         19368
+    - n_strx:          5204
+      n_type:          0xF
+      n_sect:          9
+      n_desc:          128
+      n_value:         19372
+    - n_strx:          5380
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          768
+      n_value:         0
+    - n_strx:          5410
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          512
+      n_value:         0
+    - n_strx:          5433
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          768
+      n_value:         0
+    - n_strx:          5467
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          512
+      n_value:         0
+    - n_strx:          5494
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          512
+      n_value:         0
+    - n_strx:          5513
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          512
+      n_value:         0
+    - n_strx:          5533
+      n_type:          0x1
+      n_sect:          0
+      n_desc:          1024
+      n_value:         0
+  StringTable:
+    - ' '
+    - '-[Basic3 property1]'
+    - '-[Basic3 setProperty1:]'
+    - '-[Basic3 property2]'
+    - '-[Basic3 isProperty3]'
+    - '-[Basic3 setProperty3:]'
+    - '+[Basic5 aClassMethod]'
+    - '-[Basic5 anInstanceMethod]'
+    - '-[Basic6 anInstanceMethodFromAnExtension]'
+    - '-[Basic6 property1]'
+    - '-[Basic6 setProperty1:]'
+    - '-[Basic6(Foo) property2]'
+    - '-[Basic6(Foo) setProperty2:]'
+    - '-[Basic6(Foo) anInstanceMethodFromACategory]'
+    - '-[Basic7 anInstanceMethodFromAnHiddenExtension]'
+    - '-[ExternalManagedObject foo]'
+    - '-[NSManagedObject(Simple) supportsSimple]'
+    - '+[Basic8 useSameName]'
+    - '-[Basic8 useSameName]'
+    - '-[A aMethod]'
+    - '-[Basic9 aProperty]'
+    - '-[FooClass baseMethod]'
+    - '-[FooClass protocolMethod]'
+    - '-[FooClass barMethod]'
+    - '-[FooClass(Private) privateProcotolMethod]'
+    - '__OBJC_METACLASS_RO_$_Simple'
+    - '__OBJC_CLASS_RO_$_Simple'
+    - '__OBJC_METACLASS_RO_$_Base'
+    - '__OBJC_CLASS_RO_$_Base'
+    - '__OBJC_METACLASS_RO_$_SubClass'
+    - '__OBJC_CLASS_RO_$_SubClass'
+    - '__OBJC_METACLASS_RO_$_SimpleInternalAPI'
+    - '__OBJC_CLASS_RO_$_SimpleInternalAPI'
+    - '__OBJC_METACLASS_RO_$_SimpleInternalSPI'
+    - '__OBJC_CLASS_RO_$_SimpleInternalSPI'
+    - '__OBJC_CLASS_RO_$_Basic1'
+    - '__OBJC_METACLASS_RO_$_Basic1'
+    - '__OBJC_METACLASS_RO_$_Basic2'
+    - '__OBJC_CLASS_RO_$_Basic2'
+    - '__OBJC_METACLASS_RO_$_Basic3'
+    - '__OBJC_$_INSTANCE_METHODS_Basic3'
+    - '__OBJC_$_INSTANCE_VARIABLES_Basic3'
+    - '__OBJC_$_PROP_LIST_Basic3'
+    - '__OBJC_CLASS_RO_$_Basic3'
+    - '__OBJC_METACLASS_RO_$_Basic4'
+    - '__OBJC_$_INSTANCE_VARIABLES_Basic4'
+    - '__OBJC_CLASS_RO_$_Basic4'
+    - '__OBJC_METACLASS_RO_$_Basic4_1'
+    - '__OBJC_$_INSTANCE_VARIABLES_Basic4_1'
+    - '__OBJC_CLASS_RO_$_Basic4_1'
+    - '__OBJC_METACLASS_RO_$_Basic4_2'
+    - '__OBJC_$_INSTANCE_VARIABLES_Basic4_2'
+    - '__OBJC_CLASS_RO_$_Basic4_2'
+    - '__OBJC_$_CLASS_METHODS_Basic5'
+    - '__OBJC_METACLASS_RO_$_Basic5'
+    - '__OBJC_$_INSTANCE_METHODS_Basic5'
+    - '__OBJC_CLASS_RO_$_Basic5'
+    - '__OBJC_METACLASS_RO_$_Basic6'
+    - '__OBJC_$_INSTANCE_METHODS_Basic6'
+    - '__OBJC_$_INSTANCE_VARIABLES_Basic6'
+    - '__OBJC_$_PROP_LIST_Basic6'
+    - '__OBJC_CLASS_RO_$_Basic6'
+    - '__OBJC_$_CATEGORY_INSTANCE_METHODS_Basic6_$_Foo'
+    - '__OBJC_$_PROP_LIST_Basic6_$_Foo'
+    - '__OBJC_$_CATEGORY_Basic6_$_Foo'
+    - '__OBJC_METACLASS_RO_$_Basic7'
+    - '__OBJC_$_INSTANCE_METHODS_Basic7'
+    - '__OBJC_CLASS_RO_$_Basic7'
+    - '__OBJC_METACLASS_RO_$_ExternalManagedObject'
+    - '__OBJC_$_INSTANCE_METHODS_ExternalManagedObject'
+    - '__OBJC_CLASS_RO_$_ExternalManagedObject'
+    - '__OBJC_$_CATEGORY_INSTANCE_METHODS_NSManagedObject_$_Simple'
+    - '__OBJC_$_CATEGORY_NSManagedObject_$_Simple'
+    - '__OBJC_METACLASS_RO_$_HiddenClass'
+    - '__OBJC_CLASS_RO_$_HiddenClass'
+    - '__OBJC_$_CLASS_METHODS_Basic8'
+    - '__OBJC_METACLASS_RO_$_Basic8'
+    - '__OBJC_$_INSTANCE_METHODS_Basic8'
+    - '__OBJC_CLASS_RO_$_Basic8'
+    - '__OBJC_METACLASS_RO_$_A'
+    - '__OBJC_$_INSTANCE_METHODS_A'
+    - '__OBJC_CLASS_RO_$_A'
+    - '__OBJC_METACLASS_RO_$_Basic9'
+    - '__OBJC_$_INSTANCE_METHODS_Basic9'
+    - '__OBJC_$_INSTANCE_VARIABLES_Basic9'
+    - '__OBJC_$_PROP_LIST_Basic9'
+    - '__OBJC_CLASS_RO_$_Basic9'
+    - '__OBJC_$_PROP_LIST_Basic9_$_deprecated'
+    - '__OBJC_$_CATEGORY_Basic9_$_deprecated'
+    - '__OBJC_CLASS_RO_$_FooClass'
+    - '__OBJC_METACLASS_RO_$_FooClass'
+    - '__OBJC_$_PROTOCOL_INSTANCE_METHODS_BaseProtocol'
+    - '__OBJC_$_PROTOCOL_METHOD_TYPES_BaseProtocol'
+    - '__OBJC_$_PROTOCOL_REFS_FooProtocol'
+    - '__OBJC_$_PROTOCOL_INSTANCE_METHODS_FooProtocol'
+    - '__OBJC_$_PROTOCOL_METHOD_TYPES_FooProtocol'
+    - '__OBJC_$_PROTOCOL_INSTANCE_METHODS_BarProtocol'
+    - '__OBJC_$_PROTOCOL_METHOD_TYPES_BarProtocol'
+    - '__OBJC_CLASS_PROTOCOLS_$_FooClass'
+    - '__OBJC_$_INSTANCE_METHODS_FooClass'
+    - '__OBJC_$_CATEGORY_INSTANCE_METHODS_FooClass_$_Private'
+    - '__OBJC_$_PROTOCOL_INSTANCE_METHODS_PrivateProtocol'
+    - '__OBJC_$_PROTOCOL_METHOD_TYPES_PrivateProtocol'
+    - '__OBJC_CATEGORY_PROTOCOLS_$_FooClass_$_Private'
+    - '__OBJC_$_CATEGORY_FooClass_$_Private'
+    - _extraGlobalAPI1
+    - _extraGlobalAPI2
+    - _privateGlobalVariable
+    - _publicGlobalVariable
+    - '_OBJC_EHTYPE_$_Base'
+    - '_OBJC_EHTYPE_$_SubClass'
+    - '_OBJC_EHTYPE_$_SimpleInternalAPI'
+    - '_OBJC_EHTYPE_$_SimpleInternalSPI'
+    - '_OBJC_METACLASS_$_Simple'
+    - '_OBJC_CLASS_$_Simple'
+    - '_OBJC_METACLASS_$_Base'
+    - '_OBJC_CLASS_$_Base'
+    - '_OBJC_METACLASS_$_SubClass'
+    - '_OBJC_CLASS_$_SubClass'
+    - '_OBJC_METACLASS_$_SimpleInternalAPI'
+    - '_OBJC_CLASS_$_SimpleInternalAPI'
+    - '_OBJC_METACLASS_$_SimpleInternalSPI'
+    - '_OBJC_CLASS_$_SimpleInternalSPI'
+    - '_OBJC_CLASS_$_Basic1'
+    - '_OBJC_METACLASS_$_Basic1'
+    - '_OBJC_METACLASS_$_Basic2'
+    - '_OBJC_CLASS_$_Basic2'
+    - '_OBJC_METACLASS_$_Basic3'
+    - '_OBJC_CLASS_$_Basic3'
+    - '_OBJC_METACLASS_$_Basic4'
+    - '_OBJC_CLASS_$_Basic4'
+    - '_OBJC_METACLASS_$_Basic4_1'
+    - '_OBJC_CLASS_$_Basic4_1'
+    - '_OBJC_METACLASS_$_Basic4_2'
+    - '_OBJC_CLASS_$_Basic4_2'
+    - '_OBJC_METACLASS_$_Basic5'
+    - '_OBJC_CLASS_$_Basic5'
+    - '_OBJC_METACLASS_$_Basic6'
+    - '_OBJC_CLASS_$_Basic6'
+    - '_OBJC_METACLASS_$_Basic7'
+    - '_OBJC_CLASS_$_Basic7'
+    - '_OBJC_METACLASS_$_ExternalManagedObject'
+    - '_OBJC_CLASS_$_ExternalManagedObject'
+    - '_OBJC_METACLASS_$_HiddenClass'
+    - '_OBJC_CLASS_$_HiddenClass'
+    - '_OBJC_METACLASS_$_Basic8'
+    - '_OBJC_CLASS_$_Basic8'
+    - '_OBJC_METACLASS_$_A'
+    - '_OBJC_CLASS_$_A'
+    - '_OBJC_METACLASS_$_Basic9'
+    - '_OBJC_CLASS_$_Basic9'
+    - '_OBJC_CLASS_$_FooClass'
+    - '_OBJC_METACLASS_$_FooClass'
+    - '_OBJC_IVAR_$_Basic3.property1'
+    - '_OBJC_IVAR_$_Basic3.property2'
+    - '_OBJC_IVAR_$_Basic3.property3'
+    - '_OBJC_IVAR_$_Basic4.ivar1'
+    - '_OBJC_IVAR_$_Basic4.ivar2'
+    - '_OBJC_IVAR_$_Basic4.ivar3'
+    - '_OBJC_IVAR_$_Basic4.ivar4'
+    - '_OBJC_IVAR_$_Basic4_1.ivar1'
+    - '_OBJC_IVAR_$_Basic4_1.ivar2'
+    - '_OBJC_IVAR_$_Basic4_1.ivar3'
+    - '_OBJC_IVAR_$_Basic4_1.ivar4'
+    - '_OBJC_IVAR_$_Basic4_2.ivar4'
+    - '_OBJC_IVAR_$_Basic4_2.ivar3'
+    - '_OBJC_IVAR_$_Basic4_2.ivar2'
+    - '_OBJC_IVAR_$_Basic4_2.ivar1'
+    - '_OBJC_IVAR_$_Basic6.ivar1'
+    - '_OBJC_IVAR_$_Basic6.property1'
+    - '_OBJC_IVAR_$_Basic9._aProperty'
+    - '__OBJC_PROTOCOL_$_BaseProtocol'
+    - '__OBJC_PROTOCOL_$_FooProtocol'
+    - '__OBJC_PROTOCOL_$_BarProtocol'
+    - '__OBJC_PROTOCOL_$_PrivateProtocol'
+    - _weakPublicGlobalVariable
+    - _weakPrivateGlobalVariable
+    - '__OBJC_LABEL_PROTOCOL_$_BaseProtocol'
+    - '__OBJC_LABEL_PROTOCOL_$_FooProtocol'
+    - '__OBJC_LABEL_PROTOCOL_$_BarProtocol'
+    - '__OBJC_LABEL_PROTOCOL_$_PrivateProtocol'
+    - '_OBJC_CLASS_$_NSManagedObject'
+    - '_OBJC_CLASS_$_NSObject'
+    - '_OBJC_METACLASS_$_NSManagedObject'
+    - '_OBJC_METACLASS_$_NSObject'
+    - __objc_empty_cache
+    - _objc_ehtype_vtable
+    - dyld_stub_binder
+    - ''
+    - ''
+  FunctionStarts:  [ 0x1BC0, 0x1BCA, 0x1BD3, 0x1BDD, 0x1BE7, 0x1BF0, 0x1BF6, 
+                     0x1BFC, 0x1C02, 0x1C0C, 0x1C15, 0x1C20, 0x1C26, 0x1C2C, 
+                     0x1C32, 0x1C38, 0x1C40, 0x1C46, 0x1C4C, 0x1C52, 0x1C5C, 
+                     0x1C62, 0x1C68, 0x1C6E ]
+...
diff --git a/clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI.h b/clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI.h
new file mode 100644
index 000000000000..5dd416a0619c
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI.h
@@ -0,0 +1,3 @@
+#ifndef HAVE_SEEN_PROJECT_HEADER_FIRST
+#error "Project header was not included in the correct order!"
+#endif
diff --git a/clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI2.h b/clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI2.h
new file mode 100644
index 000000000000..9bbae52d7215
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/SimpleInternalAPI2.h
@@ -0,0 +1,7 @@
+#import <Foundation/Foundation.h>
+
+__attribute__((objc_exception))
+@interface SimpleInternalAPI : NSObject
+@end
+
+#define HAVE_SEEN_PROJECT_HEADER_FIRST 1
diff --git a/clang/test/InstallAPI/Inputs/Simple/SimpleInternalSPI.h b/clang/test/InstallAPI/Inputs/Simple/SimpleInternalSPI.h
new file mode 100644
index 000000000000..a816c01abeb0
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/Simple/SimpleInternalSPI.h
@@ -0,0 +1,5 @@
+#import <Foundation/Foundation.h>
+
+__attribute__((objc_exception))
+@interface SimpleInternalSPI : NSObject
+@end
diff --git a/clang/test/InstallAPI/extra-exclude-headers.test b/clang/test/InstallAPI/extra-exclude-headers.test
new file mode 100644
index 000000000000..663ca1a5d500
--- /dev/null
+++ b/clang/test/InstallAPI/extra-exclude-headers.test
@@ -0,0 +1,207 @@
+; RUN: rm -rf %t 
+; RUN: split-file %s %t
+; RUN: mkdir -p %t/System/Library/Frameworks 
+; RUN: cp -r %S/Inputs/Simple/Simple.framework %t/System/Library/Frameworks/
+; RUN: sed -e "s|DSTROOT|%/t|g" %t/inputs.json.in > %t/inputs.json
+; RUN: yaml2obj %S/Inputs/Simple/Simple.yaml -o %t/Simple 
+
+// Add exclude options.
+; RUN: clang-installapi -target x86_64-apple-macosx10.12 \
+; RUN: -install_name /System/Library/Frameworks/Simple.framework/Versions/A/Simple \
+; RUN: -current_version 1.2.3 -compatibility_version 1 \
+; RUN: -F%t/System/Library/Frameworks \
+; RUN: %t/inputs.json -o %t/Simple.tbd \
+; RUN: --verify-against=%t/Simple --verify-mode=ErrorsAndWarnings \
+; RUN: --exclude-public-header=**/SimpleAPI.h \
+; RUN: --exclude-private-header=**/SimplePrivateSPI.h 2>&1 | FileCheck -check-prefix=WARNINGS %s
+; RUN: llvm-readtapi -compare %t/Simple.tbd %t/expected-excluded.tbd
+
+// Add extra options.
+; RUN: clang-installapi -target x86_64-apple-macosx10.12 \
+; RUN: -install_name /System/Library/Frameworks/Simple.framework/Versions/A/Simple \
+; RUN: -current_version 1.2.3 -compatibility_version 1 \
+; RUN: -F%t/System/Library/Frameworks \
+; RUN: %t/inputs.json -o %t/Simple.tbd \
+; RUN: --verify-against=%t/Simple --verify-mode=Pedantic \
+; RUN: --extra-project-header=%S/Inputs/Simple/SimpleInternalAPI2.h \
+; RUN: --extra-project-header=%S/Inputs/Simple/SimpleInternalAPI.h \
+; RUN: --extra-public-header=%S/Inputs/Simple/Extra \
+; RUN: --extra-private-header=%S/Inputs/Simple/SimpleInternalSPI.h \
+; RUN: --exclude-public-header=**/SimpleAPI.h \
+; RUN: --exclude-private-header=**/SimplePrivateSPI.h 2>&1 | FileCheck -check-prefix=PEDANTIC -allow-empty %s
+; RUN: llvm-readtapi -compare %t/Simple.tbd %t/expected-extra.tbd
+
+// Check fatal missing file input.
+; RUN: not clang-installapi -target x86_64-apple-macosx10.12 \
+; RUN: -install_name /System/Library/Frameworks/Simple.framework/Versions/A/Simple \
+; RUN: -current_version 1.2.3 -compatibility_version 1 \
+; RUN: -F%t/System/Library/Frameworks \
+; RUN: %t/inputs.json -o %t/Simple.tbd \
+; RUN: --extra-public-header=%S/Inputs/Simple/NoSuchFile.h 2>&1 | FileCheck -allow-empty -check-prefix=NOPUBLIC %s
+
+; WARNINGS: warning: no declaration was found for exported symbol '_extraGlobalAPI1' in dynamic library
+; WARNINGS: warning: no declaration was found for exported symbol '_extraGlobalAPI2' in dynamic library
+; WARNINGS: warning: no declaration was found for exported symbol '(ObjC Class) SimpleInternalSPI' in dynamic library
+; WARNINGS: warning: no declaration was found for exported symbol '(ObjC Class) SimpleInternalAPI' in dynamic library
+
+; PEDANTIC-NOT: error
+; PEDANTIC: warning: cannot find protocol definition for 'ForwardProcotol'
+
+; NOPUBLIC: error: no such public header file:
+
+;--- expected-excluded.tbd
+{
+  "main_library": {
+    "current_versions": [
+      {
+        "version": "1.2.3"
+      }
+    ],
+    "exported_symbols": [
+      {
+        "data": {
+          "global": [
+            "_publicGlobalVariable",
+            "_privateGlobalVariable"
+          ],
+          "objc_class": [
+            "ExternalManagedObject", "Basic6",
+            "Basic1", "Base", "Basic3",
+            "FooClass", "Simple",
+            "Basic4_2", "Basic5",
+            "Basic9","Basic8",
+            "Basic2", "Basic4", "A", "SubClass"
+          ],
+          "objc_eh_type": [
+            "SubClass", "Base"
+          ],
+          "objc_ivar": [
+            "Basic4.ivar2", "Basic4_2.ivar1", "Basic6.ivar1",
+            "Basic4.ivar1", "Basic4_2.ivar2"
+          ],
+          "weak": [
+            "_weakPrivateGlobalVariable", "_weakPublicGlobalVariable"
+          ]
+        }
+      }
+    ],
+    "flags": [
+      {
+        "attributes": ["not_app_extension_safe"]
+      }
+    ],
+    "install_names": [
+      {
+        "name": "/System/Library/Frameworks/Simple.framework/Versions/A/Simple"
+      }
+    ],
+    "target_info": [
+      {"min_deployment": "10.12", "target": "x86_64-macos"}
+    ]
+  },
+  "tapi_tbd_version": 5
+}
+
+;--- expected-extra.tbd
+{
+  "main_library": {
+    "current_versions": [
+      { "version": "1.2.3" }
+    ],
+    "exported_symbols": [
+      {
+        "data": {
+          "global": [
+            "_publicGlobalVariable", "_extraGlobalAPI2",
+            "_extraGlobalAPI1", "_privateGlobalVariable"
+          ],
+          "objc_class": [
+            "SubClass", "SimpleInternalSPI",
+            "Basic6", "Basic1", "Base",
+            "Basic3", "Simple", "Basic4_2",
+            "Basic5", "FooClass", "Basic9",
+            "Basic8", "Basic2", "Basic4",
+            "A", "SimpleInternalAPI",
+            "ExternalManagedObject"
+          ],
+          "objc_eh_type": [
+            "SubClass", "SimpleInternalAPI",
+            "Base", "SimpleInternalSPI"
+          ],
+          "objc_ivar": [
+            "Basic4.ivar2", "Basic4_2.ivar1",
+            "Basic6.ivar1", "Basic4.ivar1",
+            "Basic4_2.ivar2"
+          ],
+          "weak": [
+            "_weakPrivateGlobalVariable", "_weakPublicGlobalVariable"
+          ]
+        }
+      }
+    ],
+    "flags": [
+      {
+        "attributes": [ "not_app_extension_safe"]
+      }
+    ],
+    "install_names": [
+      { "name": "/System/Library/Frameworks/Simple.framework/Versions/A/Simple" }
+    ],
+    "target_info": [
+      { "min_deployment": "10.12", "target": "x86_64-macos" }
+    ]
+  },
+  "tapi_tbd_version": 5
+}
+
+;--- inputs.json.in
+{
+  "headers": [ 
+  {
+    "path" : "DSTROOT/System/Library/Frameworks/Simple.framework/Headers/Basic.h",
+    "type" : "public"
+  }, 
+  {
+    "path" : "DSTROOT/System/Library/Frameworks/Simple.framework/Headers/External.h",
+    "type" : "public"
+  }, 
+  {
+    "path" : "DSTROOT/System/Library/Frameworks/Simple.framework/Headers/Simple.h",
+    "type" : "public"
+  }, 
+  {
+    "path" : "DSTROOT/System/Library/Frameworks/Simple.framework/Headers/SimpleAPI.h",
+    "type" : "public"
+  }, 
+  {
+    "path" : "DSTROOT/System/Library/Frameworks/Simple.framework/PrivateHeaders/SimplePrivate.h",
+    "type" : "private"
+  }, 
+  {
+    "path" : "DSTROOT/System/Library/Frameworks/Simple.framework/PrivateHeaders/SimplePrivateSPI.h",
+    "type" : "private"
+  }
+  ],
+  "version": "3"
+}
+
+;--- System/Library/Frameworks/Foundation.framework/Headers/Foundation.h
+@interface NSObject 
+@end
+
+typedef unsigned char BOOL; 
+#ifndef NS_AVAILABLE
+#define NS_AVAILABLE(x,y) __attribute__((availability(macosx,introduced=x)))
+#endif 
+#ifndef NS_UNAVAILABLE
+#define NS_UNAVAILABLE  __attribute__((unavailable))
+#endif 
+#ifndef NS_DEPRECATED_MAC
+#define NS_DEPRECATED_MAC(x,y) __attribute__((availability(macosx,introduced=x,deprecated=y,message="" )));
+#endif 
+
+@interface NSManagedObject
+@end 
+
+@interface NSSet 
+@end 
diff --git a/clang/tools/clang-installapi/InstallAPIOpts.td b/clang/tools/clang-installapi/InstallAPIOpts.td
index 87f4c3327e84..ab9e1fe7f2f9 100644
--- a/clang/tools/clang-installapi/InstallAPIOpts.td
+++ b/clang/tools/clang-installapi/InstallAPIOpts.td
@@ -29,3 +29,35 @@ def verify_mode_EQ : Joined<["--"], "verify-mode=">,
   HelpText<"Specify the severity and extend of the validation. Valid modes are ErrorsOnly, ErrorsAndWarnings, and Pedantic.">;
 def demangle : Flag<["--", "-"], "demangle">,
   HelpText<"Demangle symbols when printing warnings and errors">;
+
+// Additional input options.
+def extra_project_header : Separate<["-"], "extra-project-header">,
+  MetaVarName<"<path>">,
+  HelpText<"Add additional project header location for parsing">;
+def extra_project_header_EQ : Joined<["--"], "extra-project-header=">,
+  Alias<extra_project_header>;
+def exclude_project_header : Separate<["-"], "exclude-project-header">,
+  MetaVarName<"<glob>">,
+  HelpText<"Exclude project header from parsing">;
+def exclude_project_header_EQ : Joined<["--"], "exclude-project-header=">,
+  Alias<exclude_project_header>;
+def extra_public_header : Separate<["-"], "extra-public-header">,
+  MetaVarName<"<path>">,
+  HelpText<"Add additional public header location for parsing">;
+def extra_public_header_EQ : Joined<["--"], "extra-public-header=">,
+  Alias<extra_public_header>;
+def extra_private_header : Separate<["-"], "extra-private-header">,
+  MetaVarName<"<path>">,
+  HelpText<"Add additional private header location for parsing">;
+def extra_private_header_EQ : Joined<["--"], "extra-private-header=">,
+  Alias<extra_private_header>;
+def exclude_public_header : Separate<["-"], "exclude-public-header">,
+  MetaVarName<"<glob>">,
+  HelpText<"Exclude public header from parsing">;
+def exclude_public_header_EQ : Joined<["--"], "exclude-public-header=">,
+  Alias<exclude_public_header>;
+def exclude_private_header : Separate<["-"], "exclude-private-header">,
+  MetaVarName<"<glob>">,
+  HelpText<"Exclude private header from parsing">;
+def exclude_private_header_EQ : Joined<["--"], "exclude-private-header=">,
+  Alias<exclude_private_header>;
diff --git a/clang/tools/clang-installapi/Options.cpp b/clang/tools/clang-installapi/Options.cpp
index b8696bb7896d..4f79c62724a6 100644
--- a/clang/tools/clang-installapi/Options.cpp
+++ b/clang/tools/clang-installapi/Options.cpp
@@ -10,6 +10,7 @@
 #include "clang/Driver/Driver.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/InstallAPI/FileList.h"
+#include "clang/InstallAPI/HeaderFile.h"
 #include "clang/InstallAPI/InstallAPIDiagnostic.h"
 #include "llvm/Support/Program.h"
 #include "llvm/TargetParser/Host.h"
@@ -181,6 +182,26 @@ bool Options::processFrontendOptions(InputArgList &Args) {
   return true;
 }
 
+bool Options::addFilePaths(InputArgList &Args, PathSeq &Headers,
+                           OptSpecifier ID) {
+  for (const StringRef Path : Args.getAllArgValues(ID)) {
+    if ((bool)FM->getDirectory(Path, /*CacheFailure=*/false)) {
+      auto InputHeadersOrErr = enumerateFiles(*FM, Path);
+      if (!InputHeadersOrErr) {
+        Diags->Report(diag::err_cannot_open_file)
+            << Path << toString(InputHeadersOrErr.takeError());
+        return false;
+      }
+      // Sort headers to ensure deterministic behavior.
+      sort(*InputHeadersOrErr);
+      for (std::string &H : *InputHeadersOrErr)
+        Headers.emplace_back(std::move(H));
+    } else
+      Headers.emplace_back(Path);
+  }
+  return true;
+}
+
 std::vector<const char *>
 Options::processAndFilterOutInstallAPIOptions(ArrayRef<const char *> Args) {
   std::unique_ptr<llvm::opt::OptTable> Table;
@@ -220,6 +241,35 @@ Options::processAndFilterOutInstallAPIOptions(ArrayRef<const char *> Args) {
   if (const Arg *A = ParsedArgs.getLastArg(OPT_verify_against))
     DriverOpts.DylibToVerify = A->getValue();
 
+  // Handle exclude & extra header directories or files.
+  auto handleAdditionalInputArgs = [&](PathSeq &Headers,
+                                       clang::installapi::ID OptID) {
+    if (ParsedArgs.hasArgNoClaim(OptID))
+      Headers.clear();
+    return addFilePaths(ParsedArgs, Headers, OptID);
+  };
+
+  if (!handleAdditionalInputArgs(DriverOpts.ExtraPublicHeaders,
+                                 OPT_extra_public_header))
+    return {};
+
+  if (!handleAdditionalInputArgs(DriverOpts.ExtraPrivateHeaders,
+                                 OPT_extra_private_header))
+    return {};
+  if (!handleAdditionalInputArgs(DriverOpts.ExtraProjectHeaders,
+                                 OPT_extra_project_header))
+    return {};
+
+  if (!handleAdditionalInputArgs(DriverOpts.ExcludePublicHeaders,
+                                 OPT_exclude_public_header))
+    return {};
+  if (!handleAdditionalInputArgs(DriverOpts.ExcludePrivateHeaders,
+                                 OPT_exclude_private_header))
+    return {};
+  if (!handleAdditionalInputArgs(DriverOpts.ExcludeProjectHeaders,
+                                 OPT_exclude_project_header))
+    return {};
+
   /// Any unclaimed arguments should be forwarded to the clang driver.
   std::vector<const char *> ClangDriverArgs(ParsedArgs.size());
   for (const Arg *A : ParsedArgs) {
@@ -302,6 +352,77 @@ InstallAPIContext Options::createContext() {
       return Ctx;
     }
   }
+  // After initial input has been processed, add any extra headers.
+  auto HandleExtraHeaders = [&](PathSeq &Headers, HeaderType Type) -> bool {
+    assert(Type != HeaderType::Unknown && "Missing header type.");
+    for (const StringRef Path : Headers) {
+      if (!FM->getOptionalFileRef(Path)) {
+        Diags->Report(diag::err_no_such_header_file)
+            << Path << (unsigned)Type - 1;
+        return false;
+      }
+      SmallString<PATH_MAX> FullPath(Path);
+      FM->makeAbsolutePath(FullPath);
+
+      auto IncludeName = createIncludeHeaderName(FullPath);
+      Ctx.InputHeaders.emplace_back(
+          FullPath, Type, IncludeName.has_value() ? *IncludeName : "");
+      Ctx.InputHeaders.back().setExtra();
+    }
+    return true;
+  };
+
+  if (!HandleExtraHeaders(DriverOpts.ExtraPublicHeaders, HeaderType::Public) ||
+      !HandleExtraHeaders(DriverOpts.ExtraPrivateHeaders,
+                          HeaderType::Private) ||
+      !HandleExtraHeaders(DriverOpts.ExtraProjectHeaders, HeaderType::Project))
+    return Ctx;
+
+  // After all headers have been added, consider excluded headers.
+  std::vector<std::unique_ptr<HeaderGlob>> ExcludedHeaderGlobs;
+  std::set<FileEntryRef> ExcludedHeaderFiles;
+  auto ParseGlobs = [&](const PathSeq &Paths, HeaderType Type) {
+    for (const StringRef Path : Paths) {
+      auto Glob = HeaderGlob::create(Path, Type);
+      if (Glob)
+        ExcludedHeaderGlobs.emplace_back(std::move(Glob.get()));
+      else {
+        consumeError(Glob.takeError());
+        if (auto File = FM->getFileRef(Path))
+          ExcludedHeaderFiles.emplace(*File);
+        else {
+          Diags->Report(diag::err_no_such_header_file)
+              << Path << (unsigned)Type;
+          return false;
+        }
+      }
+    }
+    return true;
+  };
+
+  if (!ParseGlobs(DriverOpts.ExcludePublicHeaders, HeaderType::Public) ||
+      !ParseGlobs(DriverOpts.ExcludePrivateHeaders, HeaderType::Private) ||
+      !ParseGlobs(DriverOpts.ExcludeProjectHeaders, HeaderType::Project))
+    return Ctx;
+
+  for (HeaderFile &Header : Ctx.InputHeaders) {
+    for (auto &Glob : ExcludedHeaderGlobs)
+      if (Glob->match(Header))
+        Header.setExcluded();
+  }
+  if (!ExcludedHeaderFiles.empty()) {
+    for (HeaderFile &Header : Ctx.InputHeaders) {
+      auto FileRef = FM->getFileRef(Header.getPath());
+      if (!FileRef)
+        continue;
+      if (ExcludedHeaderFiles.count(*FileRef))
+        Header.setExcluded();
+    }
+  }
+  // Report if glob was ignored.
+  for (const auto &Glob : ExcludedHeaderGlobs)
+    if (!Glob->didMatch())
+      Diags->Report(diag::warn_glob_did_not_match) << Glob->str();
 
   // Parse binary dylib and initialize verifier.
   if (DriverOpts.DylibToVerify.empty()) {
diff --git a/clang/tools/clang-installapi/Options.h b/clang/tools/clang-installapi/Options.h
index 2beeafc86bb0..c18309f69370 100644
--- a/clang/tools/clang-installapi/Options.h
+++ b/clang/tools/clang-installapi/Options.h
@@ -31,6 +31,24 @@ struct DriverOptions {
   /// \brief Path to input file lists (JSON).
   llvm::MachO::PathSeq FileLists;
 
+  /// \brief Paths of extra public headers.
+  PathSeq ExtraPublicHeaders;
+
+  /// \brief Paths of extra private headers.
+  PathSeq ExtraPrivateHeaders;
+
+  /// \brief Paths of extra project headers.
+  PathSeq ExtraProjectHeaders;
+
+  /// \brief List of excluded public headers.
+  PathSeq ExcludePublicHeaders;
+
+  /// \brief List of excluded private headers.
+  PathSeq ExcludePrivateHeaders;
+
+  /// \brief List of excluded project headers.
+  PathSeq ExcludeProjectHeaders;
+
   /// \brief Mappings of target triples & tapi targets to build for.
   std::map<llvm::MachO::Target, llvm::Triple> Targets;
 
@@ -103,6 +121,9 @@ public:
   std::vector<std::string> &getClangFrontendArgs() { return FrontendArgs; }
 
 private:
+  bool addFilePaths(llvm::opt::InputArgList &Args, PathSeq &Headers,
+                    llvm::opt::OptSpecifier ID);
+
   DiagnosticsEngine *Diags;
   FileManager *FM;
   std::vector<std::string> FrontendArgs;
diff --git a/llvm/include/llvm/TextAPI/Utils.h b/llvm/include/llvm/TextAPI/Utils.h
index 31d3c45f9e29..319e0abce25b 100644
--- a/llvm/include/llvm/TextAPI/Utils.h
+++ b/llvm/include/llvm/TextAPI/Utils.h
@@ -14,8 +14,10 @@
 #define LLVM_TEXTAPI_UTILS_H
 
 #include "llvm/ADT/Twine.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/Regex.h"
 
 #if !defined(PATH_MAX)
 #define PATH_MAX 1024
@@ -68,5 +70,10 @@ std::error_code make_relative(StringRef From, StringRef To,
 /// \param IsSymLink Whether path points to a symlink.
 bool isPrivateLibrary(StringRef Path, bool IsSymLink = false);
 
+/// Create a regex rule from provided glob string.
+/// \param Glob String that represents glob input.
+/// \return The equivalent regex rule.
+llvm::Expected<llvm::Regex> createRegexFromGlob(llvm::StringRef Glob);
+
 } // namespace llvm::MachO
 #endif // LLVM_TEXTAPI_UTILS_H
diff --git a/llvm/lib/TextAPI/Utils.cpp b/llvm/lib/TextAPI/Utils.cpp
index b5c3999a86ea..c54164504495 100644
--- a/llvm/lib/TextAPI/Utils.cpp
+++ b/llvm/lib/TextAPI/Utils.cpp
@@ -152,3 +152,49 @@ bool llvm::MachO::isPrivateLibrary(StringRef Path, bool IsSymLink) {
   }
   return false;
 }
+
+static StringLiteral RegexMetachars = "()^$|+.[]\\{}";
+
+llvm::Expected<Regex> llvm::MachO::createRegexFromGlob(StringRef Glob) {
+  SmallString<128> RegexString("^");
+  unsigned NumWildcards = 0;
+  for (unsigned i = 0; i < Glob.size(); ++i) {
+    char C = Glob[i];
+    switch (C) {
+    case '?':
+      RegexString += '.';
+      break;
+    case '*': {
+      const char *PrevChar = i > 0 ? Glob.data() + i - 1 : nullptr;
+      NumWildcards = 1;
+      ++i;
+      while (i < Glob.size() && Glob[i] == '*') {
+        ++NumWildcards;
+        ++i;
+      }
+      const char *NextChar = i < Glob.size() ? Glob.data() + i : nullptr;
+
+      if ((NumWildcards > 1) && (PrevChar == nullptr || *PrevChar == '/') &&
+          (NextChar == nullptr || *NextChar == '/')) {
+        RegexString += "(([^/]*(/|$))*)";
+      } else
+        RegexString += "([^/]*)";
+      break;
+    }
+    default:
+      if (RegexMetachars.find(C) != StringRef::npos)
+        RegexString.push_back('\\');
+      RegexString.push_back(C);
+    }
+  }
+  RegexString.push_back('$');
+  if (NumWildcards == 0)
+    return make_error<StringError>("not a glob", inconvertibleErrorCode());
+
+  llvm::Regex Rule = Regex(RegexString);
+  std::string Error;
+  if (!Rule.isValid(Error))
+    return make_error<StringError>(Error, inconvertibleErrorCode());
+
+  return std::move(Rule);
+}
-- 
GitLab


From d19d8e05b35221e62db76ba798995f5bc870620b Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 25 Mar 2024 14:25:23 -0700
Subject: [PATCH 198/404] [RISCV] Remove unused Constraint template parameter
 from RISCVInstrInfoZvk.td. NFC

---
 llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td | 59 +++++++++-------------
 1 file changed, 25 insertions(+), 34 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
index 67b2ad459360..09e61c10a57b 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
@@ -180,14 +180,14 @@ class ZvkMxSet<string vd_lmul> {
                                    !eq(vd_lmul, "MF8") : [V_MF8]);
 }
 
-class VPseudoUnaryNoMask_Zvk<DAGOperand RetClass, VReg OpClass, string Constraint = ""> :
+class VPseudoUnaryNoMask_Zvk<DAGOperand RetClass, VReg OpClass> :
       Pseudo<(outs RetClass:$rd),
         (ins RetClass:$merge, OpClass:$rs2, AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
         RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
   let hasSideEffects = 0;
-  let Constraints = !interleave([Constraint, "$rd = $merge"], ",");
+  let Constraints = "$rd = $merge";
   let HasVLOp = 1;
   let HasSEWOp = 1;
   let HasVecPolicyOp = 1;
@@ -196,8 +196,7 @@ class VPseudoUnaryNoMask_Zvk<DAGOperand RetClass, VReg OpClass, string Constrain
 
 class VPseudoBinaryNoMask_Zvk<VReg RetClass,
                               VReg Op1Class,
-                              DAGOperand Op2Class,
-                              string Constraint> :
+                              DAGOperand Op2Class> :
         Pseudo<(outs RetClass:$rd),
                (ins RetClass:$merge, Op1Class:$rs2, Op2Class:$rs1,
                     AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
@@ -205,7 +204,7 @@ class VPseudoBinaryNoMask_Zvk<VReg RetClass,
   let mayLoad = 0;
   let mayStore = 0;
   let hasSideEffects = 0;
-  let Constraints = !interleave([Constraint, "$rd = $merge"], ",");
+  let Constraints = "$rd = $merge";
   let HasVLOp = 1;
   let HasSEWOp = 1;
   let HasVecPolicyOp = 1;
@@ -215,98 +214,90 @@ class VPseudoBinaryNoMask_Zvk<VReg RetClass,
 multiclass VPseudoBinaryNoMask_Zvk<VReg RetClass,
                                    VReg Op1Class,
                                    DAGOperand Op2Class,
-                                   LMULInfo MInfo,
-                                   string Constraint = ""> {
+                                   LMULInfo MInfo> {
   let VLMul = MInfo.value in
-    def "_" # MInfo.MX : VPseudoBinaryNoMask_Zvk<RetClass, Op1Class, Op2Class,
-                                                 Constraint>;
+    def "_" # MInfo.MX : VPseudoBinaryNoMask_Zvk<RetClass, Op1Class, Op2Class>;
 }
 
-multiclass VPseudoUnaryV_V_NoMask_Zvk<LMULInfo m, string Constraint = ""> {
+multiclass VPseudoUnaryV_V_NoMask_Zvk<LMULInfo m> {
   let VLMul = m.value in {
-    def "_VV_" # m.MX : VPseudoUnaryNoMask_Zvk<m.vrclass, m.vrclass, Constraint>;
+    def "_VV_" # m.MX : VPseudoUnaryNoMask_Zvk<m.vrclass, m.vrclass>;
   }
 }
 
-multiclass VPseudoUnaryV_S_NoMask_Zvk<LMULInfo m, string Constraint = ""> {
+multiclass VPseudoUnaryV_S_NoMask_Zvk<LMULInfo m> {
   let VLMul = m.value in
     foreach vs2_lmul = ZvkMxSet<m.MX>.vs2_lmuls in
-      def "_VS_" # m.MX # "_" # vs2_lmul.MX : VPseudoUnaryNoMask_Zvk<m.vrclass, vs2_lmul.vrclass, Constraint>;
+      def "_VS_" # m.MX # "_" # vs2_lmul.MX : VPseudoUnaryNoMask_Zvk<m.vrclass, vs2_lmul.vrclass>;
 }
 
-multiclass VPseudoVALU_V_NoMask_Zvk<string Constraint = ""> {
+multiclass VPseudoVALU_V_NoMask_Zvk {
   foreach m = MxListVF4 in {
     defvar mx = m.MX;
     defvar WriteVIALUV_MX = !cast<SchedWrite>("WriteVIALUV_" # mx);
     defvar ReadVIALUV_MX = !cast<SchedRead>("ReadVIALUV_" # mx);
 
-    defm "" : VPseudoUnaryV_V_NoMask_Zvk<m, Constraint>,
+    defm "" : VPseudoUnaryV_V_NoMask_Zvk<m>,
               Sched<[WriteVIALUV_MX, ReadVIALUV_MX, ReadVIALUV_MX, ReadVMask]>;
   }
 }
 
-multiclass VPseudoVALU_S_NoMask_Zvk<string Constraint = ""> {
+multiclass VPseudoVALU_S_NoMask_Zvk {
   foreach m = MxListVF4 in {
     defvar mx = m.MX;
     defvar WriteVIALUV_MX = !cast<SchedWrite>("WriteVIALUV_" # mx);
     defvar ReadVIALUV_MX = !cast<SchedRead>("ReadVIALUV_" # mx);
 
-    defm "" : VPseudoUnaryV_S_NoMask_Zvk<m, Constraint>,
+    defm "" : VPseudoUnaryV_S_NoMask_Zvk<m>,
               Sched<[WriteVIALUV_MX, ReadVIALUV_MX, ReadVIALUV_MX, ReadVMask]>;
   }
 }
 
-multiclass VPseudoVALU_V_S_NoMask_Zvk<string Constraint = ""> {
-  defm "" : VPseudoVALU_V_NoMask_Zvk<Constraint>;
-  defm "" : VPseudoVALU_S_NoMask_Zvk<Constraint>;
+multiclass VPseudoVALU_V_S_NoMask_Zvk {
+  defm "" : VPseudoVALU_V_NoMask_Zvk;
+  defm "" : VPseudoVALU_S_NoMask_Zvk;
 }
 
-multiclass VPseudoVALU_VV_NoMask_Zvk<string Constraint = ""> {
+multiclass VPseudoVALU_VV_NoMask_Zvk {
   foreach m = MxListVF4 in {
     defvar mx = m.MX;
     defvar WriteVIALUV_MX = !cast<SchedWrite>("WriteVIALUV_" # mx);
     defvar ReadVIALUV_MX = !cast<SchedRead>("ReadVIALUV_" # mx);
 
-    defm _VV : VPseudoBinaryNoMask_Zvk<m.vrclass, m.vrclass, m.vrclass, m,
-                                       Constraint>,
+    defm _VV : VPseudoBinaryNoMask_Zvk<m.vrclass, m.vrclass, m.vrclass, m>,
                Sched<[WriteVIALUV_MX, ReadVIALUV_MX, ReadVIALUV_MX, ReadVMask]>;
   }
 }
 
-multiclass VPseudoVALU_VI_NoMask_Zvk<Operand ImmType = simm5,
-                                     string Constraint = ""> {
+multiclass VPseudoVALU_VI_NoMask_Zvk<Operand ImmType = simm5> {
   foreach m = MxListVF4 in {
     defvar mx = m.MX;
     defvar WriteVIALUV_MX = !cast<SchedWrite>("WriteVIALUV_" # mx);
     defvar ReadVIALUV_MX = !cast<SchedRead>("ReadVIALUV_" # mx);
 
-    defm _VI : VPseudoBinaryNoMask_Zvk<m.vrclass, m.vrclass, ImmType, m,
-                                       Constraint>,
+    defm _VI : VPseudoBinaryNoMask_Zvk<m.vrclass, m.vrclass, ImmType, m>,
                Sched<[WriteVIALUV_MX, ReadVIALUV_MX, ReadVIALUV_MX, ReadVMask]>;
   }
 }
 
-multiclass VPseudoVALU_VI_NoMaskTU_Zvk<Operand ImmType = uimm5,
-                                       string Constraint = ""> {
+multiclass VPseudoVALU_VI_NoMaskTU_Zvk<Operand ImmType = uimm5> {
   foreach m = MxListVF4 in {
     defvar mx = m.MX;
     defvar WriteVIALUV_MX = !cast<SchedWrite>("WriteVIALUV_" # mx);
     defvar ReadVIALUV_MX = !cast<SchedRead>("ReadVIALUV_" # mx);
 
-    defm _VI : VPseudoBinaryNoMask<m.vrclass, m.vrclass, ImmType, m,
-                                   Constraint>,
+    defm _VI : VPseudoBinaryNoMask<m.vrclass, m.vrclass, ImmType, m>,
                Sched<[WriteVIALUV_MX, ReadVIALUV_MX, ReadVIALUV_MX, ReadVMask]>;
   }
 }
 
-multiclass VPseudoVALU_VV_NoMaskTU_Zvk<string Constraint = ""> {
+multiclass VPseudoVALU_VV_NoMaskTU_Zvk {
   foreach m = MxListVF4 in {
     defvar mx = m.MX;
     defvar WriteVIALUV_MX = !cast<SchedWrite>("WriteVIALUV_" # mx);
     defvar ReadVIALUV_MX = !cast<SchedRead>("ReadVIALUV_" # mx);
 
-    defm _VV : VPseudoBinaryNoMask<m.vrclass, m.vrclass, m.vrclass, m,
-                                   Constraint>,
+    defm _VV : VPseudoBinaryNoMask<m.vrclass, m.vrclass, m.vrclass, m>,
                Sched<[WriteVIALUV_MX, ReadVIALUV_MX, ReadVIALUV_MX, ReadVMask]>;
   }
 }
-- 
GitLab


From 1880e41a4cd0bdd5abec955134ddd49d2b016f89 Mon Sep 17 00:00:00 2001
From: Eric <eric@efcs.ca>
Date: Mon, 25 Mar 2024 17:34:43 -0400
Subject: [PATCH 199/404] Fix the OSS fuzz build. (#86562)

The OSS fuzz build has been broken for a while because of changes
to the cmake configuration. This patch fixes the issues by defaulting to
libunwind as the default runtime
---
 libcxx/utils/ci/oss-fuzz.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcxx/utils/ci/oss-fuzz.sh b/libcxx/utils/ci/oss-fuzz.sh
index d5e7fcea6012..e5723406a9ff 100755
--- a/libcxx/utils/ci/oss-fuzz.sh
+++ b/libcxx/utils/ci/oss-fuzz.sh
@@ -14,7 +14,7 @@ MONOREPO_ROOT=${PWD}
 
 mkdir ${BUILD}
 cmake -S ${MONOREPO_ROOT}/runtimes -B ${BUILD} \
-      -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \
+      -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
       -DCMAKE_BUILD_TYPE=RelWithDebInfo \
       -DCMAKE_INSTALL_PREFIX="${INSTALL}"
 cmake --build ${BUILD} --target install-cxx-headers
-- 
GitLab


From 9855134d079a193e5c17259f024082d6bb5396bf Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Mon, 25 Mar 2024 14:36:03 -0700
Subject: [PATCH 200/404] [memprof] Use #ifdef EXPENSIVE_CHECKS (#86585)

This patch replaces:

  #if EXPENSIVE_CHECKS

with:

  #ifdef EXPENSIVE_CHECKS

to follow the existing conventions.
---
 llvm/lib/ProfileData/InstrProfReader.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index efe135a0b409..4f786a23f354 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -1262,7 +1262,7 @@ Error IndexedInstrProfReader::readHeader() {
         /*Payload=*/Start + FramePayloadOffset,
         /*Base=*/Start, memprof::FrameLookupTrait()));
 
-#if EXPENSIVE_CHECKS
+#ifdef EXPENSIVE_CHECKS
     // Go through all the records and verify that CSId has been correctly
     // populated.  Do this only under EXPENSIVE_CHECKS.  Otherwise, we
     // would defeat the purpose of OnDiskIterableChainedHashTable.
-- 
GitLab


From bce17034157fdfe4d898d30366c1eeca3442fa3d Mon Sep 17 00:00:00 2001
From: Andrei Golubev <andrey.golubev@intel.com>
Date: Mon, 25 Mar 2024 23:47:39 +0200
Subject: [PATCH 201/404] [ODS][NFC] Cast range.size() to int32_t in
 accumulation (#85629)

Using range.size() "as is" means we accumulate 'size_t' values into
'int32_t' variable. This may produce narrowing conversion warnings
(particularly, on MSVC). The surrounding code seems to cast <x>.size()
to 'int32_t' so following this practice seems safe enough.

Co-authored-by: Ovidiu Pintican <ovidiu.pintican@intel.com>
---
 mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index 0d81912afb61..3a697520dfad 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -3057,7 +3057,7 @@ void OpEmitter::genCodeForAddingArgAndRegionForBuilder(
         body << llvm::formatv(
             "static_cast<int32_t>(std::accumulate({0}.begin(), {0}.end(), 0, "
             "[](int32_t curSum, ::mlir::ValueRange range) {{ return curSum + "
-            "range.size(); }))",
+            "static_cast<int32_t>(range.size()); }))",
             operandName);
       } else {
         body << "static_cast<int32_t>(" << getArgumentName(op, i) << ".size())";
-- 
GitLab


From 930f64689c1fb487714c3836ffa43e49e46aa488 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Mon, 25 Mar 2024 14:50:58 -0700
Subject: [PATCH 202/404] [lldb] Implement coalescing of disjoint progress
 events (#84854)

This implements coalescing of progress events using a timeout, as
discussed in the RFC on Discourse [1]. This PR consists of two commits
which, depending on the feedback, I may split up into two PRs. For now,
I think it's easier to review this as a whole.

1. The first commit introduces a new generic `Alarm` class. The class
lets you to schedule a function (callback) to be executed after a given
timeout expires. You can cancel and reset a callback before its
corresponding timeout expires. It achieves this with the help of a
worker thread that sleeps until the next timeout expires. The only
guarantee it provides is that your function is called no sooner than the
requested timeout. Because the callback is called directly from the
worker thread, a long running callback could potentially block the
worker thread. I intentionally kept the implementation as simple as
possible while addressing the needs for the `ProgressManager` use case.
If we want to rely on this somewhere else, we can reassess whether we
need to address those limitations.

2. The second commit uses the Alarm class to coalesce progress events.
To recap the Discourse discussion, when multiple progress events with
the same title execute in close succession, they get broadcast as one to
`eBroadcastBitProgressCategory`. The `ProgressManager` keeps track of
the in-flight progress events and when the refcount hits zero, the Alarm
class is used to schedule broadcasting the event. If a new progress
event comes in before the alarm fires, the alarm is reset (and the
process repeats when the new progress event ends). If no new event comes
in before the timeout expires, the progress event is broadcast.

[1]
https://discourse.llvm.org/t/rfc-improve-lldb-progress-reporting/75717/
---
 lldb/include/lldb/Core/Progress.h             |  35 +++++-
 lldb/source/Core/Progress.cpp                 | 114 ++++++++++++++----
 .../SystemInitializerCommon.cpp               |   3 +
 lldb/unittests/Core/ProgressReportTest.cpp    |  39 +++++-
 4 files changed, 159 insertions(+), 32 deletions(-)

diff --git a/lldb/include/lldb/Core/Progress.h b/lldb/include/lldb/Core/Progress.h
index eb3af9816dc6..cd87be79c4f0 100644
--- a/lldb/include/lldb/Core/Progress.h
+++ b/lldb/include/lldb/Core/Progress.h
@@ -9,6 +9,7 @@
 #ifndef LLDB_CORE_PROGRESS_H
 #define LLDB_CORE_PROGRESS_H
 
+#include "lldb/Host/Alarm.h"
 #include "lldb/lldb-forward.h"
 #include "lldb/lldb-types.h"
 #include "llvm/ADT/StringMap.h"
@@ -150,9 +151,12 @@ public:
   void Increment(const Progress::ProgressData &);
   void Decrement(const Progress::ProgressData &);
 
+  static void Initialize();
+  static void Terminate();
+  static bool Enabled();
   static ProgressManager &Instance();
 
-private:
+protected:
   enum class EventType {
     Begin,
     End,
@@ -160,9 +164,32 @@ private:
   static void ReportProgress(const Progress::ProgressData &progress_data,
                              EventType type);
 
-  llvm::StringMap<std::pair<uint64_t, Progress::ProgressData>>
-      m_progress_category_map;
-  std::mutex m_progress_map_mutex;
+  static std::optional<ProgressManager> &InstanceImpl();
+
+  /// Helper function for reporting progress when the alarm in the corresponding
+  /// entry in the map expires.
+  void Expire(llvm::StringRef key);
+
+  /// Entry used for bookkeeping.
+  struct Entry {
+    /// Reference count used for overlapping events.
+    uint64_t refcount = 0;
+
+    /// Data used to emit progress events.
+    Progress::ProgressData data;
+
+    /// Alarm handle used when the refcount reaches zero.
+    Alarm::Handle handle = Alarm::INVALID_HANDLE;
+  };
+
+  /// Map used for bookkeeping.
+  llvm::StringMap<Entry> m_entries;
+
+  /// Mutex to provide the map.
+  std::mutex m_entries_mutex;
+
+  /// Alarm instance to coalesce progress events.
+  Alarm m_alarm;
 };
 
 } // namespace lldb_private
diff --git a/lldb/source/Core/Progress.cpp b/lldb/source/Core/Progress.cpp
index b4b5e98b7ba4..161038284e21 100644
--- a/lldb/source/Core/Progress.cpp
+++ b/lldb/source/Core/Progress.cpp
@@ -35,7 +35,10 @@ Progress::Progress(std::string title, std::string details,
 
   std::lock_guard<std::mutex> guard(m_mutex);
   ReportProgress();
-  ProgressManager::Instance().Increment(m_progress_data);
+
+  // Report to the ProgressManager if that subsystem is enabled.
+  if (ProgressManager::Enabled())
+    ProgressManager::Instance().Increment(m_progress_data);
 }
 
 Progress::~Progress() {
@@ -45,7 +48,10 @@ Progress::~Progress() {
   if (!m_completed)
     m_completed = m_total;
   ReportProgress();
-  ProgressManager::Instance().Decrement(m_progress_data);
+
+  // Report to the ProgressManager if that subsystem is enabled.
+  if (ProgressManager::Enabled())
+    ProgressManager::Instance().Decrement(m_progress_data);
 }
 
 void Progress::Increment(uint64_t amount,
@@ -75,45 +81,84 @@ void Progress::ReportProgress() {
   }
 }
 
-ProgressManager::ProgressManager() : m_progress_category_map() {}
+ProgressManager::ProgressManager()
+    : m_entries(), m_alarm(std::chrono::milliseconds(100)) {}
 
 ProgressManager::~ProgressManager() {}
 
+void ProgressManager::Initialize() {
+  assert(!InstanceImpl() && "Already initialized.");
+  InstanceImpl().emplace();
+}
+
+void ProgressManager::Terminate() {
+  assert(InstanceImpl() && "Already terminated.");
+  InstanceImpl().reset();
+}
+
+bool ProgressManager::Enabled() { return InstanceImpl().operator bool(); }
+
 ProgressManager &ProgressManager::Instance() {
-  static std::once_flag g_once_flag;
-  static ProgressManager *g_progress_manager = nullptr;
-  std::call_once(g_once_flag, []() {
-    // NOTE: known leak to avoid global destructor chain issues.
-    g_progress_manager = new ProgressManager();
-  });
-  return *g_progress_manager;
+  assert(InstanceImpl() && "ProgressManager must be initialized");
+  return *InstanceImpl();
+}
+
+std::optional<ProgressManager> &ProgressManager::InstanceImpl() {
+  static std::optional<ProgressManager> g_progress_manager;
+  return g_progress_manager;
 }
 
 void ProgressManager::Increment(const Progress::ProgressData &progress_data) {
-  std::lock_guard<std::mutex> lock(m_progress_map_mutex);
-  // If the current category exists in the map then it is not an initial report,
-  // therefore don't broadcast to the category bit. Also, store the current
-  // progress data in the map so that we have a note of the ID used for the
-  // initial progress report.
-  if (!m_progress_category_map.contains(progress_data.title)) {
-    m_progress_category_map[progress_data.title].second = progress_data;
+  std::lock_guard<std::mutex> lock(m_entries_mutex);
+
+  llvm::StringRef key = progress_data.title;
+  bool new_entry = !m_entries.contains(key);
+  Entry &entry = m_entries[progress_data.title];
+
+  if (new_entry) {
+    // This is a new progress event. Report progress and store the progress
+    // data.
     ReportProgress(progress_data, EventType::Begin);
+    entry.data = progress_data;
+  } else if (entry.refcount == 0) {
+    // This is an existing entry that was scheduled to be deleted but a new one
+    // came in before the timer expired.
+    assert(entry.handle != Alarm::INVALID_HANDLE);
+
+    if (!m_alarm.Cancel(entry.handle)) {
+      // The timer expired before we had a chance to cancel it. We have to treat
+      // this as an entirely new progress event.
+      ReportProgress(progress_data, EventType::Begin);
+    }
+    // Clear the alarm handle.
+    entry.handle = Alarm::INVALID_HANDLE;
   }
-  m_progress_category_map[progress_data.title].first++;
+
+  // Regardless of how we got here, we need to bump the reference count.
+  entry.refcount++;
 }
 
 void ProgressManager::Decrement(const Progress::ProgressData &progress_data) {
-  std::lock_guard<std::mutex> lock(m_progress_map_mutex);
-  auto pos = m_progress_category_map.find(progress_data.title);
+  std::lock_guard<std::mutex> lock(m_entries_mutex);
+  llvm::StringRef key = progress_data.title;
 
-  if (pos == m_progress_category_map.end())
+  if (!m_entries.contains(key))
     return;
 
-  if (pos->second.first <= 1) {
-    ReportProgress(pos->second.second, EventType::End);
-    m_progress_category_map.erase(progress_data.title);
-  } else {
-    --pos->second.first;
+  Entry &entry = m_entries[key];
+  entry.refcount--;
+
+  if (entry.refcount == 0) {
+    assert(entry.handle == Alarm::INVALID_HANDLE);
+
+    // Copy the key to a std::string so we can pass it by value to the lambda.
+    // The underlying StringRef will not exist by the time the callback is
+    // called.
+    std::string key_str = std::string(key);
+
+    // Start a timer. If it expires before we see another progress event, it
+    // will be reported.
+    entry.handle = m_alarm.Create([=]() { Expire(key_str); });
   }
 }
 
@@ -129,3 +174,20 @@ void ProgressManager::ReportProgress(
                            progress_data.debugger_id,
                            Debugger::eBroadcastBitProgressCategory);
 }
+
+void ProgressManager::Expire(llvm::StringRef key) {
+  std::lock_guard<std::mutex> lock(m_entries_mutex);
+
+  // This shouldn't happen but be resilient anyway.
+  if (!m_entries.contains(key))
+    return;
+
+  // A new event came in and the alarm fired before we had a chance to restart
+  // it.
+  if (m_entries[key].refcount != 0)
+    return;
+
+  // We're done with this entry.
+  ReportProgress(m_entries[key].data, EventType::End);
+  m_entries.erase(key);
+}
diff --git a/lldb/source/Initialization/SystemInitializerCommon.cpp b/lldb/source/Initialization/SystemInitializerCommon.cpp
index 1a172a95aa14..5d35854211c2 100644
--- a/lldb/source/Initialization/SystemInitializerCommon.cpp
+++ b/lldb/source/Initialization/SystemInitializerCommon.cpp
@@ -9,6 +9,7 @@
 #include "lldb/Initialization/SystemInitializerCommon.h"
 
 #include "Plugins/Process/gdb-remote/ProcessGDBRemoteLog.h"
+#include "lldb/Core/Progress.h"
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Host/Host.h"
 #include "lldb/Host/Socket.h"
@@ -69,6 +70,7 @@ llvm::Error SystemInitializerCommon::Initialize() {
   Diagnostics::Initialize();
   FileSystem::Initialize();
   HostInfo::Initialize(m_shlib_dir_helper);
+  ProgressManager::Initialize();
 
   llvm::Error error = Socket::Initialize();
   if (error)
@@ -97,6 +99,7 @@ void SystemInitializerCommon::Terminate() {
 #endif
 
   Socket::Terminate();
+  ProgressManager::Terminate();
   HostInfo::Terminate();
   Log::DisableAllLogChannels();
   FileSystem::Terminate();
diff --git a/lldb/unittests/Core/ProgressReportTest.cpp b/lldb/unittests/Core/ProgressReportTest.cpp
index 1f993180fd83..f0d253be9bf6 100644
--- a/lldb/unittests/Core/ProgressReportTest.cpp
+++ b/lldb/unittests/Core/ProgressReportTest.cpp
@@ -22,7 +22,7 @@
 using namespace lldb;
 using namespace lldb_private;
 
-static std::chrono::milliseconds TIMEOUT(100);
+static std::chrono::milliseconds TIMEOUT(500);
 
 class ProgressReportTest : public ::testing::Test {
 public:
@@ -56,7 +56,8 @@ protected:
 
   DebuggerSP m_debugger_sp;
   ListenerSP m_listener_sp;
-  SubsystemRAII<FileSystem, HostInfo, PlatformMacOSX> subsystems;
+  SubsystemRAII<FileSystem, HostInfo, PlatformMacOSX, ProgressManager>
+      subsystems;
 };
 
 TEST_F(ProgressReportTest, TestReportCreation) {
@@ -210,3 +211,37 @@ TEST_F(ProgressReportTest, TestOverlappingEvents) {
   // initial report.
   EXPECT_EQ(data->GetID(), expected_progress_id);
 }
+
+TEST_F(ProgressReportTest, TestProgressManagerDisjointReports) {
+  ListenerSP listener_sp =
+      CreateListenerFor(Debugger::eBroadcastBitProgressCategory);
+  EventSP event_sp;
+  const ProgressEventData *data;
+  uint64_t expected_progress_id;
+
+  { Progress progress("Coalesced report 1", "Starting report 1"); }
+  { Progress progress("Coalesced report 1", "Starting report 2"); }
+  { Progress progress("Coalesced report 1", "Starting report 3"); }
+
+  ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT));
+  data = ProgressEventData::GetEventDataFromEvent(event_sp.get());
+  expected_progress_id = data->GetID();
+
+  EXPECT_EQ(data->GetDetails(), "");
+  EXPECT_FALSE(data->IsFinite());
+  EXPECT_FALSE(data->GetCompleted());
+  EXPECT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal);
+  EXPECT_EQ(data->GetMessage(), "Coalesced report 1");
+
+  ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT));
+  data = ProgressEventData::GetEventDataFromEvent(event_sp.get());
+
+  EXPECT_EQ(data->GetID(), expected_progress_id);
+  EXPECT_EQ(data->GetDetails(), "");
+  EXPECT_FALSE(data->IsFinite());
+  EXPECT_TRUE(data->GetCompleted());
+  EXPECT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal);
+  EXPECT_EQ(data->GetMessage(), "Coalesced report 1");
+
+  ASSERT_FALSE(listener_sp->GetEvent(event_sp, TIMEOUT));
+}
-- 
GitLab


From cb994d41c3afb2bd0b25a4c5b2ac48978bf1b23d Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Mon, 25 Mar 2024 16:47:02 -0500
Subject: [PATCH 203/404] Fix build with shared libraries

/usr/bin/ld: CMakeFiles/ClangReplInterpreterTests.dir/InterpreterExtensi
onsTest.cpp.o: undefined reference to symbol '_ZN4llvm14TargetRegistry12
lookupTargetENS_9StringRefERNSt7__cxx1112basic_stringIcSt11char_traitsIc
ESaIcEEE'
/usr/bin/ld: /work/kparzysz/git/llvm.org/b/x86/lib/libLLVMMC.so.19.0git:
 error adding symbols: DSO missing from command line
collect2: error: ld returned 1 exit status

The missing symbol is `llvm::TargetRegistry::lookupTarget`, which
interestingly enough is in MC.

Add `MC` to the list of LLVM dependencies.
---
 clang/unittests/Interpreter/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clang/unittests/Interpreter/CMakeLists.txt b/clang/unittests/Interpreter/CMakeLists.txt
index b56e1e21015d..e5a77e77de75 100644
--- a/clang/unittests/Interpreter/CMakeLists.txt
+++ b/clang/unittests/Interpreter/CMakeLists.txt
@@ -1,6 +1,7 @@
 set(LLVM_LINK_COMPONENTS
   ${LLVM_TARGETS_TO_BUILD}
   Core
+  MC
   OrcJIT
   Support
   TargetParser
-- 
GitLab


From 765d4c402fe2ff614a15a762bb7cefe7289663b4 Mon Sep 17 00:00:00 2001
From: Jason Molenda <jmolenda@apple.com>
Date: Mon, 25 Mar 2024 15:00:36 -0700
Subject: [PATCH 204/404] [lldb] [ObjectFileMachO] LLVM_COV is not mapped into
 firmware memory (#86359)

It is possible to gather code coverage in a firmware environment, where
the __LLVM_COV segment will not be mapped in memory but does exist in
the binary, see

https://llvm.org/devmtg/2020-09/slides/PhippsAlan_EmbeddedCodeCoverage_LLVM_Conf_Talk_final.pdf

The __LLVM_COV segment in the binary happens to be at the same address
as the __DATA segment, so if lldb treats this segment as loaded, it
shadows the __DATA segment and address->symbol resolution can fail.

For these non-userland code cases, we need to mark __LLVM_COV as not a
loadable segment.

rdar://124475661
---
 .../Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp    | 12 ++++++++++++
 .../Plugins/ObjectFile/Mach-O/ObjectFileMachO.h      |  1 +
 2 files changed, 13 insertions(+)

diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
index bcf3a3274cf3..1caf93659956 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
@@ -905,6 +905,11 @@ ConstString ObjectFileMachO::GetSegmentNameDWARF() {
   return g_section_name;
 }
 
+ConstString ObjectFileMachO::GetSegmentNameLLVM_COV() {
+  static ConstString g_section_name("__LLVM_COV");
+  return g_section_name;
+}
+
 ConstString ObjectFileMachO::GetSectionNameEHFrame() {
   static ConstString g_section_name_eh_frame("__eh_frame");
   return g_section_name_eh_frame;
@@ -6145,6 +6150,13 @@ bool ObjectFileMachO::SectionIsLoadable(const Section *section) {
     return false;
   if (GetModule().get() != section->GetModule().get())
     return false;
+  // firmware style binaries with llvm gcov segment do
+  // not have that segment mapped into memory.
+  if (section->GetName() == GetSegmentNameLLVM_COV()) {
+    const Strata strata = GetStrata();
+    if (strata == eStrataKernel || strata == eStrataRawImage)
+      return false;
+  }
   // Be careful with __LINKEDIT and __DWARF segments
   if (section->GetName() == GetSegmentNameLINKEDIT() ||
       section->GetName() == GetSegmentNameDWARF()) {
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
index 0a47f3a7dd18..55bc688126eb 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
@@ -271,6 +271,7 @@ protected:
   static lldb_private::ConstString GetSegmentNameOBJC();
   static lldb_private::ConstString GetSegmentNameLINKEDIT();
   static lldb_private::ConstString GetSegmentNameDWARF();
+  static lldb_private::ConstString GetSegmentNameLLVM_COV();
   static lldb_private::ConstString GetSectionNameEHFrame();
 
   llvm::MachO::dysymtab_command m_dysymtab;
-- 
GitLab


From 060df78cdbbf70d5a6dfff3af1d435a5a811b886 Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com>
Date: Mon, 25 Mar 2024 18:01:46 -0400
Subject: [PATCH 205/404] [DXIL] Add Float `Dot` Intrinsic Lowering (#86071)

Completes #83626
- `CGBuiltin.cpp` - modify `getDotProductIntrinsic` to be able to emit
`dot2`, `dot3`, and `dot4` intrinsics based on element count
- `IntrinsicsDirectX.td` - for floating point add `dot2`, `dot3`, and
`dot4` inntrinsics -`DXIL.td` add dxilop intrinsic lowering for `dot2`,
`dot3`, & `dot4`.
- `DXILOpLowering.cpp` - add vector arg flattening for dot product.
- `DXILOpBuilder.h` - modify `createDXILOpCall` to take a smallVector
instead of an iterator
- `DXILOpBuilder.cpp` - modify `createDXILOpCall` by moving the small
vector up to the calling function in `DXILOpLowering.cpp`.
- Moving one function up gives us access to the `CallInst` and
`Function` which were needed to distinguish the dot product intrinsics
and get the operands without using the iterator.
---
 clang/lib/CodeGen/CGBuiltin.cpp            | 25 +++---
 clang/test/CodeGenHLSL/builtins/dot.hlsl   | 28 +++----
 llvm/include/llvm/IR/IntrinsicsDirectX.td  | 10 ++-
 llvm/lib/Target/DirectX/DXIL.td            |  9 +++
 llvm/lib/Target/DirectX/DXILOpBuilder.cpp  |  8 +-
 llvm/lib/Target/DirectX/DXILOpBuilder.h    |  5 +-
 llvm/lib/Target/DirectX/DXILOpLowering.cpp | 55 ++++++++++++-
 llvm/test/CodeGen/DirectX/dot2_error.ll    | 10 +++
 llvm/test/CodeGen/DirectX/dot3_error.ll    | 10 +++
 llvm/test/CodeGen/DirectX/dot4_error.ll    | 10 +++
 llvm/test/CodeGen/DirectX/fdot.ll          | 94 ++++++++++++++++++++++
 11 files changed, 230 insertions(+), 34 deletions(-)
 create mode 100644 llvm/test/CodeGen/DirectX/dot2_error.ll
 create mode 100644 llvm/test/CodeGen/DirectX/dot3_error.ll
 create mode 100644 llvm/test/CodeGen/DirectX/dot4_error.ll
 create mode 100644 llvm/test/CodeGen/DirectX/fdot.ll

diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 2eaceeba6177..8f4817258e3b 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -18066,15 +18066,22 @@ llvm::Value *CodeGenFunction::EmitScalarOrConstFoldImmArg(unsigned ICEArguments,
   return Arg;
 }
 
-Intrinsic::ID getDotProductIntrinsic(QualType QT) {
+Intrinsic::ID getDotProductIntrinsic(QualType QT, int elementCount) {
+  if (QT->hasFloatingRepresentation()) {
+    switch (elementCount) {
+    case 2:
+      return Intrinsic::dx_dot2;
+    case 3:
+      return Intrinsic::dx_dot3;
+    case 4:
+      return Intrinsic::dx_dot4;
+    }
+  }
   if (QT->hasSignedIntegerRepresentation())
     return Intrinsic::dx_sdot;
-  if (QT->hasUnsignedIntegerRepresentation())
-    return Intrinsic::dx_udot;
 
-  assert(QT->hasFloatingRepresentation());
-  return Intrinsic::dx_dot;
-  ;
+  assert(QT->hasUnsignedIntegerRepresentation());
+  return Intrinsic::dx_udot;
 }
 
 Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
@@ -18128,8 +18135,7 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
     assert(T0->getScalarType() == T1->getScalarType() &&
            "Dot product of vectors need the same element types.");
 
-    [[maybe_unused]] auto *VecTy0 =
-        E->getArg(0)->getType()->getAs<VectorType>();
+    auto *VecTy0 = E->getArg(0)->getType()->getAs<VectorType>();
     [[maybe_unused]] auto *VecTy1 =
         E->getArg(1)->getType()->getAs<VectorType>();
     // A HLSLVectorTruncation should have happend
@@ -18138,7 +18144,8 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
 
     return Builder.CreateIntrinsic(
         /*ReturnType=*/T0->getScalarType(),
-        getDotProductIntrinsic(E->getArg(0)->getType()),
+        getDotProductIntrinsic(E->getArg(0)->getType(),
+                               VecTy0->getNumElements()),
         ArrayRef<Value *>{Op0, Op1}, nullptr, "dx.dot");
   } break;
   case Builtin::BI__builtin_hlsl_lerp: {
diff --git a/clang/test/CodeGenHLSL/builtins/dot.hlsl b/clang/test/CodeGenHLSL/builtins/dot.hlsl
index 0f993193c00c..307d71cce3cb 100644
--- a/clang/test/CodeGenHLSL/builtins/dot.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/dot.hlsl
@@ -110,21 +110,21 @@ uint64_t test_dot_ulong4(uint64_t4 p0, uint64_t4 p1) { return dot(p0, p1); }
 // NO_HALF: ret float %dx.dot
 half test_dot_half(half p0, half p1) { return dot(p0, p1); }
 
-// NATIVE_HALF: %dx.dot = call half @llvm.dx.dot.v2f16(<2 x half> %0, <2 x half> %1)
+// NATIVE_HALF: %dx.dot = call half @llvm.dx.dot2.v2f16(<2 x half> %0, <2 x half> %1)
 // NATIVE_HALF: ret half %dx.dot
-// NO_HALF: %dx.dot = call float @llvm.dx.dot.v2f32(<2 x float> %0, <2 x float> %1)
+// NO_HALF: %dx.dot = call float @llvm.dx.dot2.v2f32(<2 x float> %0, <2 x float> %1)
 // NO_HALF: ret float %dx.dot
 half test_dot_half2(half2 p0, half2 p1) { return dot(p0, p1); }
 
-// NATIVE_HALF: %dx.dot = call half @llvm.dx.dot.v3f16(<3 x half> %0, <3 x half> %1)
+// NATIVE_HALF: %dx.dot = call half @llvm.dx.dot3.v3f16(<3 x half> %0, <3 x half> %1)
 // NATIVE_HALF: ret half %dx.dot
-// NO_HALF: %dx.dot = call float @llvm.dx.dot.v3f32(<3 x float> %0, <3 x float> %1)
+// NO_HALF: %dx.dot = call float @llvm.dx.dot3.v3f32(<3 x float> %0, <3 x float> %1)
 // NO_HALF: ret float %dx.dot
 half test_dot_half3(half3 p0, half3 p1) { return dot(p0, p1); }
 
-// NATIVE_HALF: %dx.dot = call half @llvm.dx.dot.v4f16(<4 x half> %0, <4 x half> %1)
+// NATIVE_HALF: %dx.dot = call half @llvm.dx.dot4.v4f16(<4 x half> %0, <4 x half> %1)
 // NATIVE_HALF: ret half %dx.dot
-// NO_HALF: %dx.dot = call float @llvm.dx.dot.v4f32(<4 x float> %0, <4 x float> %1)
+// NO_HALF: %dx.dot = call float @llvm.dx.dot4.v4f32(<4 x float> %0, <4 x float> %1)
 // NO_HALF: ret float %dx.dot
 half test_dot_half4(half4 p0, half4 p1) { return dot(p0, p1); }
 
@@ -132,34 +132,34 @@ half test_dot_half4(half4 p0, half4 p1) { return dot(p0, p1); }
 // CHECK: ret float %dx.dot
 float test_dot_float(float p0, float p1) { return dot(p0, p1); }
 
-// CHECK: %dx.dot = call float @llvm.dx.dot.v2f32(<2 x float> %0, <2 x float> %1)
+// CHECK: %dx.dot = call float @llvm.dx.dot2.v2f32(<2 x float> %0, <2 x float> %1)
 // CHECK: ret float %dx.dot
 float test_dot_float2(float2 p0, float2 p1) { return dot(p0, p1); }
 
-// CHECK: %dx.dot = call float @llvm.dx.dot.v3f32(<3 x float> %0, <3 x float> %1)
+// CHECK: %dx.dot = call float @llvm.dx.dot3.v3f32(<3 x float> %0, <3 x float> %1)
 // CHECK: ret float %dx.dot
 float test_dot_float3(float3 p0, float3 p1) { return dot(p0, p1); }
 
-// CHECK: %dx.dot = call float @llvm.dx.dot.v4f32(<4 x float> %0, <4 x float> %1)
+// CHECK: %dx.dot = call float @llvm.dx.dot4.v4f32(<4 x float> %0, <4 x float> %1)
 // CHECK: ret float %dx.dot
 float test_dot_float4(float4 p0, float4 p1) { return dot(p0, p1); }
 
-// CHECK:  %dx.dot = call float @llvm.dx.dot.v2f32(<2 x float> %splat.splat, <2 x float> %1)
+// CHECK:  %dx.dot = call float @llvm.dx.dot2.v2f32(<2 x float> %splat.splat, <2 x float> %1)
 // CHECK: ret float %dx.dot
 float test_dot_float2_splat(float p0, float2 p1) { return dot(p0, p1); }
 
-// CHECK:  %dx.dot = call float @llvm.dx.dot.v3f32(<3 x float> %splat.splat, <3 x float> %1)
+// CHECK:  %dx.dot = call float @llvm.dx.dot3.v3f32(<3 x float> %splat.splat, <3 x float> %1)
 // CHECK: ret float %dx.dot
 float test_dot_float3_splat(float p0, float3 p1) { return dot(p0, p1); }
 
-// CHECK:  %dx.dot = call float @llvm.dx.dot.v4f32(<4 x float> %splat.splat, <4 x float> %1)
+// CHECK:  %dx.dot = call float @llvm.dx.dot4.v4f32(<4 x float> %splat.splat, <4 x float> %1)
 // CHECK: ret float %dx.dot
 float test_dot_float4_splat(float p0, float4 p1) { return dot(p0, p1); }
 
 // CHECK: %conv = sitofp i32 %1 to float
 // CHECK: %splat.splatinsert = insertelement <2 x float> poison, float %conv, i64 0
 // CHECK: %splat.splat = shufflevector <2 x float> %splat.splatinsert, <2 x float> poison, <2 x i32> zeroinitializer
-// CHECK: %dx.dot = call float @llvm.dx.dot.v2f32(<2 x float> %0, <2 x float> %splat.splat)
+// CHECK: %dx.dot = call float @llvm.dx.dot2.v2f32(<2 x float> %0, <2 x float> %splat.splat)
 // CHECK: ret float %dx.dot
 float test_builtin_dot_float2_int_splat(float2 p0, int p1) {
   return dot(p0, p1);
@@ -168,7 +168,7 @@ float test_builtin_dot_float2_int_splat(float2 p0, int p1) {
 // CHECK: %conv = sitofp i32 %1 to float
 // CHECK: %splat.splatinsert = insertelement <3 x float> poison, float %conv, i64 0
 // CHECK: %splat.splat = shufflevector <3 x float> %splat.splatinsert, <3 x float> poison, <3 x i32> zeroinitializer
-// CHECK: %dx.dot = call float @llvm.dx.dot.v3f32(<3 x float> %0, <3 x float> %splat.splat)
+// CHECK: %dx.dot = call float @llvm.dx.dot3.v3f32(<3 x float> %0, <3 x float> %splat.splat)
 // CHECK: ret float %dx.dot
 float test_builtin_dot_float3_int_splat(float3 p0, int p1) {
   return dot(p0, p1);
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index 1164b241ba7b..a871fac46b9f 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -24,7 +24,15 @@ def int_dx_any  : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty]>;
 def int_dx_clamp : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>;
 def int_dx_uclamp : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>; 
 
-def int_dx_dot : 
+def int_dx_dot2 : 
+    Intrinsic<[LLVMVectorElementType<0>], 
+    [llvm_anyfloat_ty, LLVMScalarOrSameVectorWidth<0, LLVMVectorElementType<0>>],
+    [IntrNoMem, IntrWillReturn, Commutative] >;
+def int_dx_dot3 : 
+    Intrinsic<[LLVMVectorElementType<0>], 
+    [llvm_anyfloat_ty, LLVMScalarOrSameVectorWidth<0, LLVMVectorElementType<0>>],
+    [IntrNoMem, IntrWillReturn, Commutative] >;
+def int_dx_dot4 : 
     Intrinsic<[LLVMVectorElementType<0>], 
     [llvm_anyfloat_ty, LLVMScalarOrSameVectorWidth<0, LLVMVectorElementType<0>>],
     [IntrNoMem, IntrWillReturn, Commutative] >;
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index f7e69ebae15b..2e6d58e14fd3 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -303,6 +303,15 @@ def IMad : DXILOpMapping<48, tertiary, int_dx_imad,
                          "Signed integer arithmetic multiply/add operation. imad(m,a,b) = m * a + b.">;
 def UMad : DXILOpMapping<49, tertiary, int_dx_umad,
                          "Unsigned integer arithmetic multiply/add operation. umad(m,a,b) = m * a + b.">;
+let OpTypes = !listconcat([llvm_halforfloat_ty], !listsplat(llvm_halforfloat_ty, 4)) in
+  def Dot2 : DXILOpMapping<54, dot2, int_dx_dot2,
+                           "dot product of two float vectors Dot(a,b) = a[0]*b[0] + ... + a[n]*b[n] where n is between 0 and 1">;
+let OpTypes = !listconcat([llvm_halforfloat_ty], !listsplat(llvm_halforfloat_ty, 6)) in
+  def Dot3 : DXILOpMapping<55, dot3, int_dx_dot3,
+                           "dot product of two float vectors Dot(a,b) = a[0]*b[0] + ... + a[n]*b[n] where n is between 0 and 2">;
+let OpTypes = !listconcat([llvm_halforfloat_ty], !listsplat(llvm_halforfloat_ty, 8)) in
+  def Dot4 : DXILOpMapping<56, dot4, int_dx_dot4,
+                           "dot product of two float vectors Dot(a,b) = a[0]*b[0] + ... + a[n]*b[n] where n is between 0 and 3">;
 def ThreadId : DXILOpMapping<93, threadId, int_dx_thread_id,
                              "Reads the thread ID">;
 def GroupId  : DXILOpMapping<94, groupId, int_dx_group_id,
diff --git a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
index 0841ae95423c..0b3982ea0f43 100644
--- a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
@@ -254,7 +254,7 @@ namespace dxil {
 
 CallInst *DXILOpBuilder::createDXILOpCall(dxil::OpCode OpCode, Type *ReturnTy,
                                           Type *OverloadTy,
-                                          llvm::iterator_range<Use *> Args) {
+                                          SmallVector<Value *> Args) {
   const OpCodeProperty *Prop = getOpCodeProperty(OpCode);
 
   OverloadKind Kind = getOverloadKind(OverloadTy);
@@ -272,10 +272,8 @@ CallInst *DXILOpBuilder::createDXILOpCall(dxil::OpCode OpCode, Type *ReturnTy,
     FunctionType *DXILOpFT = getDXILOpFunctionType(Prop, ReturnTy, OverloadTy);
     DXILFn = M.getOrInsertFunction(DXILFnName, DXILOpFT);
   }
-  SmallVector<Value *> FullArgs;
-  FullArgs.emplace_back(B.getInt32((int32_t)OpCode));
-  FullArgs.append(Args.begin(), Args.end());
-  return B.CreateCall(DXILFn, FullArgs);
+
+  return B.CreateCall(DXILFn, Args);
 }
 
 Type *DXILOpBuilder::getOverloadTy(dxil::OpCode OpCode, FunctionType *FT) {
diff --git a/llvm/lib/Target/DirectX/DXILOpBuilder.h b/llvm/lib/Target/DirectX/DXILOpBuilder.h
index f3abcc6e02a4..5babeae47017 100644
--- a/llvm/lib/Target/DirectX/DXILOpBuilder.h
+++ b/llvm/lib/Target/DirectX/DXILOpBuilder.h
@@ -13,7 +13,7 @@
 #define LLVM_LIB_TARGET_DIRECTX_DXILOPBUILDER_H
 
 #include "DXILConstants.h"
-#include "llvm/ADT/iterator_range.h"
+#include "llvm/ADT/SmallVector.h"
 
 namespace llvm {
 class Module;
@@ -35,8 +35,7 @@ public:
   /// \param OverloadTy Overload type of the DXIL Op call constructed
   /// \return DXIL Op call constructed
   CallInst *createDXILOpCall(dxil::OpCode OpCode, Type *ReturnTy,
-                             Type *OverloadTy,
-                             llvm::iterator_range<Use *> Args);
+                             Type *OverloadTy, SmallVector<Value *> Args);
   Type *getOverloadTy(dxil::OpCode OpCode, FunctionType *FT);
   static const char *getOpCodeName(dxil::OpCode DXILOp);
 
diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
index 3e334b0ec298..f09e322f88e1 100644
--- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
@@ -30,6 +30,48 @@
 using namespace llvm;
 using namespace llvm::dxil;
 
+static bool isVectorArgExpansion(Function &F) {
+  switch (F.getIntrinsicID()) {
+  case Intrinsic::dx_dot2:
+  case Intrinsic::dx_dot3:
+  case Intrinsic::dx_dot4:
+    return true;
+  }
+  return false;
+}
+
+static SmallVector<Value *> populateOperands(Value *Arg, IRBuilder<> &Builder) {
+  SmallVector<Value *, 4> ExtractedElements;
+  auto *VecArg = dyn_cast<FixedVectorType>(Arg->getType());
+  for (unsigned I = 0; I < VecArg->getNumElements(); ++I) {
+    Value *Index = ConstantInt::get(Type::getInt32Ty(Arg->getContext()), I);
+    Value *ExtractedElement = Builder.CreateExtractElement(Arg, Index);
+    ExtractedElements.push_back(ExtractedElement);
+  }
+  return ExtractedElements;
+}
+
+static SmallVector<Value *> argVectorFlatten(CallInst *Orig,
+                                             IRBuilder<> &Builder) {
+  // Note: arg[NumOperands-1] is a pointer and is not needed by our flattening.
+  unsigned NumOperands = Orig->getNumOperands() - 1;
+  assert(NumOperands > 0);
+  Value *Arg0 = Orig->getOperand(0);
+  [[maybe_unused]] auto *VecArg0 = dyn_cast<FixedVectorType>(Arg0->getType());
+  assert(VecArg0);
+  SmallVector<Value *> NewOperands = populateOperands(Arg0, Builder);
+  for (unsigned I = 1; I < NumOperands; ++I) {
+    Value *Arg = Orig->getOperand(I);
+    [[maybe_unused]] auto *VecArg = dyn_cast<FixedVectorType>(Arg->getType());
+    assert(VecArg);
+    assert(VecArg0->getElementType() == VecArg->getElementType());
+    assert(VecArg0->getNumElements() == VecArg->getNumElements());
+    auto NextOperandList = populateOperands(Arg, Builder);
+    NewOperands.append(NextOperandList.begin(), NextOperandList.end());
+  }
+  return NewOperands;
+}
+
 static void lowerIntrinsic(dxil::OpCode DXILOp, Function &F, Module &M) {
   IRBuilder<> B(M.getContext());
   DXILOpBuilder DXILB(M, B);
@@ -39,9 +81,18 @@ static void lowerIntrinsic(dxil::OpCode DXILOp, Function &F, Module &M) {
     if (!CI)
       continue;
 
+    SmallVector<Value *> Args;
+    Value *DXILOpArg = B.getInt32(static_cast<unsigned>(DXILOp));
+    Args.emplace_back(DXILOpArg);
     B.SetInsertPoint(CI);
-    CallInst *DXILCI = DXILB.createDXILOpCall(DXILOp, F.getReturnType(),
-                                              OverloadTy, CI->args());
+    if (isVectorArgExpansion(F)) {
+      SmallVector<Value *> NewArgs = argVectorFlatten(CI, B);
+      Args.append(NewArgs.begin(), NewArgs.end());
+    } else
+      Args.append(CI->arg_begin(), CI->arg_end());
+
+    CallInst *DXILCI =
+        DXILB.createDXILOpCall(DXILOp, F.getReturnType(), OverloadTy, Args);
 
     CI->replaceAllUsesWith(DXILCI);
     CI->eraseFromParent();
diff --git a/llvm/test/CodeGen/DirectX/dot2_error.ll b/llvm/test/CodeGen/DirectX/dot2_error.ll
new file mode 100644
index 000000000000..a27bfaedacd5
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/dot2_error.ll
@@ -0,0 +1,10 @@
+; RUN: not opt -S -dxil-op-lower %s 2>&1 | FileCheck %s
+
+; DXIL operation dot2 does not support double overload type
+; CHECK: LLVM ERROR: Invalid Overload
+
+define noundef double @dot_double2(<2 x double> noundef %a, <2 x double> noundef %b) {
+entry:
+  %dx.dot = call double @llvm.dx.dot2.v2f64(<2 x double> %a, <2 x double> %b)
+  ret double %dx.dot
+}
diff --git a/llvm/test/CodeGen/DirectX/dot3_error.ll b/llvm/test/CodeGen/DirectX/dot3_error.ll
new file mode 100644
index 000000000000..eb69fb145038
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/dot3_error.ll
@@ -0,0 +1,10 @@
+; RUN: not opt -S -dxil-op-lower %s 2>&1 | FileCheck %s
+
+; DXIL operation dot3 does not support double overload type
+; CHECK: LLVM ERROR: Invalid Overload
+
+define noundef double @dot_double3(<3 x double> noundef %a, <3 x double> noundef %b) {
+entry:
+  %dx.dot = call double @llvm.dx.dot3.v3f64(<3 x double> %a, <3 x double> %b)
+  ret double %dx.dot
+}
diff --git a/llvm/test/CodeGen/DirectX/dot4_error.ll b/llvm/test/CodeGen/DirectX/dot4_error.ll
new file mode 100644
index 000000000000..5cd632684c0c
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/dot4_error.ll
@@ -0,0 +1,10 @@
+; RUN: not opt -S -dxil-op-lower %s 2>&1 | FileCheck %s
+
+; DXIL operation dot4 does not support double overload type
+; CHECK: LLVM ERROR: Invalid Overload
+
+define noundef double @dot_double4(<4 x double> noundef %a, <4 x double> noundef %b) {
+entry:
+  %dx.dot = call double @llvm.dx.dot4.v4f64(<4 x double> %a, <4 x double> %b)
+  ret double %dx.dot
+}
diff --git a/llvm/test/CodeGen/DirectX/fdot.ll b/llvm/test/CodeGen/DirectX/fdot.ll
new file mode 100644
index 000000000000..3e13b2ad2650
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/fdot.ll
@@ -0,0 +1,94 @@
+; RUN: opt -S  -dxil-op-lower  < %s | FileCheck %s
+
+; Make sure dxil operation function calls for dot are generated for int/uint vectors.
+
+; CHECK-LABEL: dot_half2
+define noundef half @dot_half2(<2 x half> noundef %a, <2 x half> noundef %b) {
+entry:
+; CHECK: extractelement <2 x half> %a, i32 0
+; CHECK: extractelement <2 x half> %a, i32 1
+; CHECK: extractelement <2 x half> %b, i32 0
+; CHECK: extractelement <2 x half> %b, i32 1
+; CHECK: call half @dx.op.dot2.f16(i32 54, half %{{.*}}, half %{{.*}}, half %{{.*}}, half %{{.*}})
+  %dx.dot = call half @llvm.dx.dot2.v2f16(<2 x half> %a, <2 x half> %b)
+  ret half %dx.dot
+}
+
+; CHECK-LABEL: dot_half3
+define noundef half @dot_half3(<3 x half> noundef %a, <3 x half> noundef %b) {
+entry:
+; CHECK: extractelement <3 x half> %a, i32 0
+; CHECK: extractelement <3 x half> %a, i32 1
+; CHECK: extractelement <3 x half> %a, i32 2
+; CHECK: extractelement <3 x half> %b, i32 0
+; CHECK: extractelement <3 x half> %b, i32 1
+; CHECK: extractelement <3 x half> %b, i32 2
+; CHECK: call half @dx.op.dot3.f16(i32 55, half %{{.*}}, half %{{.*}}, half %{{.*}}, half %{{.*}}, half %{{.*}}, half %{{.*}})
+  %dx.dot = call half @llvm.dx.dot3.v3f16(<3 x half> %a, <3 x half> %b)
+  ret half %dx.dot
+}
+
+; CHECK-LABEL: dot_half4
+define noundef half @dot_half4(<4 x half> noundef %a, <4 x half> noundef %b) {
+entry:
+; CHECK: extractelement <4 x half> %a, i32 0
+; CHECK: extractelement <4 x half> %a, i32 1
+; CHECK: extractelement <4 x half> %a, i32 2
+; CHECK: extractelement <4 x half> %a, i32 3
+; CHECK: extractelement <4 x half> %b, i32 0
+; CHECK: extractelement <4 x half> %b, i32 1
+; CHECK: extractelement <4 x half> %b, i32 2
+; CHECK: extractelement <4 x half> %b, i32 3
+; CHECK: call half @dx.op.dot4.f16(i32 56, half %{{.*}}, half %{{.*}}, half %{{.*}}, half %{{.*}}, half %{{.*}}, half %{{.*}}, half %{{.*}}, half %{{.*}})
+  %dx.dot = call half @llvm.dx.dot4.v4f16(<4 x half> %a, <4 x half> %b)
+  ret half %dx.dot
+}
+
+; CHECK-LABEL: dot_float2
+define noundef float @dot_float2(<2 x float> noundef %a, <2 x float> noundef %b) {
+entry:
+; CHECK: extractelement <2 x float> %a, i32 0
+; CHECK: extractelement <2 x float> %a, i32 1
+; CHECK: extractelement <2 x float> %b, i32 0
+; CHECK: extractelement <2 x float> %b, i32 1
+; CHECK: call float @dx.op.dot2.f32(i32 54, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}})
+  %dx.dot = call float @llvm.dx.dot2.v2f32(<2 x float> %a, <2 x float> %b)
+  ret float %dx.dot
+}
+
+; CHECK-LABEL: dot_float3
+define noundef float @dot_float3(<3 x float> noundef %a, <3 x float> noundef %b) {
+entry:
+; CHECK: extractelement <3 x float> %a, i32 0
+; CHECK: extractelement <3 x float> %a, i32 1
+; CHECK: extractelement <3 x float> %a, i32 2
+; CHECK: extractelement <3 x float> %b, i32 0
+; CHECK: extractelement <3 x float> %b, i32 1
+; CHECK: extractelement <3 x float> %b, i32 2
+; CHECK: call float @dx.op.dot3.f32(i32 55, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}})
+  %dx.dot = call float @llvm.dx.dot3.v3f32(<3 x float> %a, <3 x float> %b)
+  ret float %dx.dot
+}
+
+; CHECK-LABEL: dot_float4
+define noundef float @dot_float4(<4 x float> noundef %a, <4 x float> noundef %b) {
+entry:
+; CHECK: extractelement <4 x float> %a, i32 0
+; CHECK: extractelement <4 x float> %a, i32 1
+; CHECK: extractelement <4 x float> %a, i32 2
+; CHECK: extractelement <4 x float> %a, i32 3
+; CHECK: extractelement <4 x float> %b, i32 0
+; CHECK: extractelement <4 x float> %b, i32 1
+; CHECK: extractelement <4 x float> %b, i32 2
+; CHECK: extractelement <4 x float> %b, i32 3
+; CHECK: call float @dx.op.dot4.f32(i32 56, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}}, float %{{.*}})
+  %dx.dot = call float @llvm.dx.dot4.v4f32(<4 x float> %a, <4 x float> %b)
+  ret float %dx.dot
+}
+
+declare half  @llvm.dx.dot.v2f16(<2 x half> , <2 x half> )
+declare half  @llvm.dx.dot.v3f16(<3 x half> , <3 x half> )
+declare half  @llvm.dx.dot.v4f16(<4 x half> , <4 x half> )
+declare float @llvm.dx.dot.v2f32(<2 x float>, <2 x float>)
+declare float @llvm.dx.dot.v3f32(<3 x float>, <3 x float>)
+declare float @llvm.dx.dot.v4f32(<4 x float>, <4 x float>)
-- 
GitLab


From 4cea2d049f511d16cbc4605f7ecc908c851e169e Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com>
Date: Mon, 25 Mar 2024 18:02:30 -0400
Subject: [PATCH 206/404] [HLSL][DXIL] implement `sqrt` intrinsic (#86560)

completes #86187
- fix hlsl_intrinsic to cover the correct cases
- move to using `__builtin_elementwise_sqrt`
- add lowering of `Intrinsic::sqrt` to dxilop 24.
---
 clang/lib/Headers/hlsl/hlsl_intrinsics.h  | 28 +++++---
 clang/test/CodeGenHLSL/builtins/sqrt.hlsl | 78 +++++++++++++++--------
 llvm/lib/Target/DirectX/DXIL.td           |  4 ++
 llvm/test/CodeGen/DirectX/sqrt.ll         | 20 ++++++
 llvm/test/CodeGen/DirectX/sqrt_error.ll   | 10 +++
 5 files changed, 105 insertions(+), 35 deletions(-)
 create mode 100644 llvm/test/CodeGen/DirectX/sqrt.ll
 create mode 100644 llvm/test/CodeGen/DirectX/sqrt_error.ll

diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
index d9f27c0db57c..fcb64fde1b91 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
@@ -1366,14 +1366,26 @@ float4 sin(float4);
 /// \param Val The input value.
 
 _HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
-_HLSL_BUILTIN_ALIAS(__builtin_sqrtf16)
-half sqrt(half In);
-
-_HLSL_BUILTIN_ALIAS(__builtin_sqrtf)
-float sqrt(float In);
-
-_HLSL_BUILTIN_ALIAS(__builtin_sqrt)
-double sqrt(double In);
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_sqrt)
+half sqrt(half);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_sqrt)
+half2 sqrt(half2);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_sqrt)
+half3 sqrt(half3);
+_HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_sqrt)
+half4 sqrt(half4);
+
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_sqrt)
+float sqrt(float);
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_sqrt)
+float2 sqrt(float2);
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_sqrt)
+float3 sqrt(float3);
+_HLSL_BUILTIN_ALIAS(__builtin_elementwise_sqrt)
+float4 sqrt(float4);
 
 //===----------------------------------------------------------------------===//
 // trunc builtins
diff --git a/clang/test/CodeGenHLSL/builtins/sqrt.hlsl b/clang/test/CodeGenHLSL/builtins/sqrt.hlsl
index 2c2a09617cf8..adbbf69a8e06 100644
--- a/clang/test/CodeGenHLSL/builtins/sqrt.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/sqrt.hlsl
@@ -1,29 +1,53 @@
-// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -x hlsl -triple \
-// RUN:   dxil-pc-shadermodel6.2-library %s -fnative-half-type \
-// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   dxil-pc-shadermodel6.3-library %s -fnative-half-type \
+// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ 
+// RUN:   --check-prefixes=CHECK,NATIVE_HALF
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   dxil-pc-shadermodel6.3-library %s -emit-llvm -disable-llvm-passes \
+// RUN:   -o - | FileCheck %s --check-prefixes=CHECK,NO_HALF
 
-using hlsl::sqrt;
+// NATIVE_HALF: define noundef half @
+// NATIVE_HALF: %{{.*}} = call half @llvm.sqrt.f16(
+// NATIVE_HALF: ret half %{{.*}}
+// NO_HALF: define noundef float @"?test_sqrt_half@@YA$halff@$halff@@Z"(
+// NO_HALF: %{{.*}} = call float @llvm.sqrt.f32(
+// NO_HALF: ret float %{{.*}}
+half test_sqrt_half(half p0) { return sqrt(p0); }
+// NATIVE_HALF: define noundef <2 x half> @
+// NATIVE_HALF: %{{.*}} = call <2 x half> @llvm.sqrt.v2f16
+// NATIVE_HALF: ret <2 x half> %{{.*}}
+// NO_HALF: define noundef <2 x float> @
+// NO_HALF: %{{.*}} = call <2 x float> @llvm.sqrt.v2f32(
+// NO_HALF: ret <2 x float> %{{.*}}
+half2 test_sqrt_half2(half2 p0) { return sqrt(p0); }
+// NATIVE_HALF: define noundef <3 x half> @
+// NATIVE_HALF: %{{.*}} = call <3 x half> @llvm.sqrt.v3f16
+// NATIVE_HALF: ret <3 x half> %{{.*}}
+// NO_HALF: define noundef <3 x float> @
+// NO_HALF: %{{.*}} = call <3 x float> @llvm.sqrt.v3f32(
+// NO_HALF: ret <3 x float> %{{.*}}
+half3 test_sqrt_half3(half3 p0) { return sqrt(p0); }
+// NATIVE_HALF: define noundef <4 x half> @
+// NATIVE_HALF: %{{.*}} = call <4 x half> @llvm.sqrt.v4f16
+// NATIVE_HALF: ret <4 x half> %{{.*}}
+// NO_HALF: define noundef <4 x float> @
+// NO_HALF: %{{.*}} = call <4 x float> @llvm.sqrt.v4f32(
+// NO_HALF: ret <4 x float> %{{.*}}
+half4 test_sqrt_half4(half4 p0) { return sqrt(p0); }
 
-double sqrt_d(double x)
-{
-  return sqrt(x);
-}
-
-// CHECK: define noundef double @"?sqrt_d@@YANN@Z"(
-// CHECK: call double @llvm.sqrt.f64(double %0)
-
-float sqrt_f(float x)
-{
-  return sqrt(x);
-}
-
-// CHECK: define noundef float @"?sqrt_f@@YAMM@Z"(
-// CHECK: call float @llvm.sqrt.f32(float %0)
-
-half sqrt_h(half x)
-{
-  return sqrt(x);
-}
-
-// CHECK: define noundef half @"?sqrt_h@@YA$f16@$f16@@Z"(
-// CHECK: call half @llvm.sqrt.f16(half %0)
+// CHECK: define noundef float @
+// CHECK: %{{.*}} = call float @llvm.sqrt.f32(
+// CHECK: ret float %{{.*}}
+float test_sqrt_float(float p0) { return sqrt(p0); }
+// CHECK: define noundef <2 x float> @
+// CHECK: %{{.*}} = call <2 x float> @llvm.sqrt.v2f32
+// CHECK: ret <2 x float> %{{.*}}
+float2 test_sqrt_float2(float2 p0) { return sqrt(p0); }
+// CHECK: define noundef <3 x float> @
+// CHECK: %{{.*}} = call <3 x float> @llvm.sqrt.v3f32
+// CHECK: ret <3 x float> %{{.*}}
+float3 test_sqrt_float3(float3 p0) { return sqrt(p0); }
+// CHECK: define noundef <4 x float> @
+// CHECK: %{{.*}} = call <4 x float> @llvm.sqrt.v4f32
+// CHECK: ret <4 x float> %{{.*}}
+float4 test_sqrt_float4(float4 p0) { return sqrt(p0); }
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index 2e6d58e14fd3..b4bc9eda4f50 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -274,6 +274,10 @@ def Frac : DXILOpMapping<22, unary, int_dx_frac,
                          "Returns a fraction from 0 to 1 that represents the "
                          "decimal part of the input.",
                          [llvm_halforfloat_ty, LLVMMatchType<0>]>;
+def Sqrt : DXILOpMapping<24, unary, int_sqrt,
+                         "Returns the square root of the specified floating-point"
+                         "value, per component.",
+                         [llvm_halforfloat_ty, LLVMMatchType<0>]>;
 def RSqrt : DXILOpMapping<25, unary, int_dx_rsqrt,
                          "Returns the reciprocal of the square root of the specified value."
                          "rsqrt(x) = 1 / sqrt(x).",
diff --git a/llvm/test/CodeGen/DirectX/sqrt.ll b/llvm/test/CodeGen/DirectX/sqrt.ll
new file mode 100644
index 000000000000..76a572efd205
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/sqrt.ll
@@ -0,0 +1,20 @@
+; RUN: opt -S -dxil-op-lower < %s | FileCheck %s
+
+; Make sure dxil operation function calls for sqrt are generated for float and half.
+
+define noundef float @sqrt_float(float noundef %a) #0 {
+entry:
+; CHECK:call float @dx.op.unary.f32(i32 24, float %{{.*}})
+  %elt.sqrt = call float @llvm.sqrt.f32(float %a)
+  ret float %elt.sqrt
+}
+
+define noundef half @sqrt_half(half noundef %a) #0 {
+entry:
+; CHECK:call half @dx.op.unary.f16(i32 24, half %{{.*}})
+  %elt.sqrt = call half @llvm.sqrt.f16(half %a)
+  ret half %elt.sqrt
+}
+
+declare half @llvm.sqrt.f16(half)
+declare float @llvm.sqrt.f32(float)
diff --git a/llvm/test/CodeGen/DirectX/sqrt_error.ll b/llvm/test/CodeGen/DirectX/sqrt_error.ll
new file mode 100644
index 000000000000..fffa2e19b80f
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/sqrt_error.ll
@@ -0,0 +1,10 @@
+; RUN: not opt -S -dxil-op-lower %s 2>&1 | FileCheck %s
+
+; DXIL operation sqrt does not support double overload type
+; CHECK: LLVM ERROR: Invalid Overload Type
+
+define noundef double @sqrt_double(double noundef %a) {
+entry:
+  %elt.sqrt = call double @llvm.sqrt.f64(double %a)
+  ret double %elt.sqrt
+}
-- 
GitLab


From de917dc20ece9f23eaefe5354bbc9ca194ce7555 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar@redhat.com>
Date: Mon, 25 Mar 2024 15:04:11 -0700
Subject: [PATCH 207/404]  [workflow] Don't add a comment when the first run of
 the formatter passes (#86335)

This was inadvertently changed in
2120f574103c487787390263b3692c4b167f6bdf.
---
 .github/workflows/pr-code-format.yml | 2 ++
 llvm/utils/git/code-format-helper.py | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pr-code-format.yml b/.github/workflows/pr-code-format.yml
index 2ed9b05cac12..54dfe3aadbb4 100644
--- a/.github/workflows/pr-code-format.yml
+++ b/.github/workflows/pr-code-format.yml
@@ -73,7 +73,9 @@ jobs:
         # to take advantage of the new --diff_from_common_commit option
         # explicitly in code-format-helper.py and not have to diff starting at
         # the merge base.
+        # Create an empty comments file so the pr-write job doesn't fail.
         run: |
+          echo "[]" > comments &&
           python ./code-format-tools/llvm/utils/git/code-format-helper.py \
             --write-comment-to-file \
             --token ${{ secrets.GITHUB_TOKEN }} \
diff --git a/llvm/utils/git/code-format-helper.py b/llvm/utils/git/code-format-helper.py
index af1bb3b5aec5..f1207026704e 100755
--- a/llvm/utils/git/code-format-helper.py
+++ b/llvm/utils/git/code-format-helper.py
@@ -124,7 +124,8 @@ View the diff from {self.name} here.
         existing_comment = self.find_comment(pr)
 
         if args.write_comment_to_file:
-            self.comment = {"body": comment_text}
+            if create_new or existing_comment:
+                self.comment = {"body": comment_text}
             if existing_comment:
                 self.comment["id"] = existing_comment.id
             return
-- 
GitLab


From da385e82518618524d4efc56755662e982bfff68 Mon Sep 17 00:00:00 2001
From: Nathan Lanza <nathanlanza@gmail.com>
Date: Mon, 25 Mar 2024 18:08:36 -0400
Subject: [PATCH 208/404] [aarch64] Unguard GEPOpt from O3

This chunk of code currently runs only if the optimization mode is O3
AND the EnableGEPOpt flag is set. Given that this is the only use case
for the EnableGEPOpt flag, the guarding against O3 is kinda pointless.
IF the user wants to enable it then the flag should be sufficient.

Reviewers: TNorthover, aeubanks

Reviewed By: aeubanks

Pull Request: https://github.com/llvm/llvm-project/pull/86588
---
 llvm/lib/Target/AArch64/AArch64TargetMachine.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 08238fdf167b..7d4a57d792a1 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -603,7 +603,7 @@ void AArch64PassConfig::addIRPasses() {
       addPass(createFalkorMarkStridedAccessesPass());
   }
 
-  if (TM->getOptLevel() == CodeGenOptLevel::Aggressive && EnableGEPOpt) {
+  if (EnableGEPOpt) {
     // Call SeparateConstOffsetFromGEP pass to extract constants within indices
     // and lower a GEP with multiple indices to either arithmetic operations or
     // multiple GEPs with single index.
-- 
GitLab


From 1b763f230a83a15b3d055295b2b7113c5c2647d4 Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Mon, 25 Mar 2024 15:14:33 -0700
Subject: [PATCH 209/404] [BOLT] Add secondary entry points to BAT

Provide secondary entry points for `EntryDiscriminator` call info field
in YAML profile.

Increases BAT section size to:
- large binary: 39655300 bytes (1.03x the original),
- medium binary: 3834328 bytes (0.65x),
- small binary: 924 bytes (0.64x).

Depends on: https://github.com/llvm/llvm-project/pull/76911

Test Plan:
- Updated bolt-address-translation{,-yaml}.test
- Added openssl test: https://github.com/rafaelauler/bolt-tests/pull/30

Reviewers: dcci, rafaelauler, maksfb, ayermolo

Reviewed By: rafaelauler

Pull Request: https://github.com/llvm/llvm-project/pull/86218
---
 bolt/docs/BAT.md                              | 61 +++++++++++--------
 .../bolt/Profile/BoltAddressTranslation.h     |  3 +
 bolt/lib/Profile/BoltAddressTranslation.cpp   | 59 +++++++++++++++++-
 .../X86/bolt-address-translation-yaml.test    |  2 +-
 bolt/test/X86/bolt-address-translation.test   |  2 +-
 5 files changed, 97 insertions(+), 30 deletions(-)

diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md
index 2279a070263e..f23ef1abf876 100644
--- a/bolt/docs/BAT.md
+++ b/bolt/docs/BAT.md
@@ -42,21 +42,21 @@ and [BoltAddressTranslation.cpp](/bolt/lib/Profile/BoltAddressTranslation.cpp).
 ### Layout
 The general layout is as follows:
 ```
-Hot functions table header
-|------------------|
-|  Function entry  |
-| |--------------| |
-| | OutOff InOff | |
-| |--------------| |
-~~~~~~~~~~~~~~~~~~~~
+Hot functions table
+Cold functions table
 
-Cold functions table header
+Functions table:
 |------------------|
 |  Function entry  |
-| |--------------| |
-| | OutOff InOff | |
-| |--------------| |
-~~~~~~~~~~~~~~~~~~~~
+|                  |
+|     Address      |
+|   translation    |
+|      table       |
+|                  |
+| Secondary entry  |
+|      points      |
+|------------------|
+
 ```
 
 ### Functions table
@@ -74,19 +74,20 @@ internal offsets, and between hot and cold fragments, to better spread deltas
 and save space.
 
 Hot indices are delta encoded, implicitly starting at zero.
-| Entry  | Encoding | Description |
-| ------ | ------| ----------- |
-| `Address` | Continuous, Delta, ULEB128 | Function address in the output binary |
-| `HotIndex` | Delta, ULEB128 | Cold functions only: index of corresponding hot function in hot functions table |
-| `FuncHash` | 8b | Hot functions only: function hash for input function |
-| `NumBlocks` | ULEB128 | Hot functions only: number of basic blocks in the original function |
-| `NumEntries` | ULEB128 | Number of address translation entries for a function |
-| `EqualElems` | ULEB128 | Hot functions only: number of equal offsets in the beginning of a function |
-| `BranchEntries` | Bitmask, `alignTo(EqualElems, 8)` bits | Hot functions only: if `EqualElems` is non-zero, bitmask denoting entries with `BRANCHENTRY` bit |
-
-Function header is followed by `EqualElems` offsets (hot functions only) and
-`NumEntries-EqualElems` (`NumEntries` for cold functions) pairs of offsets for
-current function.
+| Entry  | Encoding | Description | Hot/Cold |
+| ------ | ------| ----------- | ------ |
+| `Address` | Continuous, Delta, ULEB128 | Function address in the output binary | Both |
+| `HotIndex` | Delta, ULEB128 | Index of corresponding hot function in hot functions table | Cold |
+| `FuncHash` | 8b | Function hash for input function | Hot |
+| `NumBlocks` | ULEB128 | Number of basic blocks in the original function | Hot |
+| `NumSecEntryPoints` | ULEB128 | Number of secondary entry points in the original function | Hot |
+| `NumEntries` | ULEB128 | Number of address translation entries for a function | Both |
+| `EqualElems` | ULEB128 | Number of equal offsets in the beginning of a function | Hot |
+| `BranchEntries` | Bitmask, `alignTo(EqualElems, 8)` bits | If `EqualElems` is non-zero, bitmask denoting entries with `BRANCHENTRY` bit | Hot |
+
+Function header is followed by *Address Translation Table* with `NumEntries`
+total entries, and *Secondary Entry Points* table with `NumSecEntryPoints`
+entries (hot functions only).
 
 ### Address translation table
 Delta encoding means that only the difference with the previous corresponding
@@ -98,8 +99,18 @@ entry is encoded. Input offsets implicitly start at zero.
 | `BBHash` | Optional, 8b | Basic block hash in input binary | BB |
 | `BBIdx`  | Optional, Delta, ULEB128 | Basic block index in input binary | BB |
 
+For hot fragments, the table omits the first `EqualElems` input offsets
+where the input offset equals output offset.
+
 `BRANCHENTRY` bit denotes whether a given offset pair is a control flow source
 (branch or call instruction). If not set, it signifies a control flow target
 (basic block offset).
 `InputAddr` is omitted for equal offsets in input and output function. In this
 case, `BRANCHENTRY` bits are encoded separately in a `BranchEntries` bitvector.
+
+### Secondary Entry Points table
+The table is emitted for hot fragments only. It contains `NumSecEntryPoints`
+offsets denoting secondary entry points, delta encoded, implicitly starting at zero.
+| Entry | Encoding | Description |
+| ----- | -------- | ----------- |
+| `SecEntryPoint` | Delta, ULEB128 | Secondary entry point offset |
diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h
index 51fdadce8085..bade0b57bf5c 100644
--- a/bolt/include/bolt/Profile/BoltAddressTranslation.h
+++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h
@@ -150,6 +150,9 @@ private:
   /// Map a function to its basic blocks count
   std::unordered_map<uint64_t, size_t> NumBasicBlocksMap;
 
+  /// Map a function to its secondary entry points vector
+  std::unordered_map<uint64_t, std::vector<uint32_t>> SecondaryEntryPointsMap;
+
   /// Links outlined cold bocks to their original function
   std::map<uint64_t, uint64_t> ColdPartSource;
 
diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp
index 57d2aff71977..aa4cde27e75c 100644
--- a/bolt/lib/Profile/BoltAddressTranslation.cpp
+++ b/bolt/lib/Profile/BoltAddressTranslation.cpp
@@ -88,14 +88,21 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) {
     if (Function.isIgnored() || (!BC.HasRelocations && !Function.isSimple()))
       continue;
 
-    // TBD: handle BAT functions w/multiple entry points.
-    if (Function.isMultiEntry())
-      continue;
+    uint32_t NumSecondaryEntryPoints = 0;
+    Function.forEachEntryPoint([&](uint64_t Offset, const MCSymbol *) {
+      if (!Offset)
+        return true;
+      ++NumSecondaryEntryPoints;
+      SecondaryEntryPointsMap[OutputAddress].push_back(Offset);
+      return true;
+    });
 
     LLVM_DEBUG(dbgs() << "Function name: " << Function.getPrintName() << "\n");
     LLVM_DEBUG(dbgs() << " Address reference: 0x"
                       << Twine::utohexstr(Function.getOutputAddress()) << "\n");
     LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n", getBFHash(OutputAddress)));
+    LLVM_DEBUG(dbgs() << " Secondary Entry Points: " << NumSecondaryEntryPoints
+                      << '\n');
 
     MapTy Map;
     for (const BinaryBasicBlock *const BB :
@@ -185,6 +192,10 @@ void BoltAddressTranslation::writeMaps(std::map<uint64_t, MapTy> &Maps,
                       << Twine::utohexstr(Address) << ".\n");
     encodeULEB128(Address - PrevAddress, OS);
     PrevAddress = Address;
+    const uint32_t NumSecondaryEntryPoints =
+        SecondaryEntryPointsMap.count(Address)
+            ? SecondaryEntryPointsMap[Address].size()
+            : 0;
     if (Cold) {
       size_t HotIndex =
           std::distance(ColdPartSource.begin(), ColdPartSource.find(Address));
@@ -199,6 +210,10 @@ void BoltAddressTranslation::writeMaps(std::map<uint64_t, MapTy> &Maps,
       size_t NumBasicBlocks = getBBHashMap(HotInputAddress).getNumBasicBlocks();
       LLVM_DEBUG(dbgs() << "Basic blocks: " << NumBasicBlocks << '\n');
       encodeULEB128(NumBasicBlocks, OS);
+      // Secondary entry points
+      encodeULEB128(NumSecondaryEntryPoints, OS);
+      LLVM_DEBUG(dbgs() << "Secondary Entry Points: " << NumSecondaryEntryPoints
+                        << '\n');
     }
     encodeULEB128(NumEntries, OS);
     // For hot fragments only: encode the number of equal offsets
@@ -244,6 +259,17 @@ void BoltAddressTranslation::writeMaps(std::map<uint64_t, MapTy> &Maps,
                                      InOffset >> 1, BBHash, BBIndex));
       }
     }
+    uint32_t PrevOffset = 0;
+    if (!Cold && NumSecondaryEntryPoints) {
+      LLVM_DEBUG(dbgs() << "Secondary entry points: ");
+      // Secondary entry point offsets, delta-encoded
+      for (uint32_t Offset : SecondaryEntryPointsMap[Address]) {
+        encodeULEB128(Offset - PrevOffset, OS);
+        LLVM_DEBUG(dbgs() << formatv("{0:x} ", Offset));
+        PrevOffset = Offset;
+      }
+      LLVM_DEBUG(dbgs() << '\n');
+    }
   }
 }
 
@@ -287,6 +313,7 @@ void BoltAddressTranslation::parseMaps(std::vector<uint64_t> &HotFuncs,
     const uint64_t Address = PrevAddress + DE.getULEB128(&Offset, &Err);
     uint64_t HotAddress = Cold ? 0 : Address;
     PrevAddress = Address;
+    uint32_t SecondaryEntryPoints = 0;
     if (Cold) {
       HotIndex += DE.getULEB128(&Offset, &Err);
       HotAddress = HotFuncs[HotIndex];
@@ -303,6 +330,12 @@ void BoltAddressTranslation::parseMaps(std::vector<uint64_t> &HotFuncs,
       LLVM_DEBUG(dbgs() << formatv("{0:x}: #bbs {1}, {2} bytes\n", Address,
                                    NumBasicBlocks,
                                    getULEB128Size(NumBasicBlocks)));
+      // Secondary entry points
+      SecondaryEntryPoints = DE.getULEB128(&Offset, &Err);
+      LLVM_DEBUG(
+          dbgs() << formatv("{0:x}: secondary entry points {1}, {2} bytes\n",
+                            Address, SecondaryEntryPoints,
+                            getULEB128Size(SecondaryEntryPoints)));
     }
     const uint32_t NumEntries = DE.getULEB128(&Offset, &Err);
     // Equal offsets, hot fragments only.
@@ -370,6 +403,19 @@ void BoltAddressTranslation::parseMaps(std::vector<uint64_t> &HotFuncs,
       });
     }
     Maps.insert(std::pair<uint64_t, MapTy>(Address, Map));
+    if (!Cold && SecondaryEntryPoints) {
+      uint32_t EntryPointOffset = 0;
+      LLVM_DEBUG(dbgs() << "Secondary entry points: ");
+      for (uint32_t EntryPointId = 0; EntryPointId != SecondaryEntryPoints;
+           ++EntryPointId) {
+        uint32_t OffsetDelta = DE.getULEB128(&Offset, &Err);
+        EntryPointOffset += OffsetDelta;
+        SecondaryEntryPointsMap[Address].push_back(EntryPointOffset);
+        LLVM_DEBUG(dbgs() << formatv("{0:x}/{1}b ", EntryPointOffset,
+                                     getULEB128Size(OffsetDelta)));
+      }
+      LLVM_DEBUG(dbgs() << '\n');
+    }
   }
 }
 
@@ -397,6 +443,13 @@ void BoltAddressTranslation::dump(raw_ostream &OS) {
         OS << formatv(" hash: {0:x}", BBHashMap.getBBHash(Val));
       OS << "\n";
     }
+    if (SecondaryEntryPointsMap.count(Address)) {
+      const std::vector<uint32_t> &SecondaryEntryPoints =
+          SecondaryEntryPointsMap[Address];
+      OS << SecondaryEntryPoints.size() << " secondary entry points:\n";
+      for (uint32_t EntryPointOffset : SecondaryEntryPoints)
+        OS << formatv("{0:x}\n", EntryPointOffset);
+    }
     OS << "\n";
   }
   const size_t NumColdParts = ColdPartSource.size();
diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test
index be3bcb2a0c6d..306050a1c97e 100644
--- a/bolt/test/X86/bolt-address-translation-yaml.test
+++ b/bolt/test/X86/bolt-address-translation-yaml.test
@@ -18,7 +18,7 @@ RUN:   | FileCheck --check-prefix CHECK-BOLT-YAML %s
 
 WRITE-BAT-CHECK: BOLT-INFO: Wrote 5 BAT maps
 WRITE-BAT-CHECK: BOLT-INFO: Wrote 4 function and 22 basic block hashes
-WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 380
+WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 384
 
 READ-BAT-CHECK-NOT: BOLT-ERROR: unable to save profile in YAML format for input file processed by BOLT
 READ-BAT-CHECK: BOLT-INFO: Parsed 5 BAT entries
diff --git a/bolt/test/X86/bolt-address-translation.test b/bolt/test/X86/bolt-address-translation.test
index 5c1db89e3c6b..63234b4c1d21 100644
--- a/bolt/test/X86/bolt-address-translation.test
+++ b/bolt/test/X86/bolt-address-translation.test
@@ -37,7 +37,7 @@
 # CHECK:      BOLT: 3 out of 7 functions were overwritten.
 # CHECK:      BOLT-INFO: Wrote 6 BAT maps
 # CHECK:      BOLT-INFO: Wrote 3 function and 58 basic block hashes
-# CHECK:      BOLT-INFO: BAT section size (bytes): 920
+# CHECK:      BOLT-INFO: BAT section size (bytes): 924
 #
 # usqrt mappings (hot part). We match against any key (left side containing
 # the bolted binary offsets) because BOLT may change where it puts instructions
-- 
GitLab


From 2c76e88e9eb284d17cf409851fb01f1d583bb22a Mon Sep 17 00:00:00 2001
From: jimingham <jingham@apple.com>
Date: Mon, 25 Mar 2024 15:17:23 -0700
Subject: [PATCH 211/404] Add register lookup as another fallback computation
 for address-expressions (#85492)

The idea behind the address-expression is that it handles all the common
expressions that produce addresses. It handles actual valid expressions
that return a scalar, and it handles useful cases that the various
source languages don't support. At present, the fallback handles:

<symbol_name>{+-}<offset>

which isn't valid C but is very handy.

This patch adds handling of:

$<reg_name>

and

$<reg_name>{+-}<offset>

That's kind of pointless in C because the C expression parser handles
that expression already. But some languages don't have a straightforward
way to represent register values like this (swift) so having this
fallback is quite a quality of life improvement.

I added a test which tests that I didn't mess up either of these
fallbacks, though it doesn't test the actually handling of registers
that I added, since the expression parser for C succeeds in that case
and returns before this code gets run.

I will add a test on the swift fork for that checks that this works the
same way for a swift frame after this check.
---
 lldb/source/Interpreter/OptionArgParser.cpp   | 60 ++++++++++++++++---
 .../commands/target/modules/lookup/Makefile   |  4 ++
 .../lookup/TestImageLookupPCExpression.py     | 27 +++++++++
 .../API/commands/target/modules/lookup/main.c | 15 +++++
 4 files changed, 99 insertions(+), 7 deletions(-)
 create mode 100644 lldb/test/API/commands/target/modules/lookup/Makefile
 create mode 100644 lldb/test/API/commands/target/modules/lookup/TestImageLookupPCExpression.py
 create mode 100644 lldb/test/API/commands/target/modules/lookup/main.c

diff --git a/lldb/source/Interpreter/OptionArgParser.cpp b/lldb/source/Interpreter/OptionArgParser.cpp
index 75ccad87467e..9a8275128ede 100644
--- a/lldb/source/Interpreter/OptionArgParser.cpp
+++ b/lldb/source/Interpreter/OptionArgParser.cpp
@@ -9,7 +9,9 @@
 #include "lldb/Interpreter/OptionArgParser.h"
 #include "lldb/DataFormatters/FormatManager.h"
 #include "lldb/Target/ABI.h"
+#include "lldb/Target/RegisterContext.h"
 #include "lldb/Target/Target.h"
+#include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/StreamString.h"
 
@@ -233,24 +235,68 @@ OptionArgParser::DoToAddress(const ExecutionContext *exe_ctx, llvm::StringRef s,
   // Since the compiler can't handle things like "main + 12" we should try to
   // do this for now. The compiler doesn't like adding offsets to function
   // pointer types.
+  // Some languages also don't have a natural representation for register
+  // values (e.g. swift) so handle simple uses of them here as well.
+  // We use a regex to parse these forms, the regex handles:
+  // $reg_name
+  // $reg_name+offset
+  // symbol_name+offset
+  //
+  // The important matching elements in the regex below are:
+  // 1: The reg name if there's no +offset
+  // 3: The symbol/reg name if there is an offset
+  // 4: +/-
+  // 5: The offset value.
   static RegularExpression g_symbol_plus_offset_regex(
-      "^(.*)([-\\+])[[:space:]]*(0x[0-9A-Fa-f]+|[0-9]+)[[:space:]]*$");
+      "^(\\$[^ +-]+)|(([^ +-]+)([-\\+])[[:space:]]*(0x[0-9A-Fa-f]+|[0-9]+)[[:space:]]*)$");
 
   llvm::SmallVector<llvm::StringRef, 4> matches;
   if (g_symbol_plus_offset_regex.Execute(sref, &matches)) {
     uint64_t offset = 0;
-    llvm::StringRef name = matches[1];
-    llvm::StringRef sign = matches[2];
-    llvm::StringRef str_offset = matches[3];
-    if (!str_offset.getAsInteger(0, offset)) {
+    llvm::StringRef name;
+    if (!matches[1].empty())
+      name = matches[1];
+    else
+      name = matches[3];
+
+    llvm::StringRef sign = matches[4];
+    llvm::StringRef str_offset = matches[5];
+
+    // Some languages don't have a natural type for register values, but it
+    // is still useful to look them up here:
+    std::optional<lldb::addr_t> register_value;
+    StackFrame *frame = exe_ctx->GetFramePtr();
+    llvm::StringRef reg_name = name;
+    if (frame && reg_name.consume_front("$")) {
+      RegisterContextSP reg_ctx_sp = frame->GetRegisterContext();
+      if (reg_ctx_sp) {
+        const RegisterInfo *reg_info = reg_ctx_sp->GetRegisterInfoByName(reg_name);
+        if (reg_info) {
+          RegisterValue reg_val;
+          bool success = reg_ctx_sp->ReadRegister(reg_info, reg_val);
+          if (success && reg_val.GetType() != RegisterValue::eTypeInvalid) {
+            register_value = reg_val.GetAsUInt64(0, &success);
+            if (!success)
+              register_value.reset();
+          }
+        }
+      } 
+    }
+    if (!str_offset.empty() && !str_offset.getAsInteger(0, offset)) {
       Status error;
-      addr = ToAddress(exe_ctx, name, LLDB_INVALID_ADDRESS, &error);
+      if (register_value)
+        addr = register_value.value();
+      else
+        addr = ToAddress(exe_ctx, name, LLDB_INVALID_ADDRESS, &error);
       if (addr != LLDB_INVALID_ADDRESS) {
         if (sign[0] == '+')
           return addr + offset;
         return addr - offset;
       }
-    }
+    } else if (register_value)
+      // In the case of register values, someone might just want to get the 
+      // value in a language whose expression parser doesn't support registers.
+      return register_value.value();
   }
 
   if (error_ptr)
diff --git a/lldb/test/API/commands/target/modules/lookup/Makefile b/lldb/test/API/commands/target/modules/lookup/Makefile
new file mode 100644
index 000000000000..695335e068c0
--- /dev/null
+++ b/lldb/test/API/commands/target/modules/lookup/Makefile
@@ -0,0 +1,4 @@
+C_SOURCES := main.c
+CFLAGS_EXTRAS := -std=c99
+
+include Makefile.rules
diff --git a/lldb/test/API/commands/target/modules/lookup/TestImageLookupPCExpression.py b/lldb/test/API/commands/target/modules/lookup/TestImageLookupPCExpression.py
new file mode 100644
index 000000000000..9872e057cbbf
--- /dev/null
+++ b/lldb/test/API/commands/target/modules/lookup/TestImageLookupPCExpression.py
@@ -0,0 +1,27 @@
+"""
+Make sure that "target modules lookup -va $pc" works
+"""
+
+
+import lldb
+import lldbsuite.test.lldbutil as lldbutil
+from lldbsuite.test.lldbtest import *
+
+
+class TestImageLookupPCInC(TestBase):
+    def test_sample_rename_this(self):
+        """There can be many tests in a test case - describe this test here."""
+        self.build()
+        self.main_source_file = lldb.SBFileSpec("main.c")
+        self.sample_test()
+
+    def sample_test(self):
+        """Make sure the address expression resolves to the right function"""
+
+        (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
+            self, "Set a breakpoint here", self.main_source_file
+        )
+
+        self.expect("target modules lookup -va $pc", substrs=["doSomething"])
+        self.expect("target modules lookup -va $pc+4", substrs=["doSomething"])
+
diff --git a/lldb/test/API/commands/target/modules/lookup/main.c b/lldb/test/API/commands/target/modules/lookup/main.c
new file mode 100644
index 000000000000..afe962f30916
--- /dev/null
+++ b/lldb/test/API/commands/target/modules/lookup/main.c
@@ -0,0 +1,15 @@
+#include <stdio.h>
+
+void
+doSomething()
+{
+  printf ("Set a breakpoint here.\n");
+  printf ("Need a bit more code.\n");
+}
+
+int
+main()
+{
+  doSomething();
+  return 0;
+}
-- 
GitLab


From 6708beb35792b5ccd6b2143b3396bf261acce676 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 25 Mar 2024 14:52:53 -0700
Subject: [PATCH 212/404] [RISCV] Remove more unneeded template parameters from
 RISCVInstrInfoZvk.td. NFC

---
 llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
index 09e61c10a57b..617ca3242cc7 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
@@ -269,24 +269,24 @@ multiclass VPseudoVALU_VV_NoMask_Zvk {
   }
 }
 
-multiclass VPseudoVALU_VI_NoMask_Zvk<Operand ImmType = simm5> {
+multiclass VPseudoVALU_VI_NoMask_Zvk {
   foreach m = MxListVF4 in {
     defvar mx = m.MX;
     defvar WriteVIALUV_MX = !cast<SchedWrite>("WriteVIALUV_" # mx);
     defvar ReadVIALUV_MX = !cast<SchedRead>("ReadVIALUV_" # mx);
 
-    defm _VI : VPseudoBinaryNoMask_Zvk<m.vrclass, m.vrclass, ImmType, m>,
+    defm _VI : VPseudoBinaryNoMask_Zvk<m.vrclass, m.vrclass, uimm5, m>,
                Sched<[WriteVIALUV_MX, ReadVIALUV_MX, ReadVIALUV_MX, ReadVMask]>;
   }
 }
 
-multiclass VPseudoVALU_VI_NoMaskTU_Zvk<Operand ImmType = uimm5> {
+multiclass VPseudoVALU_VI_NoMaskTU_Zvk {
   foreach m = MxListVF4 in {
     defvar mx = m.MX;
     defvar WriteVIALUV_MX = !cast<SchedWrite>("WriteVIALUV_" # mx);
     defvar ReadVIALUV_MX = !cast<SchedRead>("ReadVIALUV_" # mx);
 
-    defm _VI : VPseudoBinaryNoMask<m.vrclass, m.vrclass, ImmType, m>,
+    defm _VI : VPseudoBinaryNoMask<m.vrclass, m.vrclass, uimm5, m>,
                Sched<[WriteVIALUV_MX, ReadVIALUV_MX, ReadVIALUV_MX, ReadVMask]>;
   }
 }
@@ -369,7 +369,7 @@ let Predicates = [HasStdExtZvkned] in {
   defm PseudoVAESEF  : VPseudoVALU_V_S_NoMask_Zvk;
   defm PseudoVAESEM  : VPseudoVALU_V_S_NoMask_Zvk;
   defm PseudoVAESKF1 : VPseudoVALU_VI_NoMaskTU_Zvk;
-  defm PseudoVAESKF2 : VPseudoVALU_VI_NoMask_Zvk<uimm5>;
+  defm PseudoVAESKF2 : VPseudoVALU_VI_NoMask_Zvk;
   defm PseudoVAESZ   : VPseudoVALU_S_NoMask_Zvk;
 } // Predicates = [HasStdExtZvkned]
 
@@ -385,7 +385,7 @@ let Predicates = [HasStdExtZvksed] in {
 } // Predicates = [HasStdExtZvksed]
 
 let Predicates = [HasStdExtZvksh] in {
-  defm PseudoVSM3C  : VPseudoVALU_VI_NoMask_Zvk<uimm5>;
+  defm PseudoVSM3C  : VPseudoVALU_VI_NoMask_Zvk;
   defm PseudoVSM3ME : VPseudoVALU_VV_NoMaskTU_Zvk;
 } // Predicates = [HasStdExtZvksh]
 
-- 
GitLab


From 4dcb1db44f9dbfa09c220703a1b097f51d20a2a5 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Mon, 25 Mar 2024 15:25:58 -0700
Subject: [PATCH 213/404] Revert "[lldb] Implement coalescing of disjoint
 progress events (#84854)"

This reverts commit 930f64689c1fb487714c3836ffa43e49e46aa488 as it's
failing on the Linux bots.
---
 lldb/include/lldb/Core/Progress.h             |  35 +-----
 lldb/source/Core/Progress.cpp                 | 114 ++++--------------
 .../SystemInitializerCommon.cpp               |   3 -
 lldb/unittests/Core/ProgressReportTest.cpp    |  39 +-----
 4 files changed, 32 insertions(+), 159 deletions(-)

diff --git a/lldb/include/lldb/Core/Progress.h b/lldb/include/lldb/Core/Progress.h
index cd87be79c4f0..eb3af9816dc6 100644
--- a/lldb/include/lldb/Core/Progress.h
+++ b/lldb/include/lldb/Core/Progress.h
@@ -9,7 +9,6 @@
 #ifndef LLDB_CORE_PROGRESS_H
 #define LLDB_CORE_PROGRESS_H
 
-#include "lldb/Host/Alarm.h"
 #include "lldb/lldb-forward.h"
 #include "lldb/lldb-types.h"
 #include "llvm/ADT/StringMap.h"
@@ -151,12 +150,9 @@ public:
   void Increment(const Progress::ProgressData &);
   void Decrement(const Progress::ProgressData &);
 
-  static void Initialize();
-  static void Terminate();
-  static bool Enabled();
   static ProgressManager &Instance();
 
-protected:
+private:
   enum class EventType {
     Begin,
     End,
@@ -164,32 +160,9 @@ protected:
   static void ReportProgress(const Progress::ProgressData &progress_data,
                              EventType type);
 
-  static std::optional<ProgressManager> &InstanceImpl();
-
-  /// Helper function for reporting progress when the alarm in the corresponding
-  /// entry in the map expires.
-  void Expire(llvm::StringRef key);
-
-  /// Entry used for bookkeeping.
-  struct Entry {
-    /// Reference count used for overlapping events.
-    uint64_t refcount = 0;
-
-    /// Data used to emit progress events.
-    Progress::ProgressData data;
-
-    /// Alarm handle used when the refcount reaches zero.
-    Alarm::Handle handle = Alarm::INVALID_HANDLE;
-  };
-
-  /// Map used for bookkeeping.
-  llvm::StringMap<Entry> m_entries;
-
-  /// Mutex to provide the map.
-  std::mutex m_entries_mutex;
-
-  /// Alarm instance to coalesce progress events.
-  Alarm m_alarm;
+  llvm::StringMap<std::pair<uint64_t, Progress::ProgressData>>
+      m_progress_category_map;
+  std::mutex m_progress_map_mutex;
 };
 
 } // namespace lldb_private
diff --git a/lldb/source/Core/Progress.cpp b/lldb/source/Core/Progress.cpp
index 161038284e21..b4b5e98b7ba4 100644
--- a/lldb/source/Core/Progress.cpp
+++ b/lldb/source/Core/Progress.cpp
@@ -35,10 +35,7 @@ Progress::Progress(std::string title, std::string details,
 
   std::lock_guard<std::mutex> guard(m_mutex);
   ReportProgress();
-
-  // Report to the ProgressManager if that subsystem is enabled.
-  if (ProgressManager::Enabled())
-    ProgressManager::Instance().Increment(m_progress_data);
+  ProgressManager::Instance().Increment(m_progress_data);
 }
 
 Progress::~Progress() {
@@ -48,10 +45,7 @@ Progress::~Progress() {
   if (!m_completed)
     m_completed = m_total;
   ReportProgress();
-
-  // Report to the ProgressManager if that subsystem is enabled.
-  if (ProgressManager::Enabled())
-    ProgressManager::Instance().Decrement(m_progress_data);
+  ProgressManager::Instance().Decrement(m_progress_data);
 }
 
 void Progress::Increment(uint64_t amount,
@@ -81,84 +75,45 @@ void Progress::ReportProgress() {
   }
 }
 
-ProgressManager::ProgressManager()
-    : m_entries(), m_alarm(std::chrono::milliseconds(100)) {}
+ProgressManager::ProgressManager() : m_progress_category_map() {}
 
 ProgressManager::~ProgressManager() {}
 
-void ProgressManager::Initialize() {
-  assert(!InstanceImpl() && "Already initialized.");
-  InstanceImpl().emplace();
-}
-
-void ProgressManager::Terminate() {
-  assert(InstanceImpl() && "Already terminated.");
-  InstanceImpl().reset();
-}
-
-bool ProgressManager::Enabled() { return InstanceImpl().operator bool(); }
-
 ProgressManager &ProgressManager::Instance() {
-  assert(InstanceImpl() && "ProgressManager must be initialized");
-  return *InstanceImpl();
-}
-
-std::optional<ProgressManager> &ProgressManager::InstanceImpl() {
-  static std::optional<ProgressManager> g_progress_manager;
-  return g_progress_manager;
+  static std::once_flag g_once_flag;
+  static ProgressManager *g_progress_manager = nullptr;
+  std::call_once(g_once_flag, []() {
+    // NOTE: known leak to avoid global destructor chain issues.
+    g_progress_manager = new ProgressManager();
+  });
+  return *g_progress_manager;
 }
 
 void ProgressManager::Increment(const Progress::ProgressData &progress_data) {
-  std::lock_guard<std::mutex> lock(m_entries_mutex);
-
-  llvm::StringRef key = progress_data.title;
-  bool new_entry = !m_entries.contains(key);
-  Entry &entry = m_entries[progress_data.title];
-
-  if (new_entry) {
-    // This is a new progress event. Report progress and store the progress
-    // data.
+  std::lock_guard<std::mutex> lock(m_progress_map_mutex);
+  // If the current category exists in the map then it is not an initial report,
+  // therefore don't broadcast to the category bit. Also, store the current
+  // progress data in the map so that we have a note of the ID used for the
+  // initial progress report.
+  if (!m_progress_category_map.contains(progress_data.title)) {
+    m_progress_category_map[progress_data.title].second = progress_data;
     ReportProgress(progress_data, EventType::Begin);
-    entry.data = progress_data;
-  } else if (entry.refcount == 0) {
-    // This is an existing entry that was scheduled to be deleted but a new one
-    // came in before the timer expired.
-    assert(entry.handle != Alarm::INVALID_HANDLE);
-
-    if (!m_alarm.Cancel(entry.handle)) {
-      // The timer expired before we had a chance to cancel it. We have to treat
-      // this as an entirely new progress event.
-      ReportProgress(progress_data, EventType::Begin);
-    }
-    // Clear the alarm handle.
-    entry.handle = Alarm::INVALID_HANDLE;
   }
-
-  // Regardless of how we got here, we need to bump the reference count.
-  entry.refcount++;
+  m_progress_category_map[progress_data.title].first++;
 }
 
 void ProgressManager::Decrement(const Progress::ProgressData &progress_data) {
-  std::lock_guard<std::mutex> lock(m_entries_mutex);
-  llvm::StringRef key = progress_data.title;
+  std::lock_guard<std::mutex> lock(m_progress_map_mutex);
+  auto pos = m_progress_category_map.find(progress_data.title);
 
-  if (!m_entries.contains(key))
+  if (pos == m_progress_category_map.end())
     return;
 
-  Entry &entry = m_entries[key];
-  entry.refcount--;
-
-  if (entry.refcount == 0) {
-    assert(entry.handle == Alarm::INVALID_HANDLE);
-
-    // Copy the key to a std::string so we can pass it by value to the lambda.
-    // The underlying StringRef will not exist by the time the callback is
-    // called.
-    std::string key_str = std::string(key);
-
-    // Start a timer. If it expires before we see another progress event, it
-    // will be reported.
-    entry.handle = m_alarm.Create([=]() { Expire(key_str); });
+  if (pos->second.first <= 1) {
+    ReportProgress(pos->second.second, EventType::End);
+    m_progress_category_map.erase(progress_data.title);
+  } else {
+    --pos->second.first;
   }
 }
 
@@ -174,20 +129,3 @@ void ProgressManager::ReportProgress(
                            progress_data.debugger_id,
                            Debugger::eBroadcastBitProgressCategory);
 }
-
-void ProgressManager::Expire(llvm::StringRef key) {
-  std::lock_guard<std::mutex> lock(m_entries_mutex);
-
-  // This shouldn't happen but be resilient anyway.
-  if (!m_entries.contains(key))
-    return;
-
-  // A new event came in and the alarm fired before we had a chance to restart
-  // it.
-  if (m_entries[key].refcount != 0)
-    return;
-
-  // We're done with this entry.
-  ReportProgress(m_entries[key].data, EventType::End);
-  m_entries.erase(key);
-}
diff --git a/lldb/source/Initialization/SystemInitializerCommon.cpp b/lldb/source/Initialization/SystemInitializerCommon.cpp
index 5d35854211c2..1a172a95aa14 100644
--- a/lldb/source/Initialization/SystemInitializerCommon.cpp
+++ b/lldb/source/Initialization/SystemInitializerCommon.cpp
@@ -9,7 +9,6 @@
 #include "lldb/Initialization/SystemInitializerCommon.h"
 
 #include "Plugins/Process/gdb-remote/ProcessGDBRemoteLog.h"
-#include "lldb/Core/Progress.h"
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Host/Host.h"
 #include "lldb/Host/Socket.h"
@@ -70,7 +69,6 @@ llvm::Error SystemInitializerCommon::Initialize() {
   Diagnostics::Initialize();
   FileSystem::Initialize();
   HostInfo::Initialize(m_shlib_dir_helper);
-  ProgressManager::Initialize();
 
   llvm::Error error = Socket::Initialize();
   if (error)
@@ -99,7 +97,6 @@ void SystemInitializerCommon::Terminate() {
 #endif
 
   Socket::Terminate();
-  ProgressManager::Terminate();
   HostInfo::Terminate();
   Log::DisableAllLogChannels();
   FileSystem::Terminate();
diff --git a/lldb/unittests/Core/ProgressReportTest.cpp b/lldb/unittests/Core/ProgressReportTest.cpp
index f0d253be9bf6..1f993180fd83 100644
--- a/lldb/unittests/Core/ProgressReportTest.cpp
+++ b/lldb/unittests/Core/ProgressReportTest.cpp
@@ -22,7 +22,7 @@
 using namespace lldb;
 using namespace lldb_private;
 
-static std::chrono::milliseconds TIMEOUT(500);
+static std::chrono::milliseconds TIMEOUT(100);
 
 class ProgressReportTest : public ::testing::Test {
 public:
@@ -56,8 +56,7 @@ protected:
 
   DebuggerSP m_debugger_sp;
   ListenerSP m_listener_sp;
-  SubsystemRAII<FileSystem, HostInfo, PlatformMacOSX, ProgressManager>
-      subsystems;
+  SubsystemRAII<FileSystem, HostInfo, PlatformMacOSX> subsystems;
 };
 
 TEST_F(ProgressReportTest, TestReportCreation) {
@@ -211,37 +210,3 @@ TEST_F(ProgressReportTest, TestOverlappingEvents) {
   // initial report.
   EXPECT_EQ(data->GetID(), expected_progress_id);
 }
-
-TEST_F(ProgressReportTest, TestProgressManagerDisjointReports) {
-  ListenerSP listener_sp =
-      CreateListenerFor(Debugger::eBroadcastBitProgressCategory);
-  EventSP event_sp;
-  const ProgressEventData *data;
-  uint64_t expected_progress_id;
-
-  { Progress progress("Coalesced report 1", "Starting report 1"); }
-  { Progress progress("Coalesced report 1", "Starting report 2"); }
-  { Progress progress("Coalesced report 1", "Starting report 3"); }
-
-  ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT));
-  data = ProgressEventData::GetEventDataFromEvent(event_sp.get());
-  expected_progress_id = data->GetID();
-
-  EXPECT_EQ(data->GetDetails(), "");
-  EXPECT_FALSE(data->IsFinite());
-  EXPECT_FALSE(data->GetCompleted());
-  EXPECT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal);
-  EXPECT_EQ(data->GetMessage(), "Coalesced report 1");
-
-  ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT));
-  data = ProgressEventData::GetEventDataFromEvent(event_sp.get());
-
-  EXPECT_EQ(data->GetID(), expected_progress_id);
-  EXPECT_EQ(data->GetDetails(), "");
-  EXPECT_FALSE(data->IsFinite());
-  EXPECT_TRUE(data->GetCompleted());
-  EXPECT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal);
-  EXPECT_EQ(data->GetMessage(), "Coalesced report 1");
-
-  ASSERT_FALSE(listener_sp->GetEvent(event_sp, TIMEOUT));
-}
-- 
GitLab


From 8551454313233ecc70a329de0eb0064ecb675d50 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 25 Mar 2024 15:26:24 -0700
Subject: [PATCH 214/404] [RISCV] Rename $merge to $rd in Zvk*
 pseudoinstructions and patterns.

These instructions use the destination operand as a source, not
only as a passthru for mask/tail.
---
 llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
index 617ca3242cc7..80ba54529196 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
@@ -181,13 +181,13 @@ class ZvkMxSet<string vd_lmul> {
 }
 
 class VPseudoUnaryNoMask_Zvk<DAGOperand RetClass, VReg OpClass> :
-      Pseudo<(outs RetClass:$rd),
-        (ins RetClass:$merge, OpClass:$rs2, AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
+      Pseudo<(outs RetClass:$rd_wb),
+        (ins RetClass:$rd, OpClass:$rs2, AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
         RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
   let hasSideEffects = 0;
-  let Constraints = "$rd = $merge";
+  let Constraints = "$rd_wb = $rd";
   let HasVLOp = 1;
   let HasSEWOp = 1;
   let HasVecPolicyOp = 1;
@@ -197,14 +197,14 @@ class VPseudoUnaryNoMask_Zvk<DAGOperand RetClass, VReg OpClass> :
 class VPseudoBinaryNoMask_Zvk<VReg RetClass,
                               VReg Op1Class,
                               DAGOperand Op2Class> :
-        Pseudo<(outs RetClass:$rd),
-               (ins RetClass:$merge, Op1Class:$rs2, Op2Class:$rs1,
+        Pseudo<(outs RetClass:$rd_wb),
+               (ins RetClass:$rd, Op1Class:$rs2, Op2Class:$rs1,
                     AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
         RISCVVPseudo {
   let mayLoad = 0;
   let mayStore = 0;
   let hasSideEffects = 0;
-  let Constraints = "$rd = $merge";
+  let Constraints = "$rd_wb = $rd";
   let HasVLOp = 1;
   let HasSEWOp = 1;
   let HasVecPolicyOp = 1;
@@ -670,11 +670,11 @@ class VPatUnaryNoMask_Zvk<string intrinsic_name,
                           VReg result_reg_class,
                           VReg op2_reg_class> :
   Pat<(result_type (!cast<Intrinsic>(intrinsic_name)
-                   (result_type result_reg_class:$merge),
+                   (result_type result_reg_class:$rd),
                    (op2_type op2_reg_class:$rs2),
                    VLOpFrag, (XLenVT timm:$policy))),
                    (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX)
-                   (result_type result_reg_class:$merge),
+                   (result_type result_reg_class:$rd),
                    (op2_type op2_reg_class:$rs2),
                    GPR:$vl, sew, (XLenVT timm:$policy))>;
 
@@ -689,11 +689,11 @@ class VPatUnaryNoMask_VS_Zvk<string intrinsic_name,
                              VReg result_reg_class,
                              VReg op2_reg_class> :
   Pat<(result_type (!cast<Intrinsic>(intrinsic_name)
-                   (result_type result_reg_class:$merge),
+                   (result_type result_reg_class:$rd),
                    (op2_type op2_reg_class:$rs2),
                    VLOpFrag, (XLenVT timm:$policy))),
                    (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX#"_"#vs2_lmul.MX)
-                   (result_type result_reg_class:$merge),
+                   (result_type result_reg_class:$rd),
                    (op2_type op2_reg_class:$rs2),
                    GPR:$vl, sew, (XLenVT timm:$policy))>;
 
-- 
GitLab


From dd7151094f94bc45668cf8629f02fe6f1975d5c9 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 25 Mar 2024 15:46:51 -0700
Subject: [PATCH 215/404] [ELF] Simplify parseArmCMSEImportLib. NFC

---
 lld/ELF/Config.h       | 2 +-
 lld/ELF/InputFiles.cpp | 6 +++---
 lld/ELF/InputFiles.h   | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index bb3608da80b2..27274d69b7f1 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -138,7 +138,7 @@ private:
 
   std::unique_ptr<BitcodeCompiler> lto;
   std::vector<InputFile *> files;
-  std::optional<InputFile *> armCmseImpLib;
+  InputFile *armCmseImpLib = nullptr;
 
 public:
   SmallVector<std::pair<StringRef, unsigned>, 0> archiveFiles;
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index 4c614c865d24..1104433a5047 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -330,11 +330,11 @@ extern template void ObjFile<ELF32BE>::importCmseSymbols();
 extern template void ObjFile<ELF64LE>::importCmseSymbols();
 extern template void ObjFile<ELF64BE>::importCmseSymbols();
 
-template <class ELFT> static void doParseArmCMSEImportLib(InputFile *file) {
-  cast<ObjFile<ELFT>>(file)->importCmseSymbols();
+template <class ELFT> static void doParseArmCMSEImportLib(InputFile &file) {
+  cast<ObjFile<ELFT>>(file).importCmseSymbols();
 }
 
-void elf::parseArmCMSEImportLib(InputFile *file) {
+void elf::parseArmCMSEImportLib(InputFile &file) {
   invokeELFT(doParseArmCMSEImportLib, file);
 }
 
diff --git a/lld/ELF/InputFiles.h b/lld/ELF/InputFiles.h
index 3beb5a3cb9a8..7d5dd3428275 100644
--- a/lld/ELF/InputFiles.h
+++ b/lld/ELF/InputFiles.h
@@ -49,7 +49,7 @@ std::optional<MemoryBufferRef> readFile(StringRef path);
 template <class ELFT> void doParseFile(InputFile *file);
 void parseFile(InputFile *file);
 
-void parseArmCMSEImportLib(InputFile *file);
+void parseArmCMSEImportLib(InputFile &file);
 
 // The root class of input files.
 class InputFile {
-- 
GitLab


From efa54949b947edc80069d122d771c021b64c3777 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 25 Mar 2024 15:52:46 -0700
Subject: [PATCH 216/404] [RISCV] Remove ', vm' from comment for instruction
 that doesn't have mask. NFC

---
 llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
index 80ba54529196..4cf64e5b09de 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
@@ -67,7 +67,7 @@ class PALUVVNoVm<bits<6> funct6, RISCVVFormat opv, string opcodestr>
   let Inst{6-0} = OPC_OP_P.Value;
 }
 
-// op vd, vs2, imm, vm
+// op vd, vs2, imm
 class PALUVINoVm<bits<6> funct6, string opcodestr, Operand optype = simm5>
     : VALUVINoVm<funct6, opcodestr, optype> {
   let Inst{6-0} = OPC_OP_P.Value;
-- 
GitLab


From bc02c0825ae3cca078654ba5f8cd373b8266492e Mon Sep 17 00:00:00 2001
From: Iman Hosseini <hosseini.iman@yahoo.com>
Date: Mon, 25 Mar 2024 22:54:05 +0000
Subject: [PATCH 217/404] [documentation] [mlir] DataLayout.md: fix broken link
 to DLTI dialect (#86524)

The link to DLTI dialect was broken.
---
 mlir/docs/DataLayout.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/docs/DataLayout.md b/mlir/docs/DataLayout.md
index 86ad51a517ae..21430c44ca95 100644
--- a/mlir/docs/DataLayout.md
+++ b/mlir/docs/DataLayout.md
@@ -289,7 +289,7 @@ The default data layout assumes 8-bit bytes.
 
 ### DLTI Dialect
 
-The [DLTI](Dialects/DLTI.md) dialect provides the attributes implementing
+The [DLTI](../Dialects/DLTIDialect/) dialect provides the attributes implementing
 `DataLayoutSpecInterface` and `DataLayoutEntryInterface`, as well as a dialect
 attribute that can be used to attach the specification to a given operation. The
 verifier of this attribute triggers those of the specification and checks the
-- 
GitLab


From 8ebf741136c66f51053315bf4f0ef828c6f66094 Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin@nvidia.com>
Date: Mon, 25 Mar 2024 16:01:25 -0700
Subject: [PATCH 218/404] [flang][runtime] Prepare enabling PRINT of integer32
 for device. (#86247)

This commit adds required files into the offload build closure,
which means adding RT_API_ATTRS and other markers.

The implementation does not work for CUDA yet, because of
std::variant,swap,reverse usage. These issues will be resolved
separately (e.g. by using libcudacxx header files).
---
 .../flang/{Runtime => Common}/api-attrs.h     |   4 +-
 flang/include/flang/Common/optional.h         |   2 +-
 flang/include/flang/Common/real.h             |   3 +
 .../include/flang/Common/reference-wrapper.h  |   2 +-
 flang/include/flang/Common/restorer.h         |  11 +-
 flang/include/flang/Common/uint128.h          |   3 +
 flang/include/flang/Common/visit.h            |   5 +-
 .../flang/Decimal/binary-floating-point.h     |  51 ++-
 flang/include/flang/Decimal/decimal.h         |  53 +--
 .../flang/ISO_Fortran_binding_wrapper.h       |   4 +-
 flang/include/flang/Runtime/entry-names.h     |   2 +-
 flang/include/flang/Runtime/io-api.h          |  16 +-
 flang/include/flang/Runtime/iostat.h          |   3 +-
 flang/include/flang/Runtime/memory.h          |  15 +-
 flang/include/flang/Runtime/type-code.h       |   1 -
 flang/runtime/CMakeLists.txt                  |  18 +
 flang/runtime/buffer.cpp                      |   6 +
 flang/runtime/buffer.h                        |  42 +-
 flang/runtime/connection.cpp                  |  15 +-
 flang/runtime/connection.h                    |  29 +-
 flang/runtime/derived.h                       |   2 +-
 flang/runtime/descriptor-io.cpp               |   7 +-
 flang/runtime/descriptor-io.h                 |  36 +-
 flang/runtime/edit-input.cpp                  |  90 ++--
 flang/runtime/edit-input.h                    |  28 +-
 flang/runtime/edit-output.cpp                 |  84 ++--
 flang/runtime/edit-output.h                   |  76 ++--
 flang/runtime/emit-encoded.h                  |   7 +-
 flang/runtime/environment.h                   |   2 +
 flang/runtime/external-unit.cpp               |  11 +-
 flang/runtime/file.cpp                        |  22 +
 flang/runtime/file.h                          |  12 +-
 flang/runtime/format-implementation.h         |  16 +-
 flang/runtime/format.cpp                      |   2 +
 flang/runtime/format.h                        |  39 +-
 flang/runtime/freestanding-tools.h            |  27 +-
 flang/runtime/internal-unit.cpp               |  31 +-
 flang/runtime/internal-unit.h                 |  26 +-
 flang/runtime/io-api.cpp                      |  26 +-
 flang/runtime/io-error.cpp                    |  27 +-
 flang/runtime/io-error.h                      |  36 +-
 flang/runtime/io-stmt.cpp                     |  26 +-
 flang/runtime/io-stmt.h                       | 410 ++++++++++--------
 flang/runtime/iostat.cpp                      |   4 +
 flang/runtime/lock.h                          |  14 +-
 flang/runtime/memory.cpp                      |  12 +-
 flang/runtime/namelist.cpp                    |  15 +-
 flang/runtime/namelist.h                      |   3 +-
 flang/runtime/non-tbp-dio.h                   |   2 +-
 flang/runtime/numeric-templates.h             |   9 +-
 flang/runtime/pointer.cpp                     |   2 -
 flang/runtime/pseudo-unit.cpp                 |   8 +-
 flang/runtime/stat.h                          |   2 +-
 flang/runtime/terminator.h                    |   4 +-
 flang/runtime/unit.cpp                        |  25 +-
 flang/runtime/unit.h                          | 203 +++++----
 flang/runtime/utf.cpp                         |   6 +-
 flang/runtime/utf.h                           |  14 +-
 58 files changed, 978 insertions(+), 673 deletions(-)
 rename flang/include/flang/{Runtime => Common}/api-attrs.h (97%)

diff --git a/flang/include/flang/Runtime/api-attrs.h b/flang/include/flang/Common/api-attrs.h
similarity index 97%
rename from flang/include/flang/Runtime/api-attrs.h
rename to flang/include/flang/Common/api-attrs.h
index fc3eb42e1b73..4d069c6097dd 100644
--- a/flang/include/flang/Runtime/api-attrs.h
+++ b/flang/include/flang/Common/api-attrs.h
@@ -1,4 +1,4 @@
-/*===-- include/flang/Runtime/api-attrs.h ---------------------------*- C -*-=//
+/*===-- include/flang/Common/api-attrs.h ---------------------------*- C -*-=//
  *
  * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
  * See https://llvm.org/LICENSE.txt for license information.
@@ -102,7 +102,7 @@
  * to appear as part of a C++ decl-specifier.
  */
 #ifndef RT_CONST_VAR_ATTRS
-#if defined(__CUDACC__) || defined(__CUDA__)
+#if (defined(__CUDACC__) || defined(__CUDA__)) && defined(__CUDA_ARCH__)
 #define RT_CONST_VAR_ATTRS __constant__
 #else
 #define RT_CONST_VAR_ATTRS
diff --git a/flang/include/flang/Common/optional.h b/flang/include/flang/Common/optional.h
index b5623b84dbd3..c0f4278009f4 100644
--- a/flang/include/flang/Common/optional.h
+++ b/flang/include/flang/Common/optional.h
@@ -26,7 +26,7 @@
 #ifndef FORTRAN_COMMON_OPTIONAL_H
 #define FORTRAN_COMMON_OPTIONAL_H
 
-#include "flang/Runtime/api-attrs.h"
+#include "flang/Common/api-attrs.h"
 #include <optional>
 #include <type_traits>
 
diff --git a/flang/include/flang/Common/real.h b/flang/include/flang/Common/real.h
index 50aab7d89a59..49c400b368a2 100644
--- a/flang/include/flang/Common/real.h
+++ b/flang/include/flang/Common/real.h
@@ -13,6 +13,7 @@
 // The various representations are distinguished by their binary precisions
 // (number of explicit significand bits and any implicit MSB in the fraction).
 
+#include "flang/Common/api-attrs.h"
 #include <cinttypes>
 
 namespace Fortran::common {
@@ -119,6 +120,7 @@ private:
   }
 
 public:
+  RT_OFFLOAD_VAR_GROUP_BEGIN
   static constexpr int binaryPrecision{BINARY_PRECISION};
   static constexpr int bits{BitsForBinaryPrecision(binaryPrecision)};
   static constexpr bool isImplicitMSB{binaryPrecision != 64 /*x87*/};
@@ -138,6 +140,7 @@ public:
 
   static constexpr int maxHexadecimalConversionDigits{
       MaxHexadecimalConversionDigits(binaryPrecision)};
+  RT_OFFLOAD_VAR_GROUP_END
 
   static_assert(binaryPrecision > 0);
   static_assert(exponentBits > 1);
diff --git a/flang/include/flang/Common/reference-wrapper.h b/flang/include/flang/Common/reference-wrapper.h
index 66f924662d96..2983754108f9 100644
--- a/flang/include/flang/Common/reference-wrapper.h
+++ b/flang/include/flang/Common/reference-wrapper.h
@@ -25,7 +25,7 @@
 #ifndef FORTRAN_COMMON_REFERENCE_WRAPPER_H
 #define FORTRAN_COMMON_REFERENCE_WRAPPER_H
 
-#include "flang/Runtime/api-attrs.h"
+#include "flang/Common/api-attrs.h"
 #include <functional>
 #include <type_traits>
 
diff --git a/flang/include/flang/Common/restorer.h b/flang/include/flang/Common/restorer.h
index 4d5f5e4e2c81..0f1bc48620d3 100644
--- a/flang/include/flang/Common/restorer.h
+++ b/flang/include/flang/Common/restorer.h
@@ -19,11 +19,13 @@
 #ifndef FORTRAN_COMMON_RESTORER_H_
 #define FORTRAN_COMMON_RESTORER_H_
 #include "idioms.h"
+#include "flang/Common/api-attrs.h"
 namespace Fortran::common {
 template <typename A> class Restorer {
 public:
-  explicit Restorer(A &p, A original) : p_{p}, original_{std::move(original)} {}
-  ~Restorer() { p_ = std::move(original_); }
+  explicit RT_API_ATTRS Restorer(A &p, A original)
+      : p_{p}, original_{std::move(original)} {}
+  RT_API_ATTRS ~Restorer() { p_ = std::move(original_); }
 
   // Inhibit any recreation of this restorer that would result in two restorers
   // trying to restore the same reference.
@@ -38,13 +40,14 @@ private:
 };
 
 template <typename A, typename B>
-common::IfNoLvalue<Restorer<A>, B> ScopedSet(A &to, B &&from) {
+RT_API_ATTRS common::IfNoLvalue<Restorer<A>, B> ScopedSet(A &to, B &&from) {
   A original{std::move(to)};
   to = std::move(from);
   return Restorer<A>{to, std::move(original)};
 }
 template <typename A, typename B>
-common::IfNoLvalue<Restorer<A>, B> ScopedSet(A &to, const B &from) {
+RT_API_ATTRS common::IfNoLvalue<Restorer<A>, B> ScopedSet(
+    A &to, const B &from) {
   A original{std::move(to)};
   to = from;
   return Restorer<A>{to, std::move(original)};
diff --git a/flang/include/flang/Common/uint128.h b/flang/include/flang/Common/uint128.h
index 03e44eb6997d..821c8c3b08a5 100644
--- a/flang/include/flang/Common/uint128.h
+++ b/flang/include/flang/Common/uint128.h
@@ -20,6 +20,7 @@
 #endif
 
 #include "leading-zero-bit-count.h"
+#include "flang/Common/api-attrs.h"
 #include <cstdint>
 #include <type_traits>
 
@@ -260,7 +261,9 @@ private:
       return LeadingZeroBitCount(high_);
     }
   }
+  RT_VAR_GROUP_BEGIN
   static constexpr std::uint64_t topBit{std::uint64_t{1} << 63};
+  RT_VAR_GROUP_END
 #if FLANG_LITTLE_ENDIAN
   std::uint64_t low_{0}, high_{0};
 #elif FLANG_BIG_ENDIAN
diff --git a/flang/include/flang/Common/visit.h b/flang/include/flang/Common/visit.h
index f733b726189c..54f8ca70b313 100644
--- a/flang/include/flang/Common/visit.h
+++ b/flang/include/flang/Common/visit.h
@@ -21,6 +21,7 @@
 #ifndef FORTRAN_COMMON_VISIT_H_
 #define FORTRAN_COMMON_VISIT_H_
 
+#include "flang/Common/api-attrs.h"
 #include <type_traits>
 #include <variant>
 
@@ -29,7 +30,7 @@ namespace log2visit {
 
 template <std::size_t LOW, std::size_t HIGH, typename RESULT, typename VISITOR,
     typename... VARIANT>
-inline RESULT Log2VisitHelper(
+inline RT_API_ATTRS RESULT Log2VisitHelper(
     VISITOR &&visitor, std::size_t which, VARIANT &&...u) {
   if constexpr (LOW + 7 >= HIGH) {
     switch (which - LOW) {
@@ -61,7 +62,7 @@ inline RESULT Log2VisitHelper(
 }
 
 template <typename VISITOR, typename... VARIANT>
-inline auto visit(VISITOR &&visitor, VARIANT &&...u)
+inline RT_API_ATTRS auto visit(VISITOR &&visitor, VARIANT &&...u)
     -> decltype(visitor(std::get<0>(std::forward<VARIANT>(u))...)) {
   using Result = decltype(visitor(std::get<0>(std::forward<VARIANT>(u))...));
   if constexpr (sizeof...(u) == 1) {
diff --git a/flang/include/flang/Decimal/binary-floating-point.h b/flang/include/flang/Decimal/binary-floating-point.h
index d1992819f85a..4919c1f9d240 100644
--- a/flang/include/flang/Decimal/binary-floating-point.h
+++ b/flang/include/flang/Decimal/binary-floating-point.h
@@ -12,6 +12,7 @@
 // Access and manipulate the fields of an IEEE-754 binary
 // floating-point value via a generalized template.
 
+#include "flang/Common/api-attrs.h"
 #include "flang/Common/real.h"
 #include "flang/Common/uint128.h"
 #include <cinttypes>
@@ -47,9 +48,11 @@ public:
 
   using RawType = common::HostUnsignedIntType<bits>;
   static_assert(CHAR_BIT * sizeof(RawType) >= bits);
+  RT_OFFLOAD_VAR_GROUP_BEGIN
   static constexpr RawType significandMask{(RawType{1} << significandBits) - 1};
 
-  constexpr BinaryFloatingPointNumber() {} // zero
+  constexpr RT_API_ATTRS BinaryFloatingPointNumber() {} // zero
+  RT_OFFLOAD_VAR_GROUP_END
   constexpr BinaryFloatingPointNumber(
       const BinaryFloatingPointNumber &that) = default;
   constexpr BinaryFloatingPointNumber(
@@ -58,26 +61,30 @@ public:
       const BinaryFloatingPointNumber &that) = default;
   constexpr BinaryFloatingPointNumber &operator=(
       BinaryFloatingPointNumber &&that) = default;
-  constexpr explicit BinaryFloatingPointNumber(RawType raw) : raw_{raw} {}
+  constexpr explicit RT_API_ATTRS BinaryFloatingPointNumber(RawType raw)
+      : raw_{raw} {}
 
-  RawType raw() const { return raw_; }
+  RT_API_ATTRS RawType raw() const { return raw_; }
 
-  template <typename A> explicit constexpr BinaryFloatingPointNumber(A x) {
+  template <typename A>
+  explicit constexpr RT_API_ATTRS BinaryFloatingPointNumber(A x) {
     static_assert(sizeof raw_ <= sizeof x);
     std::memcpy(reinterpret_cast<void *>(&raw_),
         reinterpret_cast<const void *>(&x), sizeof raw_);
   }
 
-  constexpr int BiasedExponent() const {
+  constexpr RT_API_ATTRS int BiasedExponent() const {
     return static_cast<int>(
         (raw_ >> significandBits) & ((1 << exponentBits) - 1));
   }
-  constexpr int UnbiasedExponent() const {
+  constexpr RT_API_ATTRS int UnbiasedExponent() const {
     int biased{BiasedExponent()};
     return biased - exponentBias + (biased == 0);
   }
-  constexpr RawType Significand() const { return raw_ & significandMask; }
-  constexpr RawType Fraction() const {
+  constexpr RT_API_ATTRS RawType Significand() const {
+    return raw_ & significandMask;
+  }
+  constexpr RT_API_ATTRS RawType Fraction() const {
     RawType sig{Significand()};
     if (isImplicitMSB && BiasedExponent() > 0) {
       sig |= RawType{1} << significandBits;
@@ -85,10 +92,10 @@ public:
     return sig;
   }
 
-  constexpr bool IsZero() const {
+  constexpr RT_API_ATTRS bool IsZero() const {
     return (raw_ & ((RawType{1} << (bits - 1)) - 1)) == 0;
   }
-  constexpr bool IsNaN() const {
+  constexpr RT_API_ATTRS bool IsNaN() const {
     auto expo{BiasedExponent()};
     auto sig{Significand()};
     if constexpr (bits == 80) { // x87
@@ -102,7 +109,7 @@ public:
       return expo == maxExponent && sig != 0;
     }
   }
-  constexpr bool IsInfinite() const {
+  constexpr RT_API_ATTRS bool IsInfinite() const {
     if constexpr (bits == 80) { // x87
       return BiasedExponent() == maxExponent &&
           Significand() == ((significandMask >> 1) + 1);
@@ -110,27 +117,30 @@ public:
       return BiasedExponent() == maxExponent && Significand() == 0;
     }
   }
-  constexpr bool IsMaximalFiniteMagnitude() const {
+  constexpr RT_API_ATTRS bool IsMaximalFiniteMagnitude() const {
     return BiasedExponent() == maxExponent - 1 &&
         Significand() == significandMask;
   }
-  constexpr bool IsNegative() const { return ((raw_ >> (bits - 1)) & 1) != 0; }
+  constexpr RT_API_ATTRS bool IsNegative() const {
+    return ((raw_ >> (bits - 1)) & 1) != 0;
+  }
 
-  constexpr void Negate() { raw_ ^= RawType{1} << (bits - 1); }
+  constexpr RT_API_ATTRS void Negate() { raw_ ^= RawType{1} << (bits - 1); }
 
   // For calculating the nearest neighbors of a floating-point value
-  constexpr void Previous() {
+  constexpr RT_API_ATTRS void Previous() {
     RemoveExplicitMSB();
     --raw_;
     InsertExplicitMSB();
   }
-  constexpr void Next() {
+  constexpr RT_API_ATTRS void Next() {
     RemoveExplicitMSB();
     ++raw_;
     InsertExplicitMSB();
   }
 
-  static constexpr BinaryFloatingPointNumber Infinity(bool isNegative) {
+  static constexpr RT_API_ATTRS BinaryFloatingPointNumber Infinity(
+      bool isNegative) {
     RawType result{RawType{maxExponent} << significandBits};
     if (isNegative) {
       result |= RawType{1} << (bits - 1);
@@ -139,7 +149,8 @@ public:
   }
 
   // Returns true when the result is exact
-  constexpr bool RoundToBits(int keepBits, enum FortranRounding mode) {
+  constexpr RT_API_ATTRS bool RoundToBits(
+      int keepBits, enum FortranRounding mode) {
     if (IsNaN() || IsInfinite() || keepBits >= binaryPrecision) {
       return true;
     }
@@ -180,12 +191,12 @@ public:
   }
 
 private:
-  constexpr void RemoveExplicitMSB() {
+  constexpr RT_API_ATTRS void RemoveExplicitMSB() {
     if constexpr (!isImplicitMSB) {
       raw_ = (raw_ & (significandMask >> 1)) | ((raw_ & ~significandMask) >> 1);
     }
   }
-  constexpr void InsertExplicitMSB() {
+  constexpr RT_API_ATTRS void InsertExplicitMSB() {
     if constexpr (!isImplicitMSB) {
       constexpr RawType mask{significandMask >> 1};
       raw_ = (raw_ & mask) | ((raw_ & ~mask) << 1);
diff --git a/flang/include/flang/Decimal/decimal.h b/flang/include/flang/Decimal/decimal.h
index f0997fb63df0..443163d058e2 100644
--- a/flang/include/flang/Decimal/decimal.h
+++ b/flang/include/flang/Decimal/decimal.h
@@ -12,6 +12,7 @@
 #ifndef FORTRAN_DECIMAL_DECIMAL_H_
 #define FORTRAN_DECIMAL_DECIMAL_H_
 
+#include "flang/Common/api-attrs.h"
 #include <stddef.h>
 
 #ifdef __cplusplus
@@ -65,27 +66,27 @@ enum DecimalConversionFlags {
 
 #ifdef __cplusplus
 template <int PREC>
-ConversionToDecimalResult ConvertToDecimal(char *, size_t,
+RT_API_ATTRS ConversionToDecimalResult ConvertToDecimal(char *, size_t,
     DecimalConversionFlags, int digits, enum FortranRounding rounding,
     BinaryFloatingPointNumber<PREC> x);
 
-extern template ConversionToDecimalResult ConvertToDecimal<8>(char *, size_t,
-    enum DecimalConversionFlags, int, enum FortranRounding,
+extern template RT_API_ATTRS ConversionToDecimalResult ConvertToDecimal<8>(
+    char *, size_t, enum DecimalConversionFlags, int, enum FortranRounding,
     BinaryFloatingPointNumber<8>);
-extern template ConversionToDecimalResult ConvertToDecimal<11>(char *, size_t,
-    enum DecimalConversionFlags, int, enum FortranRounding,
+extern template RT_API_ATTRS ConversionToDecimalResult ConvertToDecimal<11>(
+    char *, size_t, enum DecimalConversionFlags, int, enum FortranRounding,
     BinaryFloatingPointNumber<11>);
-extern template ConversionToDecimalResult ConvertToDecimal<24>(char *, size_t,
-    enum DecimalConversionFlags, int, enum FortranRounding,
+extern template RT_API_ATTRS ConversionToDecimalResult ConvertToDecimal<24>(
+    char *, size_t, enum DecimalConversionFlags, int, enum FortranRounding,
     BinaryFloatingPointNumber<24>);
-extern template ConversionToDecimalResult ConvertToDecimal<53>(char *, size_t,
-    enum DecimalConversionFlags, int, enum FortranRounding,
+extern template RT_API_ATTRS ConversionToDecimalResult ConvertToDecimal<53>(
+    char *, size_t, enum DecimalConversionFlags, int, enum FortranRounding,
     BinaryFloatingPointNumber<53>);
-extern template ConversionToDecimalResult ConvertToDecimal<64>(char *, size_t,
-    enum DecimalConversionFlags, int, enum FortranRounding,
+extern template RT_API_ATTRS ConversionToDecimalResult ConvertToDecimal<64>(
+    char *, size_t, enum DecimalConversionFlags, int, enum FortranRounding,
     BinaryFloatingPointNumber<64>);
-extern template ConversionToDecimalResult ConvertToDecimal<113>(char *, size_t,
-    enum DecimalConversionFlags, int, enum FortranRounding,
+extern template RT_API_ATTRS ConversionToDecimalResult ConvertToDecimal<113>(
+    char *, size_t, enum DecimalConversionFlags, int, enum FortranRounding,
     BinaryFloatingPointNumber<113>);
 
 template <int PREC> struct ConversionToBinaryResult {
@@ -94,20 +95,20 @@ template <int PREC> struct ConversionToBinaryResult {
 };
 
 template <int PREC>
-ConversionToBinaryResult<PREC> ConvertToBinary(const char *&,
+RT_API_ATTRS ConversionToBinaryResult<PREC> ConvertToBinary(const char *&,
     enum FortranRounding = RoundNearest, const char *end = nullptr);
 
-extern template ConversionToBinaryResult<8> ConvertToBinary<8>(
+extern template RT_API_ATTRS ConversionToBinaryResult<8> ConvertToBinary<8>(
     const char *&, enum FortranRounding, const char *end);
-extern template ConversionToBinaryResult<11> ConvertToBinary<11>(
+extern template RT_API_ATTRS ConversionToBinaryResult<11> ConvertToBinary<11>(
     const char *&, enum FortranRounding, const char *end);
-extern template ConversionToBinaryResult<24> ConvertToBinary<24>(
+extern template RT_API_ATTRS ConversionToBinaryResult<24> ConvertToBinary<24>(
     const char *&, enum FortranRounding, const char *end);
-extern template ConversionToBinaryResult<53> ConvertToBinary<53>(
+extern template RT_API_ATTRS ConversionToBinaryResult<53> ConvertToBinary<53>(
     const char *&, enum FortranRounding, const char *end);
-extern template ConversionToBinaryResult<64> ConvertToBinary<64>(
+extern template RT_API_ATTRS ConversionToBinaryResult<64> ConvertToBinary<64>(
     const char *&, enum FortranRounding, const char *end);
-extern template ConversionToBinaryResult<113> ConvertToBinary<113>(
+extern template RT_API_ATTRS ConversionToBinaryResult<113> ConvertToBinary<113>(
     const char *&, enum FortranRounding, const char *end);
 } // namespace Fortran::decimal
 extern "C" {
@@ -116,21 +117,21 @@ extern "C" {
 #define NS(x) x
 #endif /* C++ */
 
-struct NS(ConversionToDecimalResult)
+RT_API_ATTRS struct NS(ConversionToDecimalResult)
     ConvertFloatToDecimal(char *, size_t, enum NS(DecimalConversionFlags),
         int digits, enum NS(FortranRounding), float);
-struct NS(ConversionToDecimalResult)
+RT_API_ATTRS struct NS(ConversionToDecimalResult)
     ConvertDoubleToDecimal(char *, size_t, enum NS(DecimalConversionFlags),
         int digits, enum NS(FortranRounding), double);
-struct NS(ConversionToDecimalResult)
+RT_API_ATTRS struct NS(ConversionToDecimalResult)
     ConvertLongDoubleToDecimal(char *, size_t, enum NS(DecimalConversionFlags),
         int digits, enum NS(FortranRounding), long double);
 
-enum NS(ConversionResultFlags)
+RT_API_ATTRS enum NS(ConversionResultFlags)
     ConvertDecimalToFloat(const char **, float *, enum NS(FortranRounding));
-enum NS(ConversionResultFlags)
+RT_API_ATTRS enum NS(ConversionResultFlags)
     ConvertDecimalToDouble(const char **, double *, enum NS(FortranRounding));
-enum NS(ConversionResultFlags) ConvertDecimalToLongDouble(
+RT_API_ATTRS enum NS(ConversionResultFlags) ConvertDecimalToLongDouble(
     const char **, long double *, enum NS(FortranRounding));
 #undef NS
 #ifdef __cplusplus
diff --git a/flang/include/flang/ISO_Fortran_binding_wrapper.h b/flang/include/flang/ISO_Fortran_binding_wrapper.h
index 83c974365e34..37289bdbabd0 100644
--- a/flang/include/flang/ISO_Fortran_binding_wrapper.h
+++ b/flang/include/flang/ISO_Fortran_binding_wrapper.h
@@ -13,7 +13,7 @@
 /* A thin wrapper around flang/include/ISO_Fortran_binding.h
  * This header file must be included when ISO_Fortran_binding.h
  * definitions/declarations are needed in Flang compiler/runtime
- * sources. The inclusion of Runtime/api-attrs.h below sets up
+ * sources. The inclusion of Common/api-attrs.h below sets up
  * proper values for the macros used in ISO_Fortran_binding.h
  * for the device offload builds.
  * flang/include/ISO_Fortran_binding.h is made a standalone
@@ -23,7 +23,7 @@
 
 /* clang-format off */
 #include <stddef.h>
-#include "Runtime/api-attrs.h"
+#include "Common/api-attrs.h"
 #ifdef __cplusplus
 namespace Fortran {
 namespace ISO {
diff --git a/flang/include/flang/Runtime/entry-names.h b/flang/include/flang/Runtime/entry-names.h
index a233edf8e987..68582b92b549 100644
--- a/flang/include/flang/Runtime/entry-names.h
+++ b/flang/include/flang/Runtime/entry-names.h
@@ -19,7 +19,7 @@
 #ifndef FORTRAN_RUNTIME_ENTRY_NAMES_H
 #define FORTRAN_RUNTIME_ENTRY_NAMES_H
 
-#include "flang/Runtime/api-attrs.h"
+#include "flang/Common/api-attrs.h"
 
 #ifndef RTNAME
 #define NAME_WITH_PREFIX_AND_REVISION(prefix, revision, name) \
diff --git a/flang/include/flang/Runtime/io-api.h b/flang/include/flang/Runtime/io-api.h
index 556cc20c5a12..1b6c4f5d6a65 100644
--- a/flang/include/flang/Runtime/io-api.h
+++ b/flang/include/flang/Runtime/io-api.h
@@ -51,13 +51,21 @@ constexpr InquiryKeywordHash HashInquiryKeyword(const char *p) {
   return hash;
 }
 
-const char *InquiryKeywordHashDecode(
+RT_API_ATTRS const char *InquiryKeywordHashDecode(
     char *buffer, std::size_t, InquiryKeywordHash);
 
 extern "C" {
 
 #define IONAME(name) RTNAME(io##name)
 
+#ifndef IODECL
+#define IODECL(name) RT_API_ATTRS IONAME(name)
+#endif
+
+#ifndef IODEF
+#define IODEF(name) RT_API_ATTRS IONAME(name)
+#endif
+
 // These functions initiate data transfer statements (READ, WRITE, PRINT).
 // Example: PRINT *, 666 is implemented as the series of calls:
 //   Cookie cookie{BeginExternalListOutput(DefaultOutputUnit,
@@ -139,7 +147,7 @@ enum Iostat IONAME(CheckUnitNumberInRange128)(common::int128_t unit,
     const char *sourceFile = nullptr, int sourceLine = 0);
 
 // External synchronous I/O initiation
-Cookie IONAME(BeginExternalListOutput)(ExternalUnit = DefaultOutputUnit,
+Cookie IODECL(BeginExternalListOutput)(ExternalUnit = DefaultOutputUnit,
     const char *sourceFile = nullptr, int sourceLine = 0);
 Cookie IONAME(BeginExternalListInput)(ExternalUnit = DefaultInputUnit,
     const char *sourceFile = nullptr, int sourceLine = 0);
@@ -253,7 +261,7 @@ bool IONAME(InputDescriptor)(Cookie, const Descriptor &);
 // Formatted (including list directed) I/O data items
 bool IONAME(OutputInteger8)(Cookie, std::int8_t);
 bool IONAME(OutputInteger16)(Cookie, std::int16_t);
-bool IONAME(OutputInteger32)(Cookie, std::int32_t);
+bool IODECL(OutputInteger32)(Cookie, std::int32_t);
 bool IONAME(OutputInteger64)(Cookie, std::int64_t);
 bool IONAME(OutputInteger128)(Cookie, common::int128_t);
 bool IONAME(InputInteger)(Cookie, std::int64_t &, int kind = 8);
@@ -357,7 +365,7 @@ bool IONAME(InquireInteger64)(
 // returned is guaranteed to only be one of the problems that the
 // EnableHandlers() call has indicated should be handled in compiled code
 // rather than by terminating the image.
-enum Iostat IONAME(EndIoStatement)(Cookie);
+enum Iostat IODECL(EndIoStatement)(Cookie);
 
 } // extern "C"
 } // namespace Fortran::runtime::io
diff --git a/flang/include/flang/Runtime/iostat.h b/flang/include/flang/Runtime/iostat.h
index afce509cf1f5..6ce7c82b424e 100644
--- a/flang/include/flang/Runtime/iostat.h
+++ b/flang/include/flang/Runtime/iostat.h
@@ -11,6 +11,7 @@
 
 #ifndef FORTRAN_RUNTIME_IOSTAT_H_
 #define FORTRAN_RUNTIME_IOSTAT_H_
+#include "flang/Common/api-attrs.h"
 #include "flang/Runtime/magic-numbers.h"
 namespace Fortran::runtime::io {
 
@@ -88,7 +89,7 @@ enum Iostat {
   IostatNonExternalDefinedUnformattedIo,
 };
 
-const char *IostatErrorString(int);
+RT_API_ATTRS const char *IostatErrorString(int);
 
 } // namespace Fortran::runtime::io
 #endif // FORTRAN_RUNTIME_IOSTAT_H_
diff --git a/flang/include/flang/Runtime/memory.h b/flang/include/flang/Runtime/memory.h
index e24c509f4e90..98412a989f89 100644
--- a/flang/include/flang/Runtime/memory.h
+++ b/flang/include/flang/Runtime/memory.h
@@ -12,7 +12,7 @@
 #ifndef FORTRAN_RUNTIME_MEMORY_H_
 #define FORTRAN_RUNTIME_MEMORY_H_
 
-#include "flang/Runtime/api-attrs.h"
+#include "flang/Common/api-attrs.h"
 #include <cassert>
 #include <memory>
 #include <type_traits>
@@ -79,6 +79,8 @@ public:
     return p;
   }
 
+  RT_DIAG_PUSH
+  RT_DIAG_DISABLE_CALL_HOST_FROM_DEVICE_WARN
   // Replace the pointer.
   RT_API_ATTRS void reset(pointer_type p = pointer_type{}) {
     std::swap(ptr_, p);
@@ -90,6 +92,7 @@ public:
 
   // Exchange the pointer with another object.
   RT_API_ATTRS void swap(OwningPtr &other) { std::swap(ptr_, other.ptr_); }
+  RT_DIAG_POP
 
   // Get the stored pointer.
   RT_API_ATTRS pointer_type get() const { return ptr_; }
@@ -128,9 +131,12 @@ inline RT_API_ATTRS bool operator!=(std::nullptr_t, const OwningPtr<X> &x) {
 
 template <typename A> class SizedNew {
 public:
-  explicit SizedNew(const Terminator &terminator) : terminator_{terminator} {}
+  explicit RT_API_ATTRS SizedNew(const Terminator &terminator)
+      : terminator_{terminator} {}
+
   template <typename... X>
-  [[nodiscard]] OwningPtr<A> operator()(std::size_t bytes, X &&...x) {
+  [[nodiscard]] RT_API_ATTRS OwningPtr<A> operator()(
+      std::size_t bytes, X &&...x) {
     return OwningPtr<A>{new (AllocateMemoryOrCrash(terminator_, bytes))
             A{std::forward<X>(x)...}};
   }
@@ -141,7 +147,8 @@ private:
 
 template <typename A> struct New : public SizedNew<A> {
   using SizedNew<A>::SizedNew;
-  template <typename... X> [[nodiscard]] OwningPtr<A> operator()(X &&...x) {
+  template <typename... X>
+  [[nodiscard]] RT_API_ATTRS OwningPtr<A> operator()(X &&...x) {
     return SizedNew<A>::operator()(sizeof(A), std::forward<X>(x)...);
   }
 };
diff --git a/flang/include/flang/Runtime/type-code.h b/flang/include/flang/Runtime/type-code.h
index f7419249c2ba..8e7314e0af1e 100644
--- a/flang/include/flang/Runtime/type-code.h
+++ b/flang/include/flang/Runtime/type-code.h
@@ -12,7 +12,6 @@
 #include "flang/Common/Fortran.h"
 #include "flang/Common/optional.h"
 #include "flang/ISO_Fortran_binding_wrapper.h"
-#include <optional>
 #include <utility>
 
 namespace Fortran::runtime {
diff --git a/flang/runtime/CMakeLists.txt b/flang/runtime/CMakeLists.txt
index 021474871154..335ef3707274 100644
--- a/flang/runtime/CMakeLists.txt
+++ b/flang/runtime/CMakeLists.txt
@@ -180,22 +180,38 @@ set(supported_files
   allocatable.cpp
   array-constructor.cpp
   assign.cpp
+  buffer.cpp
   character.cpp
+  connection.cpp
   copy.cpp
   derived-api.cpp
   derived.cpp
   descriptor.cpp
+  descriptor-io.cpp
   dot-product.cpp
+  edit-input.cpp
+  edit-output.cpp
+  environment.cpp
   extrema.cpp
+  external-unit.cpp
   findloc.cpp
+  format.cpp
   inquiry.cpp
+  internal-unit.cpp
+  io-api.cpp
+  io-error.cpp
+  io-stmt.cpp
+  iostat.cpp
   matmul-transpose.cpp
   matmul.cpp
   memory.cpp
   misc-intrinsic.cpp
+  namelist.cpp
+  non-tbp-dio.cpp
   numeric.cpp
   pointer.cpp
   product.cpp
+  pseudo-unit.cpp
   ragged.cpp
   stat.cpp
   sum.cpp
@@ -205,6 +221,8 @@ set(supported_files
   transformational.cpp
   type-code.cpp
   type-info.cpp
+  unit.cpp
+  utf.cpp
   )
 
 if (FLANG_EXPERIMENTAL_CUDA_RUNTIME)
diff --git a/flang/runtime/buffer.cpp b/flang/runtime/buffer.cpp
index 15c83bfd2492..7b4869d69c2e 100644
--- a/flang/runtime/buffer.cpp
+++ b/flang/runtime/buffer.cpp
@@ -10,14 +10,20 @@
 #include <algorithm>
 
 namespace Fortran::runtime::io {
+RT_OFFLOAD_API_GROUP_BEGIN
 
 // Here's a very old trick for shifting circular buffer data cheaply
 // without a need for a temporary array.
 void LeftShiftBufferCircularly(
     char *buffer, std::size_t bytes, std::size_t shift) {
   // Assume that we start with "efgabcd" and the left shift is 3.
+  RT_DIAG_PUSH
+  RT_DIAG_DISABLE_CALL_HOST_FROM_DEVICE_WARN
   std::reverse(buffer, buffer + shift); // "gfeabcd"
   std::reverse(buffer, buffer + bytes); // "dcbaefg"
   std::reverse(buffer, buffer + bytes - shift); // "abcdefg"
+  RT_DIAG_POP
 }
+
+RT_OFFLOAD_API_GROUP_END
 } // namespace Fortran::runtime::io
diff --git a/flang/runtime/buffer.h b/flang/runtime/buffer.h
index 93fda36f500d..ca1baea12efa 100644
--- a/flang/runtime/buffer.h
+++ b/flang/runtime/buffer.h
@@ -11,6 +11,7 @@
 #ifndef FORTRAN_RUNTIME_BUFFER_H_
 #define FORTRAN_RUNTIME_BUFFER_H_
 
+#include "freestanding-tools.h"
 #include "io-error.h"
 #include "flang/Runtime/memory.h"
 #include <algorithm>
@@ -19,7 +20,8 @@
 
 namespace Fortran::runtime::io {
 
-void LeftShiftBufferCircularly(char *, std::size_t bytes, std::size_t shift);
+RT_API_ATTRS void LeftShiftBufferCircularly(
+    char *, std::size_t bytes, std::size_t shift);
 
 // Maintains a view of a contiguous region of a file in a memory buffer.
 // The valid data in the buffer may be circular, but any active frame
@@ -48,22 +50,24 @@ template <typename STORE, std::size_t minBuffer = 65536> class FileFrame {
 public:
   using FileOffset = std::int64_t;
 
-  ~FileFrame() { FreeMemoryAndNullify(buffer_); }
+  RT_API_ATTRS ~FileFrame() { FreeMemoryAndNullify(buffer_); }
 
   // The valid data in the buffer begins at buffer_[start_] and proceeds
   // with possible wrap-around for length_ bytes.  The current frame
   // is offset by frame_ bytes into that region and is guaranteed to
   // be contiguous for at least as many bytes as were requested.
 
-  FileOffset FrameAt() const { return fileOffset_ + frame_; }
-  char *Frame() const { return buffer_ + start_ + frame_; }
-  std::size_t FrameLength() const {
+  RT_API_ATTRS FileOffset FrameAt() const { return fileOffset_ + frame_; }
+  RT_API_ATTRS char *Frame() const { return buffer_ + start_ + frame_; }
+  RT_API_ATTRS std::size_t FrameLength() const {
     return std::min<std::size_t>(length_ - frame_, size_ - (start_ + frame_));
   }
-  std::size_t BytesBufferedBeforeFrame() const { return frame_ - start_; }
+  RT_API_ATTRS std::size_t BytesBufferedBeforeFrame() const {
+    return frame_ - start_;
+  }
 
   // Returns a short frame at a non-fatal EOF.  Can return a long frame as well.
-  std::size_t ReadFrame(
+  RT_API_ATTRS std::size_t ReadFrame(
       FileOffset at, std::size_t bytes, IoErrorHandler &handler) {
     Flush(handler);
     Reallocate(bytes, handler);
@@ -92,7 +96,8 @@ public:
     return FrameLength();
   }
 
-  void WriteFrame(FileOffset at, std::size_t bytes, IoErrorHandler &handler) {
+  RT_API_ATTRS void WriteFrame(
+      FileOffset at, std::size_t bytes, IoErrorHandler &handler) {
     Reallocate(bytes, handler);
     std::int64_t newFrame{at - fileOffset_};
     if (!dirty_ || newFrame < 0 || newFrame > length_) {
@@ -110,7 +115,7 @@ public:
     length_ = std::max<std::int64_t>(length_, frame_ + bytes);
   }
 
-  void Flush(IoErrorHandler &handler, std::int64_t keep = 0) {
+  RT_API_ATTRS void Flush(IoErrorHandler &handler, std::int64_t keep = 0) {
     if (dirty_) {
       while (length_ > keep) {
         std::size_t chunk{
@@ -128,7 +133,7 @@ public:
     }
   }
 
-  void TruncateFrame(std::int64_t at, IoErrorHandler &handler) {
+  RT_API_ATTRS void TruncateFrame(std::int64_t at, IoErrorHandler &handler) {
     RUNTIME_CHECK(handler, !dirty_);
     if (at <= fileOffset_) {
       Reset(at);
@@ -138,9 +143,10 @@ public:
   }
 
 private:
-  STORE &Store() { return static_cast<STORE &>(*this); }
+  RT_API_ATTRS STORE &Store() { return static_cast<STORE &>(*this); }
 
-  void Reallocate(std::int64_t bytes, const Terminator &terminator) {
+  RT_API_ATTRS void Reallocate(
+      std::int64_t bytes, const Terminator &terminator) {
     if (bytes > size_) {
       char *old{buffer_};
       auto oldSize{size_};
@@ -160,13 +166,14 @@ private:
     }
   }
 
-  void Reset(FileOffset at) {
+  RT_API_ATTRS void Reset(FileOffset at) {
     start_ = length_ = frame_ = 0;
     fileOffset_ = at;
     dirty_ = false;
   }
 
-  void DiscardLeadingBytes(std::int64_t n, const Terminator &terminator) {
+  RT_API_ATTRS void DiscardLeadingBytes(
+      std::int64_t n, const Terminator &terminator) {
     RUNTIME_CHECK(terminator, length_ >= n);
     length_ -= n;
     if (length_ == 0) {
@@ -185,19 +192,20 @@ private:
     fileOffset_ += n;
   }
 
-  void MakeDataContiguous(IoErrorHandler &handler, std::size_t bytes) {
+  RT_API_ATTRS void MakeDataContiguous(
+      IoErrorHandler &handler, std::size_t bytes) {
     if (static_cast<std::int64_t>(start_ + bytes) > size_) {
       // Frame would wrap around; shift current data (if any) to force
       // contiguity.
       RUNTIME_CHECK(handler, length_ < size_);
       if (start_ + length_ <= size_) {
         // [......abcde..] -> [abcde........]
-        std::memmove(buffer_, buffer_ + start_, length_);
+        runtime::memmove(buffer_, buffer_ + start_, length_);
       } else {
         // [cde........ab] -> [abcde........]
         auto n{start_ + length_ - size_}; // 3 for cde
         RUNTIME_CHECK(handler, length_ >= n);
-        std::memmove(buffer_ + n, buffer_ + start_, length_ - n); // cdeab
+        runtime::memmove(buffer_ + n, buffer_ + start_, length_ - n); // cdeab
         LeftShiftBufferCircularly(buffer_, length_, n); // abcde
       }
       start_ = 0;
diff --git a/flang/runtime/connection.cpp b/flang/runtime/connection.cpp
index 91ac9a0e14e4..f24f0e832eb4 100644
--- a/flang/runtime/connection.cpp
+++ b/flang/runtime/connection.cpp
@@ -12,30 +12,31 @@
 #include <algorithm>
 
 namespace Fortran::runtime::io {
+RT_OFFLOAD_API_GROUP_BEGIN
 
-std::size_t ConnectionState::RemainingSpaceInRecord() const {
+RT_API_ATTRS std::size_t ConnectionState::RemainingSpaceInRecord() const {
   auto recl{recordLength.value_or(openRecl.value_or(
       executionEnvironment.listDirectedOutputLineLengthLimit))};
   return positionInRecord >= recl ? 0 : recl - positionInRecord;
 }
 
-bool ConnectionState::NeedAdvance(std::size_t width) const {
+RT_API_ATTRS bool ConnectionState::NeedAdvance(std::size_t width) const {
   return positionInRecord > 0 && width > RemainingSpaceInRecord();
 }
 
-bool ConnectionState::IsAtEOF() const {
+RT_API_ATTRS bool ConnectionState::IsAtEOF() const {
   return endfileRecordNumber && currentRecordNumber >= *endfileRecordNumber;
 }
 
-bool ConnectionState::IsAfterEndfile() const {
+RT_API_ATTRS bool ConnectionState::IsAfterEndfile() const {
   return endfileRecordNumber && currentRecordNumber > *endfileRecordNumber;
 }
 
-void ConnectionState::HandleAbsolutePosition(std::int64_t n) {
+RT_API_ATTRS void ConnectionState::HandleAbsolutePosition(std::int64_t n) {
   positionInRecord = std::max(n, std::int64_t{0}) + leftTabLimit.value_or(0);
 }
 
-void ConnectionState::HandleRelativePosition(std::int64_t n) {
+RT_API_ATTRS void ConnectionState::HandleRelativePosition(std::int64_t n) {
   positionInRecord = std::max(leftTabLimit.value_or(0), positionInRecord + n);
 }
 
@@ -57,4 +58,6 @@ SavedPosition::~SavedPosition() {
     conn.pinnedFrame = saved_.pinnedFrame;
   }
 }
+
+RT_OFFLOAD_API_GROUP_END
 } // namespace Fortran::runtime::io
diff --git a/flang/runtime/connection.h b/flang/runtime/connection.h
index c41970d47e7b..6f1ea90a160e 100644
--- a/flang/runtime/connection.h
+++ b/flang/runtime/connection.h
@@ -31,12 +31,12 @@ struct ConnectionAttributes {
   unsigned char internalIoCharKind{0}; // 0->external, 1/2/4->internal
   Fortran::common::optional<std::int64_t> openRecl; // RECL= on OPEN
 
-  bool IsRecordFile() const {
+  RT_API_ATTRS bool IsRecordFile() const {
     // Formatted stream files are viewed as having records, at least on input
     return access != Access::Stream || !isUnformatted.value_or(true);
   }
 
-  template <typename CHAR = char> constexpr bool useUTF8() const {
+  template <typename CHAR = char> constexpr RT_API_ATTRS bool useUTF8() const {
     // For wide CHARACTER kinds, always use UTF-8 for formatted I/O.
     // For single-byte CHARACTER, encode characters >= 0x80 with
     // UTF-8 iff the mode is set.
@@ -45,25 +45,28 @@ struct ConnectionAttributes {
 };
 
 struct ConnectionState : public ConnectionAttributes {
-  bool IsAtEOF() const; // true when read has hit EOF or endfile record
-  bool IsAfterEndfile() const; // true after ENDFILE until repositioned
+  RT_API_ATTRS bool
+  IsAtEOF() const; // true when read has hit EOF or endfile record
+  RT_API_ATTRS bool
+  IsAfterEndfile() const; // true after ENDFILE until repositioned
 
   // All positions and measurements are always in units of bytes,
   // not characters.  Multi-byte character encodings are possible in
   // both internal I/O (when the character kind of the variable is 2 or 4)
   // and external formatted I/O (when the encoding is UTF-8).
-  std::size_t RemainingSpaceInRecord() const;
-  bool NeedAdvance(std::size_t) const;
-  void HandleAbsolutePosition(std::int64_t);
-  void HandleRelativePosition(std::int64_t);
+  RT_API_ATTRS std::size_t RemainingSpaceInRecord() const;
+  RT_API_ATTRS bool NeedAdvance(std::size_t) const;
+  RT_API_ATTRS void HandleAbsolutePosition(std::int64_t);
+  RT_API_ATTRS void HandleRelativePosition(std::int64_t);
 
-  void BeginRecord() {
+  RT_API_ATTRS void BeginRecord() {
     positionInRecord = 0;
     furthestPositionInRecord = 0;
     unterminatedRecord = false;
   }
 
-  Fortran::common::optional<std::int64_t> EffectiveRecordLength() const {
+  RT_API_ATTRS Fortran::common::optional<std::int64_t>
+  EffectiveRecordLength() const {
     // When an input record is longer than an explicit RECL= from OPEN
     // it is effectively truncated on input.
     return openRecl && recordLength && *openRecl < *recordLength ? openRecl
@@ -110,9 +113,9 @@ struct ConnectionState : public ConnectionAttributes {
 // Utility class for capturing and restoring a position in an input stream.
 class SavedPosition {
 public:
-  explicit SavedPosition(IoStatementState &);
-  ~SavedPosition();
-  void Cancel() { cancelled_ = true; }
+  explicit RT_API_ATTRS SavedPosition(IoStatementState &);
+  RT_API_ATTRS ~SavedPosition();
+  RT_API_ATTRS void Cancel() { cancelled_ = true; }
 
 private:
   IoStatementState &io_;
diff --git a/flang/runtime/derived.h b/flang/runtime/derived.h
index e43ecc34a31d..b4863df8db41 100644
--- a/flang/runtime/derived.h
+++ b/flang/runtime/derived.h
@@ -11,7 +11,7 @@
 #ifndef FORTRAN_RUNTIME_DERIVED_H_
 #define FORTRAN_RUNTIME_DERIVED_H_
 
-#include "flang/Runtime/api-attrs.h"
+#include "flang/Common/api-attrs.h"
 
 namespace Fortran::runtime::typeInfo {
 class DerivedType;
diff --git a/flang/runtime/descriptor-io.cpp b/flang/runtime/descriptor-io.cpp
index 7c7323b719ad..93df51cf22d3 100644
--- a/flang/runtime/descriptor-io.cpp
+++ b/flang/runtime/descriptor-io.cpp
@@ -7,9 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "descriptor-io.h"
+#include "freestanding-tools.h"
 #include "flang/Common/restorer.h"
 
 namespace Fortran::runtime::io::descr {
+RT_OFFLOAD_API_GROUP_BEGIN
 
 // Defined formatted I/O (maybe)
 Fortran::common::optional<bool> DefinedFormattedIo(IoStatementState &io,
@@ -32,9 +34,9 @@ Fortran::common::optional<bool> DefinedFormattedIo(IoStatementState &io,
       ioType[1] = 'T';
       std::memcpy(ioType + 2, edit.ioType, edit.ioTypeChars);
     } else {
-      std::strcpy(
+      runtime::strcpy(
           ioType, io.mutableModes().inNamelist ? "NAMELIST" : "LISTDIRECTED");
-      ioTypeLen = std::strlen(ioType);
+      ioTypeLen = runtime::strlen(ioType);
     }
     StaticDescriptor<1, true> vListStatDesc;
     Descriptor &vListDesc{vListStatDesc.descriptor()};
@@ -150,4 +152,5 @@ bool DefinedUnformattedIo(IoStatementState &io, const Descriptor &descriptor,
   return handler.GetIoStat() == IostatOk;
 }
 
+RT_OFFLOAD_API_GROUP_END
 } // namespace Fortran::runtime::io::descr
diff --git a/flang/runtime/descriptor-io.h b/flang/runtime/descriptor-io.h
index b6b0fefcff87..7063858d6191 100644
--- a/flang/runtime/descriptor-io.h
+++ b/flang/runtime/descriptor-io.h
@@ -28,8 +28,8 @@
 
 namespace Fortran::runtime::io::descr {
 template <typename A>
-inline A &ExtractElement(IoStatementState &io, const Descriptor &descriptor,
-    const SubscriptValue subscripts[]) {
+inline RT_API_ATTRS A &ExtractElement(IoStatementState &io,
+    const Descriptor &descriptor, const SubscriptValue subscripts[]) {
   A *p{descriptor.Element<A>(subscripts)};
   if (!p) {
     io.GetIoErrorHandler().Crash("Bad address for I/O item -- null base "
@@ -45,7 +45,7 @@ inline A &ExtractElement(IoStatementState &io, const Descriptor &descriptor,
 // NAMELIST array output.
 
 template <int KIND, Direction DIR>
-inline bool FormattedIntegerIO(
+inline RT_API_ATTRS bool FormattedIntegerIO(
     IoStatementState &io, const Descriptor &descriptor) {
   std::size_t numElements{descriptor.Elements()};
   SubscriptValue subscripts[maxRank];
@@ -78,7 +78,7 @@ inline bool FormattedIntegerIO(
 }
 
 template <int KIND, Direction DIR>
-inline bool FormattedRealIO(
+inline RT_API_ATTRS bool FormattedRealIO(
     IoStatementState &io, const Descriptor &descriptor) {
   std::size_t numElements{descriptor.Elements()};
   SubscriptValue subscripts[maxRank];
@@ -111,7 +111,7 @@ inline bool FormattedRealIO(
 }
 
 template <int KIND, Direction DIR>
-inline bool FormattedComplexIO(
+inline RT_API_ATTRS bool FormattedComplexIO(
     IoStatementState &io, const Descriptor &descriptor) {
   std::size_t numElements{descriptor.Elements()};
   SubscriptValue subscripts[maxRank];
@@ -159,7 +159,7 @@ inline bool FormattedComplexIO(
 }
 
 template <typename A, Direction DIR>
-inline bool FormattedCharacterIO(
+inline RT_API_ATTRS bool FormattedCharacterIO(
     IoStatementState &io, const Descriptor &descriptor) {
   std::size_t numElements{descriptor.Elements()};
   SubscriptValue subscripts[maxRank];
@@ -199,7 +199,7 @@ inline bool FormattedCharacterIO(
 }
 
 template <int KIND, Direction DIR>
-inline bool FormattedLogicalIO(
+inline RT_API_ATTRS bool FormattedLogicalIO(
     IoStatementState &io, const Descriptor &descriptor) {
   std::size_t numElements{descriptor.Elements()};
   SubscriptValue subscripts[maxRank];
@@ -241,12 +241,12 @@ inline bool FormattedLogicalIO(
 }
 
 template <Direction DIR>
-static bool DescriptorIO(IoStatementState &, const Descriptor &,
+static RT_API_ATTRS bool DescriptorIO(IoStatementState &, const Descriptor &,
     const NonTbpDefinedIoTable * = nullptr);
 
 // For intrinsic (not defined) derived type I/O, formatted & unformatted
 template <Direction DIR>
-static bool DefaultComponentIO(IoStatementState &io,
+static RT_API_ATTRS bool DefaultComponentIO(IoStatementState &io,
     const typeInfo::Component &component, const Descriptor &origDescriptor,
     const SubscriptValue origSubscripts[], Terminator &terminator,
     const NonTbpDefinedIoTable *table) {
@@ -269,7 +269,7 @@ static bool DefaultComponentIO(IoStatementState &io,
 }
 
 template <Direction DIR>
-static bool DefaultComponentwiseFormattedIO(IoStatementState &io,
+static RT_API_ATTRS bool DefaultComponentwiseFormattedIO(IoStatementState &io,
     const Descriptor &descriptor, const typeInfo::DerivedType &type,
     const NonTbpDefinedIoTable *table, const SubscriptValue subscripts[]) {
   IoErrorHandler &handler{io.GetIoErrorHandler()};
@@ -295,7 +295,7 @@ static bool DefaultComponentwiseFormattedIO(IoStatementState &io,
 }
 
 template <Direction DIR>
-static bool DefaultComponentwiseUnformattedIO(IoStatementState &io,
+static RT_API_ATTRS bool DefaultComponentwiseUnformattedIO(IoStatementState &io,
     const Descriptor &descriptor, const typeInfo::DerivedType &type,
     const NonTbpDefinedIoTable *table) {
   IoErrorHandler &handler{io.GetIoErrorHandler()};
@@ -322,12 +322,12 @@ static bool DefaultComponentwiseUnformattedIO(IoStatementState &io,
   return true;
 }
 
-Fortran::common::optional<bool> DefinedFormattedIo(IoStatementState &,
-    const Descriptor &, const typeInfo::DerivedType &,
+RT_API_ATTRS Fortran::common::optional<bool> DefinedFormattedIo(
+    IoStatementState &, const Descriptor &, const typeInfo::DerivedType &,
     const typeInfo::SpecialBinding &, const SubscriptValue[]);
 
 template <Direction DIR>
-static bool FormattedDerivedTypeIO(IoStatementState &io,
+static RT_API_ATTRS bool FormattedDerivedTypeIO(IoStatementState &io,
     const Descriptor &descriptor, const NonTbpDefinedIoTable *table) {
   IoErrorHandler &handler{io.GetIoErrorHandler()};
   // Derived type information must be present for formatted I/O.
@@ -385,12 +385,12 @@ static bool FormattedDerivedTypeIO(IoStatementState &io,
   return true;
 }
 
-bool DefinedUnformattedIo(IoStatementState &, const Descriptor &,
+RT_API_ATTRS bool DefinedUnformattedIo(IoStatementState &, const Descriptor &,
     const typeInfo::DerivedType &, const typeInfo::SpecialBinding &);
 
 // Unformatted I/O
 template <Direction DIR>
-static bool UnformattedDescriptorIO(IoStatementState &io,
+static RT_API_ATTRS bool UnformattedDescriptorIO(IoStatementState &io,
     const Descriptor &descriptor, const NonTbpDefinedIoTable *table = nullptr) {
   IoErrorHandler &handler{io.GetIoErrorHandler()};
   const DescriptorAddendum *addendum{descriptor.Addendum()};
@@ -488,8 +488,8 @@ static bool UnformattedDescriptorIO(IoStatementState &io,
 }
 
 template <Direction DIR>
-static bool DescriptorIO(IoStatementState &io, const Descriptor &descriptor,
-    const NonTbpDefinedIoTable *table) {
+static RT_API_ATTRS bool DescriptorIO(IoStatementState &io,
+    const Descriptor &descriptor, const NonTbpDefinedIoTable *table) {
   IoErrorHandler &handler{io.GetIoErrorHandler()};
   if (handler.InError()) {
     return false;
diff --git a/flang/runtime/edit-input.cpp b/flang/runtime/edit-input.cpp
index fbeb1a595b32..935b7c299b25 100644
--- a/flang/runtime/edit-input.cpp
+++ b/flang/runtime/edit-input.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "edit-input.h"
+#include "freestanding-tools.h"
 #include "namelist.h"
 #include "utf.h"
 #include "flang/Common/optional.h"
@@ -16,17 +17,19 @@
 #include <cfenv>
 
 namespace Fortran::runtime::io {
+RT_OFFLOAD_API_GROUP_BEGIN
 
 // Checks that a list-directed input value has been entirely consumed and
 // doesn't contain unparsed characters before the next value separator.
-static inline bool IsCharValueSeparator(const DataEdit &edit, char32_t ch) {
+static inline RT_API_ATTRS bool IsCharValueSeparator(
+    const DataEdit &edit, char32_t ch) {
   char32_t comma{
       edit.modes.editingFlags & decimalComma ? char32_t{';'} : char32_t{','}};
   return ch == ' ' || ch == '\t' || ch == comma || ch == '/' ||
       (edit.IsNamelist() && (ch == '&' || ch == '$'));
 }
 
-static bool CheckCompleteListDirectedField(
+static RT_API_ATTRS bool CheckCompleteListDirectedField(
     IoStatementState &io, const DataEdit &edit) {
   if (edit.IsListDirected()) {
     std::size_t byteCount;
@@ -52,7 +55,7 @@ static bool CheckCompleteListDirectedField(
 }
 
 template <int LOG2_BASE>
-static bool EditBOZInput(
+static RT_API_ATTRS bool EditBOZInput(
     IoStatementState &io, const DataEdit &edit, void *n, std::size_t bytes) {
   // Skip leading white space & zeroes
   Fortran::common::optional<int> remaining{io.CueUpInput(edit)};
@@ -151,13 +154,13 @@ static bool EditBOZInput(
   return CheckCompleteListDirectedField(io, edit);
 }
 
-static inline char32_t GetRadixPointChar(const DataEdit &edit) {
+static inline RT_API_ATTRS char32_t GetRadixPointChar(const DataEdit &edit) {
   return edit.modes.editingFlags & decimalComma ? char32_t{','} : char32_t{'.'};
 }
 
 // Prepares input from a field, and returns the sign, if any, else '\0'.
-static char ScanNumericPrefix(IoStatementState &io, const DataEdit &edit,
-    Fortran::common::optional<char32_t> &next,
+static RT_API_ATTRS char ScanNumericPrefix(IoStatementState &io,
+    const DataEdit &edit, Fortran::common::optional<char32_t> &next,
     Fortran::common::optional<int> &remaining) {
   remaining = io.CueUpInput(edit);
   next = io.NextInField(remaining, edit);
@@ -174,7 +177,7 @@ static char ScanNumericPrefix(IoStatementState &io, const DataEdit &edit,
   return sign;
 }
 
-bool EditIntegerInput(
+RT_API_ATTRS bool EditIntegerInput(
     IoStatementState &io, const DataEdit &edit, void *n, int kind) {
   RUNTIME_CHECK(io.GetIoErrorHandler(), kind >= 1 && !(kind & (kind - 1)));
   switch (edit.descriptor) {
@@ -279,18 +282,20 @@ struct ScannedRealInput {
   int exponent{0}; // adjusted as necessary; binary if isHexadecimal
   bool isHexadecimal{false}; // 0X...
 };
-static ScannedRealInput ScanRealInput(
+static RT_API_ATTRS ScannedRealInput ScanRealInput(
     char *buffer, int bufferSize, IoStatementState &io, const DataEdit &edit) {
   Fortran::common::optional<int> remaining;
   Fortran::common::optional<char32_t> next;
   int got{0};
   Fortran::common::optional<int> radixPointOffset;
-  auto Put{[&](char ch) -> void {
+  // The following lambda definition violates the conding style,
+  // but cuda-11.8 nvcc hits an internal error with the brace initialization.
+  auto Put = [&](char ch) -> void {
     if (got < bufferSize) {
       buffer[got] = ch;
     }
     ++got;
-  }};
+  };
   char sign{ScanNumericPrefix(io, edit, next, remaining)};
   if (sign == '-') {
     Put('-');
@@ -487,13 +492,21 @@ static ScannedRealInput ScanRealInput(
   return {got, exponent, isHexadecimal};
 }
 
-static void RaiseFPExceptions(decimal::ConversionResultFlags flags) {
+static RT_API_ATTRS void RaiseFPExceptions(
+    decimal::ConversionResultFlags flags) {
 #undef RAISE
+#if defined(RT_DEVICE_COMPILATION)
+  Terminator terminator(__FILE__, __LINE__);
+#define RAISE(e) \
+  terminator.Crash( \
+      "not implemented yet: raising FP exception in device code: %s", #e);
+#else // !defined(RT_DEVICE_COMPILATION)
 #ifdef feraisexcept // a macro in some environments; omit std::
 #define RAISE feraiseexcept
 #else
 #define RAISE std::feraiseexcept
 #endif
+#endif // !defined(RT_DEVICE_COMPILATION)
   if (flags & decimal::ConversionResultFlags::Overflow) {
     RAISE(FE_OVERFLOW);
   }
@@ -514,7 +527,7 @@ static void RaiseFPExceptions(decimal::ConversionResultFlags flags) {
 // converter without modification, this fast path for real input
 // saves time by avoiding memory copies and reformatting of the exponent.
 template <int PRECISION>
-static bool TryFastPathRealDecimalInput(
+static RT_API_ATTRS bool TryFastPathRealDecimalInput(
     IoStatementState &io, const DataEdit &edit, void *n) {
   if (edit.modes.editingFlags & (blankZero | decimalComma)) {
     return false;
@@ -586,7 +599,8 @@ static bool TryFastPathRealDecimalInput(
 }
 
 template <int binaryPrecision>
-decimal::ConversionToBinaryResult<binaryPrecision> ConvertHexadecimal(
+RT_API_ATTRS decimal::ConversionToBinaryResult<binaryPrecision>
+ConvertHexadecimal(
     const char *&p, enum decimal::FortranRounding rounding, int expo) {
   using RealType = decimal::BinaryFloatingPointNumber<binaryPrecision>;
   using RawType = typename RealType::RawType;
@@ -702,7 +716,8 @@ decimal::ConversionToBinaryResult<binaryPrecision> ConvertHexadecimal(
 }
 
 template <int KIND>
-bool EditCommonRealInput(IoStatementState &io, const DataEdit &edit, void *n) {
+RT_API_ATTRS bool EditCommonRealInput(
+    IoStatementState &io, const DataEdit &edit, void *n) {
   constexpr int binaryPrecision{common::PrecisionOfRealKind(KIND)};
   if (TryFastPathRealDecimalInput<binaryPrecision>(io, edit, n)) {
     return CheckCompleteListDirectedField(io, edit);
@@ -798,7 +813,8 @@ bool EditCommonRealInput(IoStatementState &io, const DataEdit &edit, void *n) {
 }
 
 template <int KIND>
-bool EditRealInput(IoStatementState &io, const DataEdit &edit, void *n) {
+RT_API_ATTRS bool EditRealInput(
+    IoStatementState &io, const DataEdit &edit, void *n) {
   switch (edit.descriptor) {
   case DataEdit::ListDirected:
     if (IsNamelistNameOrSlash(io)) {
@@ -832,7 +848,8 @@ bool EditRealInput(IoStatementState &io, const DataEdit &edit, void *n) {
 }
 
 // 13.7.3 in Fortran 2018
-bool EditLogicalInput(IoStatementState &io, const DataEdit &edit, bool &x) {
+RT_API_ATTRS bool EditLogicalInput(
+    IoStatementState &io, const DataEdit &edit, bool &x) {
   switch (edit.descriptor) {
   case DataEdit::ListDirected:
     if (IsNamelistNameOrSlash(io)) {
@@ -882,7 +899,7 @@ bool EditLogicalInput(IoStatementState &io, const DataEdit &edit, bool &x) {
 
 // See 13.10.3.1 paragraphs 7-9 in Fortran 2018
 template <typename CHAR>
-static bool EditDelimitedCharacterInput(
+static RT_API_ATTRS bool EditDelimitedCharacterInput(
     IoStatementState &io, CHAR *x, std::size_t length, char32_t delimiter) {
   bool result{true};
   while (true) {
@@ -911,12 +928,12 @@ static bool EditDelimitedCharacterInput(
       --length;
     }
   }
-  std::fill_n(x, length, ' ');
+  Fortran::runtime::fill_n(x, length, ' ');
   return result;
 }
 
 template <typename CHAR>
-static bool EditListDirectedCharacterInput(
+static RT_API_ATTRS bool EditListDirectedCharacterInput(
     IoStatementState &io, CHAR *x, std::size_t length, const DataEdit &edit) {
   std::size_t byteCount{0};
   auto ch{io.GetCurrentChar(byteCount)};
@@ -961,13 +978,13 @@ static bool EditListDirectedCharacterInput(
       remaining = --length > 0 ? maxUTF8Bytes : 0;
     }
   }
-  std::fill_n(x, length, ' ');
+  Fortran::runtime::fill_n(x, length, ' ');
   return true;
 }
 
 template <typename CHAR>
-bool EditCharacterInput(IoStatementState &io, const DataEdit &edit, CHAR *x,
-    std::size_t lengthChars) {
+RT_API_ATTRS bool EditCharacterInput(IoStatementState &io, const DataEdit &edit,
+    CHAR *x, std::size_t lengthChars) {
   switch (edit.descriptor) {
   case DataEdit::ListDirected:
     return EditListDirectedCharacterInput(io, x, lengthChars, edit);
@@ -1011,7 +1028,7 @@ bool EditCharacterInput(IoStatementState &io, const DataEdit &edit, CHAR *x,
         if (io.CheckForEndOfRecord(readyBytes)) {
           if (readyBytes == 0) {
             // PAD='YES' and no more data
-            std::fill_n(x, lengthChars, ' ');
+            Fortran::runtime::fill_n(x, lengthChars, ' ');
             return !io.GetIoErrorHandler().InError();
           } else {
             // Do partial read(s) then pad on last iteration
@@ -1088,23 +1105,30 @@ bool EditCharacterInput(IoStatementState &io, const DataEdit &edit, CHAR *x,
     readyBytes -= chunkBytes;
   }
   // Pad the remainder of the input variable, if any.
-  std::fill_n(x, lengthChars, ' ');
+  Fortran::runtime::fill_n(x, lengthChars, ' ');
   return CheckCompleteListDirectedField(io, edit);
 }
 
-template bool EditRealInput<2>(IoStatementState &, const DataEdit &, void *);
-template bool EditRealInput<3>(IoStatementState &, const DataEdit &, void *);
-template bool EditRealInput<4>(IoStatementState &, const DataEdit &, void *);
-template bool EditRealInput<8>(IoStatementState &, const DataEdit &, void *);
-template bool EditRealInput<10>(IoStatementState &, const DataEdit &, void *);
+template RT_API_ATTRS bool EditRealInput<2>(
+    IoStatementState &, const DataEdit &, void *);
+template RT_API_ATTRS bool EditRealInput<3>(
+    IoStatementState &, const DataEdit &, void *);
+template RT_API_ATTRS bool EditRealInput<4>(
+    IoStatementState &, const DataEdit &, void *);
+template RT_API_ATTRS bool EditRealInput<8>(
+    IoStatementState &, const DataEdit &, void *);
+template RT_API_ATTRS bool EditRealInput<10>(
+    IoStatementState &, const DataEdit &, void *);
 // TODO: double/double
-template bool EditRealInput<16>(IoStatementState &, const DataEdit &, void *);
+template RT_API_ATTRS bool EditRealInput<16>(
+    IoStatementState &, const DataEdit &, void *);
 
-template bool EditCharacterInput(
+template RT_API_ATTRS bool EditCharacterInput(
     IoStatementState &, const DataEdit &, char *, std::size_t);
-template bool EditCharacterInput(
+template RT_API_ATTRS bool EditCharacterInput(
     IoStatementState &, const DataEdit &, char16_t *, std::size_t);
-template bool EditCharacterInput(
+template RT_API_ATTRS bool EditCharacterInput(
     IoStatementState &, const DataEdit &, char32_t *, std::size_t);
 
+RT_OFFLOAD_API_GROUP_END
 } // namespace Fortran::runtime::io
diff --git a/flang/runtime/edit-input.h b/flang/runtime/edit-input.h
index 61844a1199a7..a90180b8ee2e 100644
--- a/flang/runtime/edit-input.h
+++ b/flang/runtime/edit-input.h
@@ -15,36 +15,38 @@
 
 namespace Fortran::runtime::io {
 
-bool EditIntegerInput(IoStatementState &, const DataEdit &, void *, int kind);
+RT_API_ATTRS bool EditIntegerInput(
+    IoStatementState &, const DataEdit &, void *, int kind);
 
 template <int KIND>
-bool EditRealInput(IoStatementState &, const DataEdit &, void *);
+RT_API_ATTRS bool EditRealInput(IoStatementState &, const DataEdit &, void *);
 
-bool EditLogicalInput(IoStatementState &, const DataEdit &, bool &);
+RT_API_ATTRS bool EditLogicalInput(
+    IoStatementState &, const DataEdit &, bool &);
 
 template <typename CHAR>
-bool EditCharacterInput(
+RT_API_ATTRS bool EditCharacterInput(
     IoStatementState &, const DataEdit &, CHAR *, std::size_t);
 
-extern template bool EditRealInput<2>(
+extern template RT_API_ATTRS bool EditRealInput<2>(
     IoStatementState &, const DataEdit &, void *);
-extern template bool EditRealInput<3>(
+extern template RT_API_ATTRS bool EditRealInput<3>(
     IoStatementState &, const DataEdit &, void *);
-extern template bool EditRealInput<4>(
+extern template RT_API_ATTRS bool EditRealInput<4>(
     IoStatementState &, const DataEdit &, void *);
-extern template bool EditRealInput<8>(
+extern template RT_API_ATTRS bool EditRealInput<8>(
     IoStatementState &, const DataEdit &, void *);
-extern template bool EditRealInput<10>(
+extern template RT_API_ATTRS bool EditRealInput<10>(
     IoStatementState &, const DataEdit &, void *);
 // TODO: double/double
-extern template bool EditRealInput<16>(
+extern template RT_API_ATTRS bool EditRealInput<16>(
     IoStatementState &, const DataEdit &, void *);
 
-extern template bool EditCharacterInput(
+extern template RT_API_ATTRS bool EditCharacterInput(
     IoStatementState &, const DataEdit &, char *, std::size_t);
-extern template bool EditCharacterInput(
+extern template RT_API_ATTRS bool EditCharacterInput(
     IoStatementState &, const DataEdit &, char16_t *, std::size_t);
-extern template bool EditCharacterInput(
+extern template RT_API_ATTRS bool EditCharacterInput(
     IoStatementState &, const DataEdit &, char32_t *, std::size_t);
 
 } // namespace Fortran::runtime::io
diff --git a/flang/runtime/edit-output.cpp b/flang/runtime/edit-output.cpp
index 7267540370fc..b710c298babe 100644
--- a/flang/runtime/edit-output.cpp
+++ b/flang/runtime/edit-output.cpp
@@ -14,9 +14,10 @@
 #include <algorithm>
 
 namespace Fortran::runtime::io {
+RT_OFFLOAD_API_GROUP_BEGIN
 
 // In output statement, add a space between numbers and characters.
-static void addSpaceBeforeCharacter(IoStatementState &io) {
+static RT_API_ATTRS void addSpaceBeforeCharacter(IoStatementState &io) {
   if (auto *list{io.get_if<ListDirectedStatementState<Direction::Output>>()}) {
     list->set_lastWasUndelimitedCharacter(false);
   }
@@ -26,8 +27,8 @@ static void addSpaceBeforeCharacter(IoStatementState &io) {
 // representation of what is interpreted to be a single unsigned integer value.
 // When used with character data, endianness is exposed.
 template <int LOG2_BASE>
-static bool EditBOZOutput(IoStatementState &io, const DataEdit &edit,
-    const unsigned char *data0, std::size_t bytes) {
+static RT_API_ATTRS bool EditBOZOutput(IoStatementState &io,
+    const DataEdit &edit, const unsigned char *data0, std::size_t bytes) {
   addSpaceBeforeCharacter(io);
   int digits{static_cast<int>((bytes * 8) / LOG2_BASE)};
   int get{static_cast<int>(bytes * 8) - digits * LOG2_BASE};
@@ -107,7 +108,7 @@ static bool EditBOZOutput(IoStatementState &io, const DataEdit &edit,
 }
 
 template <int KIND>
-bool EditIntegerOutput(IoStatementState &io, const DataEdit &edit,
+bool RT_API_ATTRS EditIntegerOutput(IoStatementState &io, const DataEdit &edit,
     common::HostSignedIntType<8 * KIND> n) {
   addSpaceBeforeCharacter(io);
   char buffer[130], *end{&buffer[sizeof buffer]}, *p{end};
@@ -187,7 +188,7 @@ bool EditIntegerOutput(IoStatementState &io, const DataEdit &edit,
 }
 
 // Formats the exponent (see table 13.1 for all the cases)
-const char *RealOutputEditingBase::FormatExponent(
+RT_API_ATTRS const char *RealOutputEditingBase::FormatExponent(
     int expo, const DataEdit &edit, int &length) {
   char *eEnd{&exponent_[sizeof exponent_]};
   char *exponent{eEnd};
@@ -226,7 +227,7 @@ const char *RealOutputEditingBase::FormatExponent(
   return overflow ? nullptr : exponent;
 }
 
-bool RealOutputEditingBase::EmitPrefix(
+RT_API_ATTRS bool RealOutputEditingBase::EmitPrefix(
     const DataEdit &edit, std::size_t length, std::size_t width) {
   if (edit.IsListDirected()) {
     int prefixLength{edit.descriptor == DataEdit::ListDirectedRealPart ? 2
@@ -247,7 +248,7 @@ bool RealOutputEditingBase::EmitPrefix(
   }
 }
 
-bool RealOutputEditingBase::EmitSuffix(const DataEdit &edit) {
+RT_API_ATTRS bool RealOutputEditingBase::EmitSuffix(const DataEdit &edit) {
   if (edit.descriptor == DataEdit::ListDirectedRealPart) {
     return EmitAscii(
         io_, edit.modes.editingFlags & decimalComma ? ";" : ",", 1);
@@ -259,8 +260,10 @@ bool RealOutputEditingBase::EmitSuffix(const DataEdit &edit) {
 }
 
 template <int KIND>
-decimal::ConversionToDecimalResult RealOutputEditing<KIND>::ConvertToDecimal(
+RT_API_ATTRS decimal::ConversionToDecimalResult
+RealOutputEditing<KIND>::ConvertToDecimal(
     int significantDigits, enum decimal::FortranRounding rounding, int flags) {
+#if !defined(RT_DEVICE_COMPILATION)
   auto converted{decimal::ConvertToDecimal<binaryPrecision>(buffer_,
       sizeof buffer_, static_cast<enum decimal::DecimalConversionFlags>(flags),
       significantDigits, rounding, x_)};
@@ -270,9 +273,13 @@ decimal::ConversionToDecimalResult RealOutputEditing<KIND>::ConvertToDecimal(
         sizeof buffer_);
   }
   return converted;
+#else // defined(RT_DEVICE_COMPILATION)
+  // TODO: enable Decimal library build for the device.
+  io_.GetIoErrorHandler().Crash("not implemented yet: decimal conversion");
+#endif // defined(RT_DEVICE_COMPILATION)
 }
 
-static bool IsInfOrNaN(const char *p, int length) {
+static RT_API_ATTRS bool IsInfOrNaN(const char *p, int length) {
   if (!p || length < 1) {
     return false;
   }
@@ -287,7 +294,8 @@ static bool IsInfOrNaN(const char *p, int length) {
 
 // 13.7.2.3.3 in F'2018
 template <int KIND>
-bool RealOutputEditing<KIND>::EditEorDOutput(const DataEdit &edit) {
+RT_API_ATTRS bool RealOutputEditing<KIND>::EditEorDOutput(
+    const DataEdit &edit) {
   addSpaceBeforeCharacter(io_);
   int editDigits{edit.digits.value_or(0)}; // 'd' field
   int editWidth{edit.width.value_or(0)}; // 'w' field
@@ -423,7 +431,7 @@ bool RealOutputEditing<KIND>::EditEorDOutput(const DataEdit &edit) {
 
 // 13.7.2.3.2 in F'2018
 template <int KIND>
-bool RealOutputEditing<KIND>::EditFOutput(const DataEdit &edit) {
+RT_API_ATTRS bool RealOutputEditing<KIND>::EditFOutput(const DataEdit &edit) {
   addSpaceBeforeCharacter(io_);
   int fracDigits{edit.digits.value_or(0)}; // 'd' field
   const int editWidth{edit.width.value_or(0)}; // 'w' field
@@ -553,12 +561,12 @@ bool RealOutputEditing<KIND>::EditFOutput(const DataEdit &edit) {
 
 // 13.7.5.2.3 in F'2018
 template <int KIND>
-DataEdit RealOutputEditing<KIND>::EditForGOutput(DataEdit edit) {
+RT_API_ATTRS DataEdit RealOutputEditing<KIND>::EditForGOutput(DataEdit edit) {
   edit.descriptor = 'E';
   edit.variation = 'G'; // to suppress error for Ew.0
   int editWidth{edit.width.value_or(0)};
-  int significantDigits{
-      edit.digits.value_or(BinaryFloatingPoint::decimalPrecision)}; // 'd'
+  int significantDigits{edit.digits.value_or(
+      static_cast<int>(BinaryFloatingPoint::decimalPrecision))}; // 'd'
   if (editWidth > 0 && significantDigits == 0) {
     return edit; // Gw.0Ee -> Ew.0Ee for w > 0
   }
@@ -597,7 +605,8 @@ DataEdit RealOutputEditing<KIND>::EditForGOutput(DataEdit edit) {
 
 // 13.10.4 in F'2018
 template <int KIND>
-bool RealOutputEditing<KIND>::EditListDirectedOutput(const DataEdit &edit) {
+RT_API_ATTRS bool RealOutputEditing<KIND>::EditListDirectedOutput(
+    const DataEdit &edit) {
   decimal::ConversionToDecimalResult converted{
       ConvertToDecimal(1, edit.modes.round)};
   if (IsInfOrNaN(converted.str, static_cast<int>(converted.length))) {
@@ -631,9 +640,9 @@ bool RealOutputEditing<KIND>::EditListDirectedOutput(const DataEdit &edit) {
 // E.g., 2. is edited into 0X8.0P-2 rather than 0X2.0P0.  This implementation
 // follows that precedent so as to avoid a gratuitous incompatibility.
 template <int KIND>
-auto RealOutputEditing<KIND>::ConvertToHexadecimal(
-    int significantDigits, enum decimal::FortranRounding rounding, int flags)
-    -> ConvertToHexadecimalResult {
+RT_API_ATTRS auto RealOutputEditing<KIND>::ConvertToHexadecimal(
+    int significantDigits, enum decimal::FortranRounding rounding,
+    int flags) -> ConvertToHexadecimalResult {
   if (x_.IsNaN() || x_.IsInfinite()) {
     auto converted{ConvertToDecimal(significantDigits, rounding, flags)};
     return {converted.str, static_cast<int>(converted.length), 0};
@@ -689,7 +698,7 @@ auto RealOutputEditing<KIND>::ConvertToHexadecimal(
 }
 
 template <int KIND>
-bool RealOutputEditing<KIND>::EditEXOutput(const DataEdit &edit) {
+RT_API_ATTRS bool RealOutputEditing<KIND>::EditEXOutput(const DataEdit &edit) {
   addSpaceBeforeCharacter(io_);
   int editDigits{edit.digits.value_or(0)}; // 'd' field
   int significantDigits{editDigits + 1};
@@ -740,7 +749,8 @@ bool RealOutputEditing<KIND>::EditEXOutput(const DataEdit &edit) {
           EmitAscii(io_, exponent, expoLength);
 }
 
-template <int KIND> bool RealOutputEditing<KIND>::Edit(const DataEdit &edit) {
+template <int KIND>
+RT_API_ATTRS bool RealOutputEditing<KIND>::Edit(const DataEdit &edit) {
   switch (edit.descriptor) {
   case 'D':
     return EditEorDOutput(edit);
@@ -783,13 +793,14 @@ template <int KIND> bool RealOutputEditing<KIND>::Edit(const DataEdit &edit) {
   return false;
 }
 
-bool ListDirectedLogicalOutput(IoStatementState &io,
+RT_API_ATTRS bool ListDirectedLogicalOutput(IoStatementState &io,
     ListDirectedStatementState<Direction::Output> &list, bool truth) {
   return list.EmitLeadingSpaceOrAdvance(io) &&
       EmitAscii(io, truth ? "T" : "F", 1);
 }
 
-bool EditLogicalOutput(IoStatementState &io, const DataEdit &edit, bool truth) {
+RT_API_ATTRS bool EditLogicalOutput(
+    IoStatementState &io, const DataEdit &edit, bool truth) {
   switch (edit.descriptor) {
   case 'L':
   case 'G':
@@ -813,7 +824,7 @@ bool EditLogicalOutput(IoStatementState &io, const DataEdit &edit, bool truth) {
 }
 
 template <typename CHAR>
-bool ListDirectedCharacterOutput(IoStatementState &io,
+RT_API_ATTRS bool ListDirectedCharacterOutput(IoStatementState &io,
     ListDirectedStatementState<Direction::Output> &list, const CHAR *x,
     std::size_t length) {
   bool ok{true};
@@ -870,8 +881,8 @@ bool ListDirectedCharacterOutput(IoStatementState &io,
 }
 
 template <typename CHAR>
-bool EditCharacterOutput(IoStatementState &io, const DataEdit &edit,
-    const CHAR *x, std::size_t length) {
+RT_API_ATTRS bool EditCharacterOutput(IoStatementState &io,
+    const DataEdit &edit, const CHAR *x, std::size_t length) {
   int len{static_cast<int>(length)};
   int width{edit.width.value_or(len)};
   switch (edit.descriptor) {
@@ -903,15 +914,15 @@ bool EditCharacterOutput(IoStatementState &io, const DataEdit &edit,
       EmitEncoded(io, x, std::min(width, len));
 }
 
-template bool EditIntegerOutput<1>(
+template RT_API_ATTRS bool EditIntegerOutput<1>(
     IoStatementState &, const DataEdit &, std::int8_t);
-template bool EditIntegerOutput<2>(
+template RT_API_ATTRS bool EditIntegerOutput<2>(
     IoStatementState &, const DataEdit &, std::int16_t);
-template bool EditIntegerOutput<4>(
+template RT_API_ATTRS bool EditIntegerOutput<4>(
     IoStatementState &, const DataEdit &, std::int32_t);
-template bool EditIntegerOutput<8>(
+template RT_API_ATTRS bool EditIntegerOutput<8>(
     IoStatementState &, const DataEdit &, std::int64_t);
-template bool EditIntegerOutput<16>(
+template RT_API_ATTRS bool EditIntegerOutput<16>(
     IoStatementState &, const DataEdit &, common::int128_t);
 
 template class RealOutputEditing<2>;
@@ -922,21 +933,22 @@ template class RealOutputEditing<10>;
 // TODO: double/double
 template class RealOutputEditing<16>;
 
-template bool ListDirectedCharacterOutput(IoStatementState &,
+template RT_API_ATTRS bool ListDirectedCharacterOutput(IoStatementState &,
     ListDirectedStatementState<Direction::Output> &, const char *,
     std::size_t chars);
-template bool ListDirectedCharacterOutput(IoStatementState &,
+template RT_API_ATTRS bool ListDirectedCharacterOutput(IoStatementState &,
     ListDirectedStatementState<Direction::Output> &, const char16_t *,
     std::size_t chars);
-template bool ListDirectedCharacterOutput(IoStatementState &,
+template RT_API_ATTRS bool ListDirectedCharacterOutput(IoStatementState &,
     ListDirectedStatementState<Direction::Output> &, const char32_t *,
     std::size_t chars);
 
-template bool EditCharacterOutput(
+template RT_API_ATTRS bool EditCharacterOutput(
     IoStatementState &, const DataEdit &, const char *, std::size_t chars);
-template bool EditCharacterOutput(
+template RT_API_ATTRS bool EditCharacterOutput(
     IoStatementState &, const DataEdit &, const char16_t *, std::size_t chars);
-template bool EditCharacterOutput(
+template RT_API_ATTRS bool EditCharacterOutput(
     IoStatementState &, const DataEdit &, const char32_t *, std::size_t chars);
 
+RT_OFFLOAD_API_GROUP_END
 } // namespace Fortran::runtime::io
diff --git a/flang/runtime/edit-output.h b/flang/runtime/edit-output.h
index 4e6d6b25b4dd..365bc2e2a4d1 100644
--- a/flang/runtime/edit-output.h
+++ b/flang/runtime/edit-output.h
@@ -30,18 +30,20 @@ namespace Fortran::runtime::io {
 // one edit descriptor with a repeat factor may safely serve to edit
 // multiple elements of an array.
 template <int KIND>
-bool EditIntegerOutput(
+RT_API_ATTRS bool EditIntegerOutput(
     IoStatementState &, const DataEdit &, common::HostSignedIntType<8 * KIND>);
 
 // Encapsulates the state of a REAL output conversion.
 class RealOutputEditingBase {
 protected:
-  explicit RealOutputEditingBase(IoStatementState &io) : io_{io} {}
+  explicit RT_API_ATTRS RealOutputEditingBase(IoStatementState &io) : io_{io} {}
 
   // Returns null when the exponent overflows a fixed-size output field.
-  const char *FormatExponent(int, const DataEdit &edit, int &length);
-  bool EmitPrefix(const DataEdit &, std::size_t length, std::size_t width);
-  bool EmitSuffix(const DataEdit &);
+  RT_API_ATTRS const char *FormatExponent(
+      int, const DataEdit &edit, int &length);
+  RT_API_ATTRS bool EmitPrefix(
+      const DataEdit &, std::size_t length, std::size_t width);
+  RT_API_ATTRS bool EmitSuffix(const DataEdit &);
 
   IoStatementState &io_;
   int trailingBlanks_{0}; // created when Gw editing maps to Fw
@@ -50,27 +52,29 @@ protected:
 
 template <int KIND> class RealOutputEditing : public RealOutputEditingBase {
 public:
+  RT_VAR_GROUP_BEGIN
   static constexpr int binaryPrecision{common::PrecisionOfRealKind(KIND)};
+  RT_VAR_GROUP_END
   using BinaryFloatingPoint =
       decimal::BinaryFloatingPointNumber<binaryPrecision>;
   template <typename A>
-  RealOutputEditing(IoStatementState &io, A x)
+  RT_API_ATTRS RealOutputEditing(IoStatementState &io, A x)
       : RealOutputEditingBase{io}, x_{x} {}
-  bool Edit(const DataEdit &);
+  RT_API_ATTRS bool Edit(const DataEdit &);
 
 private:
   // The DataEdit arguments here are const references or copies so that
   // the original DataEdit can safely serve multiple array elements when
   // it has a repeat count.
-  bool EditEorDOutput(const DataEdit &);
-  bool EditFOutput(const DataEdit &);
-  DataEdit EditForGOutput(DataEdit); // returns an E or F edit
-  bool EditEXOutput(const DataEdit &);
-  bool EditListDirectedOutput(const DataEdit &);
+  RT_API_ATTRS bool EditEorDOutput(const DataEdit &);
+  RT_API_ATTRS bool EditFOutput(const DataEdit &);
+  RT_API_ATTRS DataEdit EditForGOutput(DataEdit); // returns an E or F edit
+  RT_API_ATTRS bool EditEXOutput(const DataEdit &);
+  RT_API_ATTRS bool EditListDirectedOutput(const DataEdit &);
 
-  bool IsZero() const { return x_.IsZero(); }
+  RT_API_ATTRS bool IsZero() const { return x_.IsZero(); }
 
-  decimal::ConversionToDecimalResult ConvertToDecimal(
+  RT_API_ATTRS decimal::ConversionToDecimalResult ConvertToDecimal(
       int significantDigits, enum decimal::FortranRounding, int flags = 0);
 
   struct ConvertToHexadecimalResult {
@@ -78,7 +82,7 @@ private:
     int length;
     int exponent;
   };
-  ConvertToHexadecimalResult ConvertToHexadecimal(
+  RT_API_ATTRS ConvertToHexadecimalResult ConvertToHexadecimal(
       int significantDigits, enum decimal::FortranRounding, int flags = 0);
 
   BinaryFloatingPoint x_;
@@ -86,43 +90,43 @@ private:
       EXTRA_DECIMAL_CONVERSION_SPACE];
 };
 
-bool ListDirectedLogicalOutput(
+RT_API_ATTRS bool ListDirectedLogicalOutput(
     IoStatementState &, ListDirectedStatementState<Direction::Output> &, bool);
-bool EditLogicalOutput(IoStatementState &, const DataEdit &, bool);
+RT_API_ATTRS bool EditLogicalOutput(IoStatementState &, const DataEdit &, bool);
 
 template <typename CHAR>
-bool ListDirectedCharacterOutput(IoStatementState &,
+RT_API_ATTRS bool ListDirectedCharacterOutput(IoStatementState &,
     ListDirectedStatementState<Direction::Output> &, const CHAR *,
     std::size_t chars);
-extern template bool ListDirectedCharacterOutput(IoStatementState &,
-    ListDirectedStatementState<Direction::Output> &, const char *,
-    std::size_t chars);
-extern template bool ListDirectedCharacterOutput(IoStatementState &,
-    ListDirectedStatementState<Direction::Output> &, const char16_t *,
-    std::size_t chars);
-extern template bool ListDirectedCharacterOutput(IoStatementState &,
-    ListDirectedStatementState<Direction::Output> &, const char32_t *,
-    std::size_t chars);
+extern template RT_API_ATTRS bool ListDirectedCharacterOutput(
+    IoStatementState &, ListDirectedStatementState<Direction::Output> &,
+    const char *, std::size_t chars);
+extern template RT_API_ATTRS bool ListDirectedCharacterOutput(
+    IoStatementState &, ListDirectedStatementState<Direction::Output> &,
+    const char16_t *, std::size_t chars);
+extern template RT_API_ATTRS bool ListDirectedCharacterOutput(
+    IoStatementState &, ListDirectedStatementState<Direction::Output> &,
+    const char32_t *, std::size_t chars);
 
 template <typename CHAR>
-bool EditCharacterOutput(
+RT_API_ATTRS bool EditCharacterOutput(
     IoStatementState &, const DataEdit &, const CHAR *, std::size_t chars);
-extern template bool EditCharacterOutput(
+extern template RT_API_ATTRS bool EditCharacterOutput(
     IoStatementState &, const DataEdit &, const char *, std::size_t chars);
-extern template bool EditCharacterOutput(
+extern template RT_API_ATTRS bool EditCharacterOutput(
     IoStatementState &, const DataEdit &, const char16_t *, std::size_t chars);
-extern template bool EditCharacterOutput(
+extern template RT_API_ATTRS bool EditCharacterOutput(
     IoStatementState &, const DataEdit &, const char32_t *, std::size_t chars);
 
-extern template bool EditIntegerOutput<1>(
+extern template RT_API_ATTRS bool EditIntegerOutput<1>(
     IoStatementState &, const DataEdit &, std::int8_t);
-extern template bool EditIntegerOutput<2>(
+extern template RT_API_ATTRS bool EditIntegerOutput<2>(
     IoStatementState &, const DataEdit &, std::int16_t);
-extern template bool EditIntegerOutput<4>(
+extern template RT_API_ATTRS bool EditIntegerOutput<4>(
     IoStatementState &, const DataEdit &, std::int32_t);
-extern template bool EditIntegerOutput<8>(
+extern template RT_API_ATTRS bool EditIntegerOutput<8>(
     IoStatementState &, const DataEdit &, std::int64_t);
-extern template bool EditIntegerOutput<16>(
+extern template RT_API_ATTRS bool EditIntegerOutput<16>(
     IoStatementState &, const DataEdit &, common::int128_t);
 
 extern template class RealOutputEditing<2>;
diff --git a/flang/runtime/emit-encoded.h b/flang/runtime/emit-encoded.h
index 864848c3b19c..ac8c7d758a0d 100644
--- a/flang/runtime/emit-encoded.h
+++ b/flang/runtime/emit-encoded.h
@@ -19,7 +19,8 @@
 namespace Fortran::runtime::io {
 
 template <typename CONTEXT, typename CHAR>
-bool EmitEncoded(CONTEXT &to, const CHAR *data, std::size_t chars) {
+RT_API_ATTRS bool EmitEncoded(
+    CONTEXT &to, const CHAR *data, std::size_t chars) {
   ConnectionState &connection{to.GetConnectionState()};
   if (connection.access == Access::Stream &&
       connection.internalIoCharKind == 0) {
@@ -74,7 +75,7 @@ bool EmitEncoded(CONTEXT &to, const CHAR *data, std::size_t chars) {
 }
 
 template <typename CONTEXT>
-bool EmitAscii(CONTEXT &to, const char *data, std::size_t chars) {
+RT_API_ATTRS bool EmitAscii(CONTEXT &to, const char *data, std::size_t chars) {
   ConnectionState &connection{to.GetConnectionState()};
   if (connection.internalIoCharKind <= 1 &&
       connection.access != Access::Stream) {
@@ -85,7 +86,7 @@ bool EmitAscii(CONTEXT &to, const char *data, std::size_t chars) {
 }
 
 template <typename CONTEXT>
-bool EmitRepeated(CONTEXT &to, char ch, std::size_t n) {
+RT_API_ATTRS bool EmitRepeated(CONTEXT &to, char ch, std::size_t n) {
   if (n <= 0) {
     return true;
   }
diff --git a/flang/runtime/environment.h b/flang/runtime/environment.h
index 9bc115850961..6c56993fb1d6 100644
--- a/flang/runtime/environment.h
+++ b/flang/runtime/environment.h
@@ -18,6 +18,7 @@ namespace Fortran::runtime {
 
 class Terminator;
 
+RT_OFFLOAD_VAR_GROUP_BEGIN
 #if FLANG_BIG_ENDIAN
 constexpr bool isHostLittleEndian{false};
 #elif FLANG_LITTLE_ENDIAN
@@ -25,6 +26,7 @@ constexpr bool isHostLittleEndian{true};
 #else
 #error host endianness is not known
 #endif
+RT_OFFLOAD_VAR_GROUP_END
 
 // External unformatted I/O data conversions
 enum class Convert { Unknown, Native, LittleEndian, BigEndian, Swap };
diff --git a/flang/runtime/external-unit.cpp b/flang/runtime/external-unit.cpp
index 9d650ceca4a8..b48549d54587 100644
--- a/flang/runtime/external-unit.cpp
+++ b/flang/runtime/external-unit.cpp
@@ -10,14 +10,17 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "tools.h"
-
-#if !defined(RT_USE_PSEUDO_FILE_UNIT)
-
 #include "io-error.h"
 #include "lock.h"
+#include "tools.h"
 #include "unit-map.h"
 #include "unit.h"
+
+// NOTE: the header files above may define OpenMP declare target
+// variables, so they have to be included unconditionally
+// so that the offload entries are consistent between host and device.
+#if !defined(RT_USE_PSEUDO_FILE_UNIT)
+
 #include <cstdio>
 #include <limits>
 
diff --git a/flang/runtime/file.cpp b/flang/runtime/file.cpp
index 6ca5776f812a..67764f1f5626 100644
--- a/flang/runtime/file.cpp
+++ b/flang/runtime/file.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "file.h"
+#include "tools.h"
 #include "flang/Runtime/magic-numbers.h"
 #include "flang/Runtime/memory.h"
 #include <algorithm>
@@ -424,6 +425,7 @@ void OpenFile::CloseFd(IoErrorHandler &handler) {
   }
 }
 
+#if !defined(RT_DEVICE_COMPILATION)
 bool IsATerminal(int fd) { return ::isatty(fd); }
 
 #if defined(_WIN32) && !defined(F_OK)
@@ -455,5 +457,25 @@ std::int64_t SizeInBytes(const char *path) {
   // No Fortran compiler signals an error
   return -1;
 }
+#else // defined(RT_DEVICE_COMPILATION)
+bool IsATerminal(int fd) {
+  Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION);
+}
+bool IsExtant(const char *path) {
+  Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION);
+}
+bool MayRead(const char *path) {
+  Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION);
+}
+bool MayWrite(const char *path) {
+  Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION);
+}
+bool MayReadAndWrite(const char *path) {
+  Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION);
+}
+std::int64_t SizeInBytes(const char *path) {
+  Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION);
+}
+#endif // defined(RT_DEVICE_COMPILATION)
 
 } // namespace Fortran::runtime::io
diff --git a/flang/runtime/file.h b/flang/runtime/file.h
index 17deeb0e2f82..c06acbb9904c 100644
--- a/flang/runtime/file.h
+++ b/flang/runtime/file.h
@@ -106,11 +106,11 @@ private:
   OwningPtr<Pending> pending_;
 };
 
-bool IsATerminal(int fd);
-bool IsExtant(const char *path);
-bool MayRead(const char *path);
-bool MayWrite(const char *path);
-bool MayReadAndWrite(const char *path);
-std::int64_t SizeInBytes(const char *path);
+RT_API_ATTRS bool IsATerminal(int fd);
+RT_API_ATTRS bool IsExtant(const char *path);
+RT_API_ATTRS bool MayRead(const char *path);
+RT_API_ATTRS bool MayWrite(const char *path);
+RT_API_ATTRS bool MayReadAndWrite(const char *path);
+RT_API_ATTRS std::int64_t SizeInBytes(const char *path);
 } // namespace Fortran::runtime::io
 #endif // FORTRAN_RUNTIME_FILE_H_
diff --git a/flang/runtime/format-implementation.h b/flang/runtime/format-implementation.h
index b84e3208271b..45d4bd641f6f 100644
--- a/flang/runtime/format-implementation.h
+++ b/flang/runtime/format-implementation.h
@@ -25,7 +25,7 @@
 namespace Fortran::runtime::io {
 
 template <typename CONTEXT>
-FormatControl<CONTEXT>::FormatControl(const Terminator &terminator,
+RT_API_ATTRS FormatControl<CONTEXT>::FormatControl(const Terminator &terminator,
     const CharType *format, std::size_t formatLength,
     const Descriptor *formatDescriptor, int maxHeight)
     : maxHeight_{static_cast<std::uint8_t>(maxHeight)}, format_{format},
@@ -63,7 +63,7 @@ FormatControl<CONTEXT>::FormatControl(const Terminator &terminator,
 }
 
 template <typename CONTEXT>
-int FormatControl<CONTEXT>::GetIntField(
+RT_API_ATTRS int FormatControl<CONTEXT>::GetIntField(
     IoErrorHandler &handler, CharType firstCh, bool *hadError) {
   CharType ch{firstCh ? firstCh : PeekNext()};
   bool negate{ch == '-'};
@@ -114,7 +114,8 @@ int FormatControl<CONTEXT>::GetIntField(
 }
 
 template <typename CONTEXT>
-static void HandleControl(CONTEXT &context, char ch, char next, int n) {
+static RT_API_ATTRS void HandleControl(
+    CONTEXT &context, char ch, char next, int n) {
   MutableModes &modes{context.mutableModes()};
   switch (ch) {
   case 'B':
@@ -221,7 +222,8 @@ static void HandleControl(CONTEXT &context, char ch, char next, int n) {
 // Generally assumes that the format string has survived the common
 // format validator gauntlet.
 template <typename CONTEXT>
-int FormatControl<CONTEXT>::CueUpNextDataEdit(Context &context, bool stop) {
+RT_API_ATTRS int FormatControl<CONTEXT>::CueUpNextDataEdit(
+    Context &context, bool stop) {
   bool hitUnlimitedLoopEnd{false};
   // Do repetitions remain on an unparenthesized data edit?
   while (height_ > 1 && format_[stack_[height_ - 1].start] != '(') {
@@ -419,8 +421,8 @@ int FormatControl<CONTEXT>::CueUpNextDataEdit(Context &context, bool stop) {
 
 // Returns the next data edit descriptor
 template <typename CONTEXT>
-Fortran::common::optional<DataEdit> FormatControl<CONTEXT>::GetNextDataEdit(
-    Context &context, int maxRepeat) {
+RT_API_ATTRS Fortran::common::optional<DataEdit>
+FormatControl<CONTEXT>::GetNextDataEdit(Context &context, int maxRepeat) {
   int repeat{CueUpNextDataEdit(context)};
   auto start{offset_};
   DataEdit edit;
@@ -524,7 +526,7 @@ Fortran::common::optional<DataEdit> FormatControl<CONTEXT>::GetNextDataEdit(
 }
 
 template <typename CONTEXT>
-void FormatControl<CONTEXT>::Finish(Context &context) {
+RT_API_ATTRS void FormatControl<CONTEXT>::Finish(Context &context) {
   CueUpNextDataEdit(context, true /* stop at colon or end of FORMAT */);
   if (freeFormat_) {
     FreeMemory(const_cast<CharType *>(format_));
diff --git a/flang/runtime/format.cpp b/flang/runtime/format.cpp
index f219c29aaed1..433acce4b737 100644
--- a/flang/runtime/format.cpp
+++ b/flang/runtime/format.cpp
@@ -9,6 +9,7 @@
 #include "format-implementation.h"
 
 namespace Fortran::runtime::io {
+RT_OFFLOAD_API_GROUP_BEGIN
 template class FormatControl<
     InternalFormattedIoStatementState<Direction::Output>>;
 template class FormatControl<
@@ -19,4 +20,5 @@ template class FormatControl<
     ExternalFormattedIoStatementState<Direction::Input>>;
 template class FormatControl<ChildFormattedIoStatementState<Direction::Output>>;
 template class FormatControl<ChildFormattedIoStatementState<Direction::Input>>;
+RT_OFFLOAD_API_GROUP_END
 } // namespace Fortran::runtime::io
diff --git a/flang/runtime/format.h b/flang/runtime/format.h
index e7d945599640..f57cf9204487 100644
--- a/flang/runtime/format.h
+++ b/flang/runtime/format.h
@@ -12,6 +12,7 @@
 #define FORTRAN_RUNTIME_FORMAT_H_
 
 #include "environment.h"
+#include "freestanding-tools.h"
 #include "io-error.h"
 #include "flang/Common/Fortran.h"
 #include "flang/Common/optional.h"
@@ -49,20 +50,21 @@ struct DataEdit {
   char descriptor; // capitalized: one of A, I, B, O, Z, F, E(N/S/X), D, G
 
   // Special internal data edit descriptors for list-directed & NAMELIST I/O
+  RT_OFFLOAD_VAR_GROUP_BEGIN
   static constexpr char ListDirected{'g'}; // non-COMPLEX list-directed
   static constexpr char ListDirectedRealPart{'r'}; // emit "(r," or "(r;"
   static constexpr char ListDirectedImaginaryPart{'z'}; // emit "z)"
   static constexpr char ListDirectedNullValue{'n'}; // see 13.10.3.2
-  constexpr bool IsListDirected() const {
+  static constexpr char DefinedDerivedType{'d'}; // DT defined I/O
+  RT_OFFLOAD_VAR_GROUP_END
+  constexpr RT_API_ATTRS bool IsListDirected() const {
     return descriptor == ListDirected || descriptor == ListDirectedRealPart ||
         descriptor == ListDirectedImaginaryPart;
   }
-  constexpr bool IsNamelist() const {
+  constexpr RT_API_ATTRS bool IsNamelist() const {
     return IsListDirected() && modes.inNamelist;
   }
 
-  static constexpr char DefinedDerivedType{'d'}; // DT defined I/O
-
   char variation{'\0'}; // N, S, or X for EN, ES, EX; G/l for original G/list
   Fortran::common::optional<int> width; // the 'w' field; optional for A
   Fortran::common::optional<int> digits; // the 'm' or 'd' field
@@ -72,8 +74,10 @@ struct DataEdit {
 
   // "iotype" &/or "v_list" values for a DT'iotype'(v_list)
   // defined I/O data edit descriptor
+  RT_OFFLOAD_VAR_GROUP_BEGIN
   static constexpr std::size_t maxIoTypeChars{32};
   static constexpr std::size_t maxVListEntries{4};
+  RT_OFFLOAD_VAR_GROUP_END
   std::uint8_t ioTypeChars{0};
   std::uint8_t vListEntries{0};
   char ioType[maxIoTypeChars];
@@ -88,13 +92,13 @@ public:
   using Context = CONTEXT;
   using CharType = char; // formats are always default kind CHARACTER
 
-  FormatControl() {}
-  FormatControl(const Terminator &, const CharType *format,
+  RT_API_ATTRS FormatControl() {}
+  RT_API_ATTRS FormatControl(const Terminator &, const CharType *format,
       std::size_t formatLength, const Descriptor *formatDescriptor = nullptr,
       int maxHeight = maxMaxHeight);
 
   // For attempting to allocate in a user-supplied stack area
-  static std::size_t GetNeededSize(int maxHeight) {
+  static RT_API_ATTRS std::size_t GetNeededSize(int maxHeight) {
     return sizeof(FormatControl) -
         sizeof(Iteration) * (maxMaxHeight - maxHeight);
   }
@@ -102,14 +106,15 @@ public:
   // Extracts the next data edit descriptor, handling control edit descriptors
   // along the way.  If maxRepeat==0, this is a peek at the next data edit
   // descriptor.
-  Fortran::common::optional<DataEdit> GetNextDataEdit(
+  RT_API_ATTRS Fortran::common::optional<DataEdit> GetNextDataEdit(
       Context &, int maxRepeat = 1);
 
   // Emit any remaining character literals after the last data item (on output)
   // and perform remaining record positioning actions.
-  void Finish(Context &);
+  RT_API_ATTRS void Finish(Context &);
 
 private:
+  RT_OFFLOAD_VAR_GROUP_BEGIN
   static constexpr std::uint8_t maxMaxHeight{100};
 
   struct Iteration {
@@ -117,19 +122,20 @@ private:
     int start{0}; // offset in format_ of '(' or a repeated edit descriptor
     int remaining{0}; // while >0, decrement and iterate
   };
+  RT_OFFLOAD_VAR_GROUP_END
 
-  void SkipBlanks() {
+  RT_API_ATTRS void SkipBlanks() {
     while (offset_ < formatLength_ &&
         (format_[offset_] == ' ' || format_[offset_] == '\t' ||
             format_[offset_] == '\v')) {
       ++offset_;
     }
   }
-  CharType PeekNext() {
+  RT_API_ATTRS CharType PeekNext() {
     SkipBlanks();
     return offset_ < formatLength_ ? format_[offset_] : '\0';
   }
-  CharType GetNextChar(IoErrorHandler &handler) {
+  RT_API_ATTRS CharType GetNextChar(IoErrorHandler &handler) {
     SkipBlanks();
     if (offset_ >= formatLength_) {
       if (formatLength_ == 0) {
@@ -143,7 +149,7 @@ private:
     }
     return format_[offset_++];
   }
-  int GetIntField(
+  RT_API_ATTRS int GetIntField(
       IoErrorHandler &, CharType firstCh = '\0', bool *hadError = nullptr);
 
   // Advances through the FORMAT until the next data edit
@@ -151,13 +157,14 @@ private:
   // along the way.  Returns the repeat count that appeared
   // before the descriptor (defaulting to 1) and leaves offset_
   // pointing to the data edit.
-  int CueUpNextDataEdit(Context &, bool stop = false);
+  RT_API_ATTRS int CueUpNextDataEdit(Context &, bool stop = false);
 
-  static constexpr CharType Capitalize(CharType ch) {
+  static constexpr RT_API_ATTRS CharType Capitalize(CharType ch) {
     return ch >= 'a' && ch <= 'z' ? ch + 'A' - 'a' : ch;
   }
 
-  void ReportBadFormat(Context &context, const char *msg, int offset) const {
+  RT_API_ATTRS void ReportBadFormat(
+      Context &context, const char *msg, int offset) const {
     if constexpr (std::is_same_v<CharType, char>) {
       // Echo the bad format in the error message, but trim any leading or
       // trailing spaces.
diff --git a/flang/runtime/freestanding-tools.h b/flang/runtime/freestanding-tools.h
index 682b4c9b8929..451bf13b9fa6 100644
--- a/flang/runtime/freestanding-tools.h
+++ b/flang/runtime/freestanding-tools.h
@@ -9,7 +9,7 @@
 #ifndef FORTRAN_RUNTIME_FREESTANDING_TOOLS_H_
 #define FORTRAN_RUNTIME_FREESTANDING_TOOLS_H_
 
-#include "flang/Runtime/api-attrs.h"
+#include "flang/Common/api-attrs.h"
 #include "flang/Runtime/c-or-cpp.h"
 #include <algorithm>
 #include <cstring>
@@ -47,14 +47,19 @@
 #define STD_MEMCHR_UNSUPPORTED 1
 #endif
 
+#if !defined(STD_STRCPY_UNSUPPORTED) && \
+    (defined(__CUDACC__) || defined(__CUDA__)) && defined(__CUDA_ARCH__)
+#define STD_STRCPY_UNSUPPORTED 1
+#endif
+
 namespace Fortran::runtime {
 
 #if STD_FILL_N_UNSUPPORTED
 // Provides alternative implementation for std::fill_n(), if
 // it is not supported.
-template <typename A>
-static inline RT_API_ATTRS void fill_n(
-    A *start, std::size_t count, const A &value) {
+template <typename A, typename B>
+static inline RT_API_ATTRS std::enable_if_t<std::is_convertible_v<B, A>, void>
+fill_n(A *start, std::size_t count, const B &value) {
   for (std::size_t j{0}; j < count; ++j) {
     start[j] = value;
   }
@@ -157,5 +162,19 @@ static inline RT_API_ATTRS const void *memchr(
 using std::memchr;
 #endif // !STD_MEMCMP_UNSUPPORTED
 
+#if STD_STRCPY_UNSUPPORTED
+// Provides alternative implementation for std::strcpy(), if
+// it is not supported.
+static inline RT_API_ATTRS char *strcpy(char *dest, const char *src) {
+  char *result{dest};
+  do {
+    *dest++ = *src;
+  } while (*src++ != '\0');
+  return result;
+}
+#else // !STD_STRCPY_UNSUPPORTED
+using std::strcpy;
+#endif // !STD_STRCPY_UNSUPPORTED
+
 } // namespace Fortran::runtime
 #endif // FORTRAN_RUNTIME_FREESTANDING_TOOLS_H_
diff --git a/flang/runtime/internal-unit.cpp b/flang/runtime/internal-unit.cpp
index 66140e005887..35766306ccef 100644
--- a/flang/runtime/internal-unit.cpp
+++ b/flang/runtime/internal-unit.cpp
@@ -7,15 +7,17 @@
 //===----------------------------------------------------------------------===//
 
 #include "internal-unit.h"
+#include "freestanding-tools.h"
 #include "io-error.h"
 #include "flang/Runtime/descriptor.h"
 #include <algorithm>
 #include <type_traits>
 
 namespace Fortran::runtime::io {
+RT_OFFLOAD_API_GROUP_BEGIN
 
 template <Direction DIR>
-InternalDescriptorUnit<DIR>::InternalDescriptorUnit(
+RT_API_ATTRS InternalDescriptorUnit<DIR>::InternalDescriptorUnit(
     Scalar scalar, std::size_t length, int kind) {
   internalIoCharKind = kind;
   recordLength = length;
@@ -26,7 +28,7 @@ InternalDescriptorUnit<DIR>::InternalDescriptorUnit(
 }
 
 template <Direction DIR>
-InternalDescriptorUnit<DIR>::InternalDescriptorUnit(
+RT_API_ATTRS InternalDescriptorUnit<DIR>::InternalDescriptorUnit(
     const Descriptor &that, const Terminator &terminator) {
   auto thatType{that.type().GetCategoryAndKind()};
   RUNTIME_CHECK(terminator, thatType.has_value());
@@ -42,7 +44,7 @@ InternalDescriptorUnit<DIR>::InternalDescriptorUnit(
 }
 
 template <Direction DIR>
-bool InternalDescriptorUnit<DIR>::Emit(
+RT_API_ATTRS bool InternalDescriptorUnit<DIR>::Emit(
     const char *data, std::size_t bytes, IoErrorHandler &handler) {
   if constexpr (DIR == Direction::Input) {
     handler.Crash("InternalDescriptorUnit<Direction::Input>::Emit() called");
@@ -76,7 +78,7 @@ bool InternalDescriptorUnit<DIR>::Emit(
 }
 
 template <Direction DIR>
-std::size_t InternalDescriptorUnit<DIR>::GetNextInputBytes(
+RT_API_ATTRS std::size_t InternalDescriptorUnit<DIR>::GetNextInputBytes(
     const char *&p, IoErrorHandler &handler) {
   if constexpr (DIR == Direction::Output) {
     handler.Crash("InternalDescriptorUnit<Direction::Output>::"
@@ -97,7 +99,8 @@ std::size_t InternalDescriptorUnit<DIR>::GetNextInputBytes(
 }
 
 template <Direction DIR>
-bool InternalDescriptorUnit<DIR>::AdvanceRecord(IoErrorHandler &handler) {
+RT_API_ATTRS bool InternalDescriptorUnit<DIR>::AdvanceRecord(
+    IoErrorHandler &handler) {
   if (currentRecordNumber >= endfileRecordNumber.value_or(0)) {
     if constexpr (DIR == Direction::Input) {
       handler.SignalEnd();
@@ -115,24 +118,25 @@ bool InternalDescriptorUnit<DIR>::AdvanceRecord(IoErrorHandler &handler) {
 }
 
 template <Direction DIR>
-void InternalDescriptorUnit<DIR>::BlankFill(char *at, std::size_t bytes) {
+RT_API_ATTRS void InternalDescriptorUnit<DIR>::BlankFill(
+    char *at, std::size_t bytes) {
   switch (internalIoCharKind) {
   case 2:
-    std::fill_n(reinterpret_cast<char16_t *>(at), bytes / 2,
+    Fortran::runtime::fill_n(reinterpret_cast<char16_t *>(at), bytes / 2,
         static_cast<char16_t>(' '));
     break;
   case 4:
-    std::fill_n(reinterpret_cast<char32_t *>(at), bytes / 4,
+    Fortran::runtime::fill_n(reinterpret_cast<char32_t *>(at), bytes / 4,
         static_cast<char32_t>(' '));
     break;
   default:
-    std::fill_n(at, bytes, ' ');
+    Fortran::runtime::fill_n(at, bytes, ' ');
     break;
   }
 }
 
 template <Direction DIR>
-void InternalDescriptorUnit<DIR>::BlankFillOutputRecord() {
+RT_API_ATTRS void InternalDescriptorUnit<DIR>::BlankFillOutputRecord() {
   if constexpr (DIR == Direction::Output) {
     if (furthestPositionInRecord <
         recordLength.value_or(furthestPositionInRecord)) {
@@ -143,18 +147,21 @@ void InternalDescriptorUnit<DIR>::BlankFillOutputRecord() {
 }
 
 template <Direction DIR>
-void InternalDescriptorUnit<DIR>::BackspaceRecord(IoErrorHandler &handler) {
+RT_API_ATTRS void InternalDescriptorUnit<DIR>::BackspaceRecord(
+    IoErrorHandler &handler) {
   RUNTIME_CHECK(handler, currentRecordNumber > 1);
   --currentRecordNumber;
   BeginRecord();
 }
 
 template <Direction DIR>
-std::int64_t InternalDescriptorUnit<DIR>::InquirePos() {
+RT_API_ATTRS std::int64_t InternalDescriptorUnit<DIR>::InquirePos() {
   return (currentRecordNumber - 1) * recordLength.value_or(0) +
       positionInRecord + 1;
 }
 
 template class InternalDescriptorUnit<Direction::Output>;
 template class InternalDescriptorUnit<Direction::Input>;
+
+RT_OFFLOAD_API_GROUP_END
 } // namespace Fortran::runtime::io
diff --git a/flang/runtime/internal-unit.h b/flang/runtime/internal-unit.h
index b536ffb831d5..bcd38b62468a 100644
--- a/flang/runtime/internal-unit.h
+++ b/flang/runtime/internal-unit.h
@@ -26,26 +26,28 @@ template <Direction DIR> class InternalDescriptorUnit : public ConnectionState {
 public:
   using Scalar =
       std::conditional_t<DIR == Direction::Input, const char *, char *>;
-  InternalDescriptorUnit(Scalar, std::size_t chars, int kind);
-  InternalDescriptorUnit(const Descriptor &, const Terminator &);
+  RT_API_ATTRS InternalDescriptorUnit(Scalar, std::size_t chars, int kind);
+  RT_API_ATTRS InternalDescriptorUnit(const Descriptor &, const Terminator &);
 
-  bool Emit(const char *, std::size_t, IoErrorHandler &);
-  std::size_t GetNextInputBytes(const char *&, IoErrorHandler &);
-  bool AdvanceRecord(IoErrorHandler &);
-  void BackspaceRecord(IoErrorHandler &);
-  std::int64_t InquirePos();
+  RT_API_ATTRS bool Emit(const char *, std::size_t, IoErrorHandler &);
+  RT_API_ATTRS std::size_t GetNextInputBytes(const char *&, IoErrorHandler &);
+  RT_API_ATTRS bool AdvanceRecord(IoErrorHandler &);
+  RT_API_ATTRS void BackspaceRecord(IoErrorHandler &);
+  RT_API_ATTRS std::int64_t InquirePos();
 
 private:
-  Descriptor &descriptor() { return staticDescriptor_.descriptor(); }
-  const Descriptor &descriptor() const {
+  RT_API_ATTRS Descriptor &descriptor() {
     return staticDescriptor_.descriptor();
   }
-  Scalar CurrentRecord() const {
+  RT_API_ATTRS const Descriptor &descriptor() const {
+    return staticDescriptor_.descriptor();
+  }
+  RT_API_ATTRS Scalar CurrentRecord() const {
     return descriptor().template ZeroBasedIndexedElement<char>(
         currentRecordNumber - 1);
   }
-  void BlankFill(char *, std::size_t);
-  void BlankFillOutputRecord();
+  RT_API_ATTRS void BlankFill(char *, std::size_t);
+  RT_API_ATTRS void BlankFillOutputRecord();
 
   StaticDescriptor<maxRank, true /*addendum*/> staticDescriptor_;
 };
diff --git a/flang/runtime/io-api.cpp b/flang/runtime/io-api.cpp
index 094db5572f15..0f259f4715bf 100644
--- a/flang/runtime/io-api.cpp
+++ b/flang/runtime/io-api.cpp
@@ -99,7 +99,7 @@ Cookie IONAME(BeginInternalArrayFormattedInput)(const Descriptor &descriptor,
 }
 
 template <Direction DIR>
-Cookie BeginInternalListIO(
+RT_API_ATTRS Cookie BeginInternalListIO(
     std::conditional_t<DIR == Direction::Input, const char, char> *internal,
     std::size_t internalLength, void ** /*scratchArea*/,
     std::size_t /*scratchBytes*/, const char *sourceFile, int sourceLine) {
@@ -156,8 +156,8 @@ Cookie IONAME(BeginInternalFormattedInput)(const char *internal,
       sourceFile, sourceLine);
 }
 
-static Cookie NoopUnit(const Terminator &terminator, int unitNumber,
-    enum Iostat iostat = IostatOk) {
+static RT_API_ATTRS Cookie NoopUnit(const Terminator &terminator,
+    int unitNumber, enum Iostat iostat = IostatOk) {
   Cookie cookie{&New<NoopStatementState>{terminator}(
       terminator.sourceFileName(), terminator.sourceLine(), unitNumber)
                      .release()
@@ -168,9 +168,9 @@ static Cookie NoopUnit(const Terminator &terminator, int unitNumber,
   return cookie;
 }
 
-static ExternalFileUnit *GetOrCreateUnit(int unitNumber, Direction direction,
-    Fortran::common::optional<bool> isUnformatted, const Terminator &terminator,
-    Cookie &errorCookie) {
+static RT_API_ATTRS ExternalFileUnit *GetOrCreateUnit(int unitNumber,
+    Direction direction, Fortran::common::optional<bool> isUnformatted,
+    const Terminator &terminator, Cookie &errorCookie) {
   if (ExternalFileUnit *
       unit{ExternalFileUnit::LookUpOrCreateAnonymous(
           unitNumber, direction, isUnformatted, terminator)}) {
@@ -183,7 +183,7 @@ static ExternalFileUnit *GetOrCreateUnit(int unitNumber, Direction direction,
 }
 
 template <Direction DIR, template <Direction> class STATE, typename... A>
-Cookie BeginExternalListIO(
+RT_API_ATTRS Cookie BeginExternalListIO(
     int unitNumber, const char *sourceFile, int sourceLine, A &&...xs) {
   Terminator terminator{sourceFile, sourceLine};
   Cookie errorCookie{nullptr};
@@ -227,11 +227,13 @@ Cookie BeginExternalListIO(
   }
 }
 
-Cookie IONAME(BeginExternalListOutput)(
+RT_EXT_API_GROUP_BEGIN
+Cookie IODEF(BeginExternalListOutput)(
     ExternalUnit unitNumber, const char *sourceFile, int sourceLine) {
   return BeginExternalListIO<Direction::Output, ExternalListIoStatementState>(
       unitNumber, sourceFile, sourceLine);
 }
+RT_EXT_API_GROUP_END
 
 Cookie IONAME(BeginExternalListInput)(
     ExternalUnit unitNumber, const char *sourceFile, int sourceLine) {
@@ -1163,7 +1165,8 @@ bool IONAME(OutputInteger16)(Cookie cookie, std::int16_t n) {
   return descr::DescriptorIO<Direction::Output>(*cookie, descriptor);
 }
 
-bool IONAME(OutputInteger32)(Cookie cookie, std::int32_t n) {
+RT_EXT_API_GROUP_BEGIN
+bool IODEF(OutputInteger32)(Cookie cookie, std::int32_t n) {
   if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputInteger32")) {
     return false;
   }
@@ -1173,6 +1176,7 @@ bool IONAME(OutputInteger32)(Cookie cookie, std::int32_t n) {
       TypeCategory::Integer, 4, reinterpret_cast<void *>(&n), 0);
   return descr::DescriptorIO<Direction::Output>(*cookie, descriptor);
 }
+RT_EXT_API_GROUP_END
 
 bool IONAME(OutputInteger64)(Cookie cookie, std::int64_t n) {
   if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputInteger64")) {
@@ -1448,10 +1452,12 @@ bool IONAME(InquireInteger64)(
   return false;
 }
 
-enum Iostat IONAME(EndIoStatement)(Cookie cookie) {
+RT_EXT_API_GROUP_BEGIN
+enum Iostat IODEF(EndIoStatement)(Cookie cookie) {
   IoStatementState &io{*cookie};
   return static_cast<enum Iostat>(io.EndIoStatement());
 }
+RT_EXT_API_GROUP_END
 
 template <typename INT>
 static enum Iostat CheckUnitNumberInRangeImpl(INT unit, bool handleError,
diff --git a/flang/runtime/io-error.cpp b/flang/runtime/io-error.cpp
index c8f6675c60a6..02f237f05bea 100644
--- a/flang/runtime/io-error.cpp
+++ b/flang/runtime/io-error.cpp
@@ -16,6 +16,7 @@
 #include <cstring>
 
 namespace Fortran::runtime::io {
+RT_OFFLOAD_API_GROUP_BEGIN
 
 void IoErrorHandler::SignalError(int iostatOrErrno, const char *msg, ...) {
   // Note that IOMSG= alone without IOSTAT=/END=/EOR=/ERR= does not suffice
@@ -44,12 +45,20 @@ void IoErrorHandler::SignalError(int iostatOrErrno, const char *msg, ...) {
       if (ioStat_ <= 0) {
         ioStat_ = iostatOrErrno; // priority over END=/EOR=
         if (msg && (flags_ & hasIoMsg)) {
+#if !defined(RT_DEVICE_COMPILATION)
           char buffer[256];
           va_list ap;
           va_start(ap, msg);
           std::vsnprintf(buffer, sizeof buffer, msg, ap);
-          ioMsg_ = SaveDefaultCharacter(buffer, std::strlen(buffer) + 1, *this);
           va_end(ap);
+#else
+          const char *buffer = "not implemented yet: IOSTAT with varargs";
+#endif
+          ioMsg_ = SaveDefaultCharacter(
+              buffer, Fortran::runtime::strlen(buffer) + 1, *this);
+#if !defined(RT_DEVICE_COMPILATION)
+          va_end(ap);
+#endif
         }
       }
       return;
@@ -58,15 +67,23 @@ void IoErrorHandler::SignalError(int iostatOrErrno, const char *msg, ...) {
   }
   // I/O error not caught!
   if (msg) {
+#if !defined(RT_DEVICE_COMPILATION)
     va_list ap;
     va_start(ap, msg);
     CrashArgs(msg, ap);
     va_end(ap);
+#else
+    Crash("not implemented yet: IOSTAT with varargs");
+#endif
   } else if (const char *errstr{IostatErrorString(iostatOrErrno)}) {
     Crash(errstr);
   } else {
+#if !defined(RT_DEVICE_COMPILATION)
     Crash("I/O error (errno=%d): %s", iostatOrErrno,
         std::strerror(iostatOrErrno));
+#else
+    Crash("I/O error (errno=%d)", iostatOrErrno);
+#endif
   }
 }
 
@@ -85,8 +102,6 @@ void IoErrorHandler::Forward(
   }
 }
 
-void IoErrorHandler::SignalErrno() { SignalError(errno); }
-
 void IoErrorHandler::SignalEnd() { SignalError(IostatEnd); }
 
 void IoErrorHandler::SignalEor() { SignalError(IostatEor); }
@@ -97,6 +112,10 @@ void IoErrorHandler::SignalPendingError() {
   SignalError(error);
 }
 
+RT_OFFLOAD_API_GROUP_END
+
+void IoErrorHandler::SignalErrno() { SignalError(errno); }
+
 bool IoErrorHandler::GetIoMsg(char *buffer, std::size_t bufferLength) {
   const char *msg{ioMsg_.get()};
   if (!msg) {
@@ -132,7 +151,7 @@ bool IoErrorHandler::GetIoMsg(char *buffer, std::size_t bufferLength) {
     ToFortranDefaultCharacter(buffer, bufferLength, msg);
     return true;
   } else if (ok) {
-    std::size_t copied{std::strlen(buffer)};
+    std::size_t copied{Fortran::runtime::strlen(buffer)};
     if (copied < bufferLength) {
       std::memset(buffer + copied, ' ', bufferLength - copied);
     }
diff --git a/flang/runtime/io-error.h b/flang/runtime/io-error.h
index 565e7153351e..0fe11c9185c0 100644
--- a/flang/runtime/io-error.h
+++ b/flang/runtime/io-error.h
@@ -26,14 +26,15 @@ namespace Fortran::runtime::io {
 class IoErrorHandler : public Terminator {
 public:
   using Terminator::Terminator;
-  explicit IoErrorHandler(const Terminator &that) : Terminator{that} {}
-  void HasIoStat() { flags_ |= hasIoStat; }
-  void HasErrLabel() { flags_ |= hasErr; }
-  void HasEndLabel() { flags_ |= hasEnd; }
-  void HasEorLabel() { flags_ |= hasEor; }
-  void HasIoMsg() { flags_ |= hasIoMsg; }
+  explicit RT_API_ATTRS IoErrorHandler(const Terminator &that)
+      : Terminator{that} {}
+  RT_API_ATTRS void HasIoStat() { flags_ |= hasIoStat; }
+  RT_API_ATTRS void HasErrLabel() { flags_ |= hasErr; }
+  RT_API_ATTRS void HasEndLabel() { flags_ |= hasEnd; }
+  RT_API_ATTRS void HasEorLabel() { flags_ |= hasEor; }
+  RT_API_ATTRS void HasIoMsg() { flags_ |= hasIoMsg; }
 
-  bool InError() const {
+  RT_API_ATTRS bool InError() const {
     return ioStat_ != IostatOk || pendingError_ != IostatOk;
   }
 
@@ -41,22 +42,25 @@ public:
   // Begin...() API routines before it is known whether they
   // have error handling control list items.  Such statements
   // have an ErroneousIoStatementState with a pending error.
-  void SetPendingError(int iostat) { pendingError_ = iostat; }
+  RT_API_ATTRS void SetPendingError(int iostat) { pendingError_ = iostat; }
 
-  void SignalError(int iostatOrErrno, const char *msg, ...);
-  void SignalError(int iostatOrErrno);
-  template <typename... X> void SignalError(const char *msg, X &&...xs) {
+  RT_API_ATTRS void SignalError(int iostatOrErrno, const char *msg, ...);
+  RT_API_ATTRS void SignalError(int iostatOrErrno);
+  template <typename... X>
+  RT_API_ATTRS void SignalError(const char *msg, X &&...xs) {
     SignalError(IostatGenericError, msg, std::forward<X>(xs)...);
   }
 
-  void Forward(int iostatOrErrno, const char *, std::size_t);
+  RT_API_ATTRS void Forward(int iostatOrErrno, const char *, std::size_t);
 
   void SignalErrno(); // SignalError(errno)
-  void SignalEnd(); // input only; EOF on internal write is an error
-  void SignalEor(); // non-advancing input only; EOR on write is an error
-  void SignalPendingError();
+  RT_API_ATTRS void
+  SignalEnd(); // input only; EOF on internal write is an error
+  RT_API_ATTRS void
+  SignalEor(); // non-advancing input only; EOR on write is an error
+  RT_API_ATTRS void SignalPendingError();
 
-  int GetIoStat() const { return ioStat_; }
+  RT_API_ATTRS int GetIoStat() const { return ioStat_; }
   bool GetIoMsg(char *, std::size_t);
 
 private:
diff --git a/flang/runtime/io-stmt.cpp b/flang/runtime/io-stmt.cpp
index e3f1214324d8..022e4c806bf6 100644
--- a/flang/runtime/io-stmt.cpp
+++ b/flang/runtime/io-stmt.cpp
@@ -21,6 +21,7 @@
 #include <type_traits>
 
 namespace Fortran::runtime::io {
+RT_OFFLOAD_API_GROUP_BEGIN
 
 bool IoStatementBase::Emit(const char *, std::size_t, std::size_t) {
   return false;
@@ -44,10 +45,6 @@ Fortran::common::optional<DataEdit> IoStatementBase::GetNextDataEdit(
   return Fortran::common::nullopt;
 }
 
-ExternalFileUnit *IoStatementBase::GetExternalFileUnit() const {
-  return nullptr;
-}
-
 bool IoStatementBase::BeginReadingRecord() { return true; }
 
 void IoStatementBase::FinishReadingRecord() {}
@@ -56,6 +53,12 @@ void IoStatementBase::HandleAbsolutePosition(std::int64_t) {}
 
 void IoStatementBase::HandleRelativePosition(std::int64_t) {}
 
+std::int64_t IoStatementBase::InquirePos() { return 0; }
+
+ExternalFileUnit *IoStatementBase::GetExternalFileUnit() const {
+  return nullptr;
+}
+
 bool IoStatementBase::Inquire(InquiryKeywordHash, char *, std::size_t) {
   return false;
 }
@@ -70,8 +73,6 @@ bool IoStatementBase::Inquire(InquiryKeywordHash, std::int64_t &) {
   return false;
 }
 
-std::int64_t IoStatementBase::InquirePos() { return 0; }
-
 void IoStatementBase::BadInquiryKeywordHashCrash(InquiryKeywordHash inquiry) {
   char buffer[16];
   const char *decode{InquiryKeywordHashDecode(buffer, sizeof buffer, inquiry)};
@@ -142,21 +143,23 @@ std::int64_t InternalIoStatementState<DIR>::InquirePos() {
 }
 
 template <Direction DIR, typename CHAR>
+RT_API_ATTRS
 InternalFormattedIoStatementState<DIR, CHAR>::InternalFormattedIoStatementState(
     Buffer buffer, std::size_t length, const CharType *format,
     std::size_t formatLength, const Descriptor *formatDescriptor,
     const char *sourceFile, int sourceLine)
     : InternalIoStatementState<DIR>{buffer, length, sourceFile, sourceLine},
-      ioStatementState_{*this}, format_{*this, format, formatLength,
-                                    formatDescriptor} {}
+      ioStatementState_{*this},
+      format_{*this, format, formatLength, formatDescriptor} {}
 
 template <Direction DIR, typename CHAR>
+RT_API_ATTRS
 InternalFormattedIoStatementState<DIR, CHAR>::InternalFormattedIoStatementState(
     const Descriptor &d, const CharType *format, std::size_t formatLength,
     const Descriptor *formatDescriptor, const char *sourceFile, int sourceLine)
     : InternalIoStatementState<DIR>{d, sourceFile, sourceLine},
-      ioStatementState_{*this}, format_{*this, format, formatLength,
-                                    formatDescriptor} {}
+      ioStatementState_{*this},
+      format_{*this, format, formatLength, formatDescriptor} {}
 
 template <Direction DIR, typename CHAR>
 void InternalFormattedIoStatementState<DIR, CHAR>::CompleteOperation() {
@@ -1004,7 +1007,9 @@ void ExternalMiscIoStatementState::CompleteOperation() {
   switch (which_) {
   case Flush:
     ext.FlushOutput(*this);
+#if !defined(RT_DEVICE_COMPILATION)
     std::fflush(nullptr); // flushes C stdio output streams (12.9(2))
+#endif
     break;
   case Backspace:
     ext.BackspaceRecord(*this);
@@ -1508,4 +1513,5 @@ int ErroneousIoStatementState::EndIoStatement() {
   return IoStatementBase::EndIoStatement();
 }
 
+RT_OFFLOAD_API_GROUP_END
 } // namespace Fortran::runtime::io
diff --git a/flang/runtime/io-stmt.h b/flang/runtime/io-stmt.h
index e00d54980aae..e0dafa9c763c 100644
--- a/flang/runtime/io-stmt.h
+++ b/flang/runtime/io-stmt.h
@@ -61,8 +61,8 @@ using IoDirectionState = std::conditional_t<D == Direction::Input,
 template <Direction D> class FormattedIoStatementState {};
 template <> class FormattedIoStatementState<Direction::Input> {
 public:
-  std::size_t GetEditDescriptorChars() const;
-  void GotChar(int);
+  RT_API_ATTRS std::size_t GetEditDescriptorChars() const;
+  RT_API_ATTRS void GotChar(int);
 
 private:
   // Account of characters read for edit descriptors (i.e., formatted I/O
@@ -73,7 +73,7 @@ private:
 // The Cookie type in the I/O API is a pointer (for C) to this class.
 class IoStatementState {
 public:
-  template <typename A> explicit IoStatementState(A &x) : u_{x} {}
+  template <typename A> explicit RT_API_ATTRS IoStatementState(A &x) : u_{x} {}
 
   // These member functions each project themselves into the active alternative.
   // They're used by per-data-item routines in the I/O API (e.g., OutputReal64)
@@ -85,34 +85,39 @@ public:
   // It is called by EndIoStatement(), but it can be invoked earlier to
   // catch errors for (e.g.) GetIoMsg() and GetNewUnit().  If called
   // more than once, it is a no-op.
-  void CompleteOperation();
+  RT_API_ATTRS void CompleteOperation();
   // Completes an I/O statement and reclaims storage.
-  int EndIoStatement();
-
-  bool Emit(const char *, std::size_t bytes, std::size_t elementBytes = 0);
-  bool Receive(char *, std::size_t, std::size_t elementBytes = 0);
-  std::size_t GetNextInputBytes(const char *&);
-  bool AdvanceRecord(int = 1);
-  void BackspaceRecord();
-  void HandleRelativePosition(std::int64_t byteOffset);
-  void HandleAbsolutePosition(std::int64_t byteOffset); // for r* in list I/O
-  Fortran::common::optional<DataEdit> GetNextDataEdit(int maxRepeat = 1);
-  ExternalFileUnit *GetExternalFileUnit() const; // null if internal unit
-  bool BeginReadingRecord();
-  void FinishReadingRecord();
-  bool Inquire(InquiryKeywordHash, char *, std::size_t);
-  bool Inquire(InquiryKeywordHash, bool &);
-  bool Inquire(InquiryKeywordHash, std::int64_t, bool &); // PENDING=
-  bool Inquire(InquiryKeywordHash, std::int64_t &);
-  std::int64_t InquirePos();
-  void GotChar(signed int = 1); // for READ(SIZE=); can be <0
-
-  MutableModes &mutableModes();
-  ConnectionState &GetConnectionState();
-  IoErrorHandler &GetIoErrorHandler() const;
+  RT_API_ATTRS int EndIoStatement();
+
+  RT_API_ATTRS bool Emit(
+      const char *, std::size_t bytes, std::size_t elementBytes = 0);
+  RT_API_ATTRS bool Receive(char *, std::size_t, std::size_t elementBytes = 0);
+  RT_API_ATTRS std::size_t GetNextInputBytes(const char *&);
+  RT_API_ATTRS bool AdvanceRecord(int = 1);
+  RT_API_ATTRS void BackspaceRecord();
+  RT_API_ATTRS void HandleRelativePosition(std::int64_t byteOffset);
+  RT_API_ATTRS void HandleAbsolutePosition(
+      std::int64_t byteOffset); // for r* in list I/O
+  RT_API_ATTRS Fortran::common::optional<DataEdit> GetNextDataEdit(
+      int maxRepeat = 1);
+  RT_API_ATTRS ExternalFileUnit *
+  GetExternalFileUnit() const; // null if internal unit
+  RT_API_ATTRS bool BeginReadingRecord();
+  RT_API_ATTRS void FinishReadingRecord();
+  RT_API_ATTRS bool Inquire(InquiryKeywordHash, char *, std::size_t);
+  RT_API_ATTRS bool Inquire(InquiryKeywordHash, bool &);
+  RT_API_ATTRS bool Inquire(
+      InquiryKeywordHash, std::int64_t, bool &); // PENDING=
+  RT_API_ATTRS bool Inquire(InquiryKeywordHash, std::int64_t &);
+  RT_API_ATTRS std::int64_t InquirePos();
+  RT_API_ATTRS void GotChar(signed int = 1); // for READ(SIZE=); can be <0
+
+  RT_API_ATTRS MutableModes &mutableModes();
+  RT_API_ATTRS ConnectionState &GetConnectionState();
+  RT_API_ATTRS IoErrorHandler &GetIoErrorHandler() const;
 
   // N.B.: this also works with base classes
-  template <typename A> A *get_if() const {
+  template <typename A> RT_API_ATTRS A *get_if() const {
     return common::visit(
         [](auto &x) -> A * {
           if constexpr (std::is_convertible_v<decltype(x.get()), A &>) {
@@ -124,7 +129,8 @@ public:
   }
 
   // Vacant after the end of the current record
-  Fortran::common::optional<char32_t> GetCurrentChar(std::size_t &byteCount);
+  RT_API_ATTRS Fortran::common::optional<char32_t> GetCurrentChar(
+      std::size_t &byteCount);
 
   // The "remaining" arguments to CueUpInput(), SkipSpaces(), & NextInField()
   // are always in units of bytes, not characters; the distinction matters
@@ -132,7 +138,7 @@ public:
 
   // For fixed-width fields, return the number of remaining bytes.
   // Skip over leading blanks.
-  Fortran::common::optional<int> CueUpInput(const DataEdit &edit) {
+  RT_API_ATTRS Fortran::common::optional<int> CueUpInput(const DataEdit &edit) {
     Fortran::common::optional<int> remaining;
     if (edit.IsListDirected()) {
       std::size_t byteCount{0};
@@ -150,7 +156,7 @@ public:
     return remaining;
   }
 
-  Fortran::common::optional<char32_t> SkipSpaces(
+  RT_API_ATTRS Fortran::common::optional<char32_t> SkipSpaces(
       Fortran::common::optional<int> &remaining) {
     while (!remaining || *remaining > 0) {
       std::size_t byteCount{0};
@@ -175,15 +181,16 @@ public:
 
   // Acquires the next input character, respecting any applicable field width
   // or separator character.
-  Fortran::common::optional<char32_t> NextInField(
+  RT_API_ATTRS Fortran::common::optional<char32_t> NextInField(
       Fortran::common::optional<int> &remaining, const DataEdit &);
 
   // Detect and signal any end-of-record condition after input.
   // Returns true if at EOR and remaining input should be padded with blanks.
-  bool CheckForEndOfRecord(std::size_t afterReading);
+  RT_API_ATTRS bool CheckForEndOfRecord(std::size_t afterReading);
 
   // Skips spaces, advances records, and ignores NAMELIST comments
-  Fortran::common::optional<char32_t> GetNextNonBlank(std::size_t &byteCount) {
+  RT_API_ATTRS Fortran::common::optional<char32_t> GetNextNonBlank(
+      std::size_t &byteCount) {
     auto ch{GetCurrentChar(byteCount)};
     bool inNamelist{mutableModes().inNamelist};
     while (!ch || *ch == ' ' || *ch == '\t' || (inNamelist && *ch == '!')) {
@@ -197,7 +204,8 @@ public:
     return ch;
   }
 
-  template <Direction D> bool CheckFormattedStmtType(const char *name) {
+  template <Direction D>
+  RT_API_ATTRS bool CheckFormattedStmtType(const char *name) {
     if (get_if<FormattedIoStatementState<D>>()) {
       return true;
     } else {
@@ -260,31 +268,33 @@ class IoStatementBase : public IoErrorHandler {
 public:
   using IoErrorHandler::IoErrorHandler;
 
-  bool completedOperation() const { return completedOperation_; }
+  RT_API_ATTRS bool completedOperation() const { return completedOperation_; }
 
-  void CompleteOperation() { completedOperation_ = true; }
-  int EndIoStatement() { return GetIoStat(); }
+  RT_API_ATTRS void CompleteOperation() { completedOperation_ = true; }
+  RT_API_ATTRS int EndIoStatement() { return GetIoStat(); }
 
   // These are default no-op backstops that can be overridden by descendants.
-  bool Emit(const char *, std::size_t bytes, std::size_t elementBytes = 0);
-  bool Receive(char *, std::size_t bytes, std::size_t elementBytes = 0);
-  std::size_t GetNextInputBytes(const char *&);
-  bool AdvanceRecord(int);
-  void BackspaceRecord();
-  void HandleRelativePosition(std::int64_t);
-  void HandleAbsolutePosition(std::int64_t);
-  Fortran::common::optional<DataEdit> GetNextDataEdit(
+  RT_API_ATTRS bool Emit(
+      const char *, std::size_t bytes, std::size_t elementBytes = 0);
+  RT_API_ATTRS bool Receive(
+      char *, std::size_t bytes, std::size_t elementBytes = 0);
+  RT_API_ATTRS std::size_t GetNextInputBytes(const char *&);
+  RT_API_ATTRS bool AdvanceRecord(int);
+  RT_API_ATTRS void BackspaceRecord();
+  RT_API_ATTRS void HandleRelativePosition(std::int64_t);
+  RT_API_ATTRS void HandleAbsolutePosition(std::int64_t);
+  RT_API_ATTRS Fortran::common::optional<DataEdit> GetNextDataEdit(
       IoStatementState &, int maxRepeat = 1);
-  ExternalFileUnit *GetExternalFileUnit() const;
-  bool BeginReadingRecord();
-  void FinishReadingRecord();
-  bool Inquire(InquiryKeywordHash, char *, std::size_t);
-  bool Inquire(InquiryKeywordHash, bool &);
-  bool Inquire(InquiryKeywordHash, std::int64_t, bool &);
-  bool Inquire(InquiryKeywordHash, std::int64_t &);
-  std::int64_t InquirePos();
+  RT_API_ATTRS ExternalFileUnit *GetExternalFileUnit() const;
+  RT_API_ATTRS bool BeginReadingRecord();
+  RT_API_ATTRS void FinishReadingRecord();
+  RT_API_ATTRS bool Inquire(InquiryKeywordHash, char *, std::size_t);
+  RT_API_ATTRS bool Inquire(InquiryKeywordHash, bool &);
+  RT_API_ATTRS bool Inquire(InquiryKeywordHash, std::int64_t, bool &);
+  RT_API_ATTRS bool Inquire(InquiryKeywordHash, std::int64_t &);
+  RT_API_ATTRS std::int64_t InquirePos();
 
-  void BadInquiryKeywordHashCrash(InquiryKeywordHash);
+  RT_API_ATTRS void BadInquiryKeywordHashCrash(InquiryKeywordHash);
 
 protected:
   bool completedOperation_{false};
@@ -296,14 +306,14 @@ template <>
 class ListDirectedStatementState<Direction::Output>
     : public FormattedIoStatementState<Direction::Output> {
 public:
-  bool EmitLeadingSpaceOrAdvance(
+  RT_API_ATTRS bool EmitLeadingSpaceOrAdvance(
       IoStatementState &, std::size_t = 1, bool isCharacter = false);
-  Fortran::common::optional<DataEdit> GetNextDataEdit(
+  RT_API_ATTRS Fortran::common::optional<DataEdit> GetNextDataEdit(
       IoStatementState &, int maxRepeat = 1);
-  bool lastWasUndelimitedCharacter() const {
+  RT_API_ATTRS bool lastWasUndelimitedCharacter() const {
     return lastWasUndelimitedCharacter_;
   }
-  void set_lastWasUndelimitedCharacter(bool yes = true) {
+  RT_API_ATTRS void set_lastWasUndelimitedCharacter(bool yes = true) {
     lastWasUndelimitedCharacter_ = yes;
   }
 
@@ -314,20 +324,20 @@ template <>
 class ListDirectedStatementState<Direction::Input>
     : public FormattedIoStatementState<Direction::Input> {
 public:
-  bool inNamelistSequence() const { return inNamelistSequence_; }
-  int EndIoStatement();
+  RT_API_ATTRS bool inNamelistSequence() const { return inNamelistSequence_; }
+  RT_API_ATTRS int EndIoStatement();
 
   // Skips value separators, handles repetition and null values.
   // Vacant when '/' appears; present with descriptor == ListDirectedNullValue
   // when a null value appears.
-  Fortran::common::optional<DataEdit> GetNextDataEdit(
+  RT_API_ATTRS Fortran::common::optional<DataEdit> GetNextDataEdit(
       IoStatementState &, int maxRepeat = 1);
 
   // Each NAMELIST input item is treated like a distinct list-directed
   // input statement.  This member function resets some state so that
   // repetition and null values work correctly for each successive
   // NAMELIST input item.
-  void ResetForNextNamelistItem(bool inNamelistSequence) {
+  RT_API_ATTRS void ResetForNextNamelistItem(bool inNamelistSequence) {
     remaining_ = 0;
     if (repeatPosition_) {
       repeatPosition_->Cancel();
@@ -353,21 +363,22 @@ class InternalIoStatementState : public IoStatementBase,
 public:
   using Buffer =
       std::conditional_t<DIR == Direction::Input, const char *, char *>;
-  InternalIoStatementState(Buffer, std::size_t,
+  RT_API_ATTRS InternalIoStatementState(Buffer, std::size_t,
       const char *sourceFile = nullptr, int sourceLine = 0);
-  InternalIoStatementState(
+  RT_API_ATTRS InternalIoStatementState(
       const Descriptor &, const char *sourceFile = nullptr, int sourceLine = 0);
-  int EndIoStatement();
-
-  bool Emit(const char *data, std::size_t bytes, std::size_t elementBytes = 0);
-  std::size_t GetNextInputBytes(const char *&);
-  bool AdvanceRecord(int = 1);
-  void BackspaceRecord();
-  ConnectionState &GetConnectionState() { return unit_; }
-  MutableModes &mutableModes() { return unit_.modes; }
-  void HandleRelativePosition(std::int64_t);
-  void HandleAbsolutePosition(std::int64_t);
-  std::int64_t InquirePos();
+  RT_API_ATTRS int EndIoStatement();
+
+  RT_API_ATTRS bool Emit(
+      const char *data, std::size_t bytes, std::size_t elementBytes = 0);
+  RT_API_ATTRS std::size_t GetNextInputBytes(const char *&);
+  RT_API_ATTRS bool AdvanceRecord(int = 1);
+  RT_API_ATTRS void BackspaceRecord();
+  RT_API_ATTRS ConnectionState &GetConnectionState() { return unit_; }
+  RT_API_ATTRS MutableModes &mutableModes() { return unit_.modes; }
+  RT_API_ATTRS void HandleRelativePosition(std::int64_t);
+  RT_API_ATTRS void HandleAbsolutePosition(std::int64_t);
+  RT_API_ATTRS std::int64_t InquirePos();
 
 protected:
   bool free_{true};
@@ -381,17 +392,20 @@ class InternalFormattedIoStatementState
 public:
   using CharType = CHAR;
   using typename InternalIoStatementState<DIR>::Buffer;
-  InternalFormattedIoStatementState(Buffer internal, std::size_t internalLength,
+  RT_API_ATTRS InternalFormattedIoStatementState(Buffer internal,
+      std::size_t internalLength, const CharType *format,
+      std::size_t formatLength, const Descriptor *formatDescriptor = nullptr,
+      const char *sourceFile = nullptr, int sourceLine = 0);
+  RT_API_ATTRS InternalFormattedIoStatementState(const Descriptor &,
       const CharType *format, std::size_t formatLength,
       const Descriptor *formatDescriptor = nullptr,
       const char *sourceFile = nullptr, int sourceLine = 0);
-  InternalFormattedIoStatementState(const Descriptor &, const CharType *format,
-      std::size_t formatLength, const Descriptor *formatDescriptor = nullptr,
-      const char *sourceFile = nullptr, int sourceLine = 0);
-  IoStatementState &ioStatementState() { return ioStatementState_; }
-  void CompleteOperation();
-  int EndIoStatement();
-  Fortran::common::optional<DataEdit> GetNextDataEdit(
+  RT_API_ATTRS IoStatementState &ioStatementState() {
+    return ioStatementState_;
+  }
+  RT_API_ATTRS void CompleteOperation();
+  RT_API_ATTRS int EndIoStatement();
+  RT_API_ATTRS Fortran::common::optional<DataEdit> GetNextDataEdit(
       IoStatementState &, int maxRepeat = 1) {
     return format_.GetNextDataEdit(*this, maxRepeat);
   }
@@ -408,14 +422,17 @@ class InternalListIoStatementState : public InternalIoStatementState<DIR>,
                                      public ListDirectedStatementState<DIR> {
 public:
   using typename InternalIoStatementState<DIR>::Buffer;
-  InternalListIoStatementState(Buffer internal, std::size_t internalLength,
-      const char *sourceFile = nullptr, int sourceLine = 0);
-  InternalListIoStatementState(
+  RT_API_ATTRS InternalListIoStatementState(Buffer internal,
+      std::size_t internalLength, const char *sourceFile = nullptr,
+      int sourceLine = 0);
+  RT_API_ATTRS InternalListIoStatementState(
       const Descriptor &, const char *sourceFile = nullptr, int sourceLine = 0);
-  IoStatementState &ioStatementState() { return ioStatementState_; }
+  RT_API_ATTRS IoStatementState &ioStatementState() {
+    return ioStatementState_;
+  }
   using ListDirectedStatementState<DIR>::GetNextDataEdit;
-  void CompleteOperation();
-  int EndIoStatement();
+  RT_API_ATTRS void CompleteOperation();
+  RT_API_ATTRS int EndIoStatement();
 
 private:
   IoStatementState ioStatementState_; // points to *this
@@ -424,16 +441,16 @@ private:
 
 class ExternalIoStatementBase : public IoStatementBase {
 public:
-  ExternalIoStatementBase(
+  RT_API_ATTRS ExternalIoStatementBase(
       ExternalFileUnit &, const char *sourceFile = nullptr, int sourceLine = 0);
-  ExternalFileUnit &unit() { return unit_; }
-  MutableModes &mutableModes();
-  ConnectionState &GetConnectionState();
-  int asynchronousID() const { return asynchronousID_; }
-  int EndIoStatement();
-  ExternalFileUnit *GetExternalFileUnit() const { return &unit_; }
-  void SetAsynchronous();
-  std::int64_t InquirePos();
+  RT_API_ATTRS ExternalFileUnit &unit() { return unit_; }
+  RT_API_ATTRS MutableModes &mutableModes();
+  RT_API_ATTRS ConnectionState &GetConnectionState();
+  RT_API_ATTRS int asynchronousID() const { return asynchronousID_; }
+  RT_API_ATTRS int EndIoStatement();
+  RT_API_ATTRS ExternalFileUnit *GetExternalFileUnit() const { return &unit_; }
+  RT_API_ATTRS void SetAsynchronous();
+  RT_API_ATTRS std::int64_t InquirePos();
 
 private:
   ExternalFileUnit &unit_;
@@ -444,19 +461,20 @@ template <Direction DIR>
 class ExternalIoStatementState : public ExternalIoStatementBase,
                                  public IoDirectionState<DIR> {
 public:
-  ExternalIoStatementState(
+  RT_API_ATTRS ExternalIoStatementState(
       ExternalFileUnit &, const char *sourceFile = nullptr, int sourceLine = 0);
-  MutableModes &mutableModes() { return mutableModes_; }
-  void CompleteOperation();
-  int EndIoStatement();
-  bool Emit(const char *, std::size_t bytes, std::size_t elementBytes = 0);
-  std::size_t GetNextInputBytes(const char *&);
-  bool AdvanceRecord(int = 1);
-  void BackspaceRecord();
-  void HandleRelativePosition(std::int64_t);
-  void HandleAbsolutePosition(std::int64_t);
-  bool BeginReadingRecord();
-  void FinishReadingRecord();
+  RT_API_ATTRS MutableModes &mutableModes() { return mutableModes_; }
+  RT_API_ATTRS void CompleteOperation();
+  RT_API_ATTRS int EndIoStatement();
+  RT_API_ATTRS bool Emit(
+      const char *, std::size_t bytes, std::size_t elementBytes = 0);
+  RT_API_ATTRS std::size_t GetNextInputBytes(const char *&);
+  RT_API_ATTRS bool AdvanceRecord(int = 1);
+  RT_API_ATTRS void BackspaceRecord();
+  RT_API_ATTRS void HandleRelativePosition(std::int64_t);
+  RT_API_ATTRS void HandleAbsolutePosition(std::int64_t);
+  RT_API_ATTRS bool BeginReadingRecord();
+  RT_API_ATTRS void FinishReadingRecord();
 
 private:
   // These are forked from ConnectionState's modes at the beginning
@@ -471,12 +489,13 @@ class ExternalFormattedIoStatementState
       public FormattedIoStatementState<DIR> {
 public:
   using CharType = CHAR;
-  ExternalFormattedIoStatementState(ExternalFileUnit &, const CharType *format,
-      std::size_t formatLength, const Descriptor *formatDescriptor = nullptr,
+  RT_API_ATTRS ExternalFormattedIoStatementState(ExternalFileUnit &,
+      const CharType *format, std::size_t formatLength,
+      const Descriptor *formatDescriptor = nullptr,
       const char *sourceFile = nullptr, int sourceLine = 0);
-  void CompleteOperation();
-  int EndIoStatement();
-  Fortran::common::optional<DataEdit> GetNextDataEdit(
+  RT_API_ATTRS void CompleteOperation();
+  RT_API_ATTRS int EndIoStatement();
+  RT_API_ATTRS Fortran::common::optional<DataEdit> GetNextDataEdit(
       IoStatementState &, int maxRepeat = 1) {
     return format_.GetNextDataEdit(*this, maxRepeat);
   }
@@ -491,7 +510,7 @@ class ExternalListIoStatementState : public ExternalIoStatementState<DIR>,
 public:
   using ExternalIoStatementState<DIR>::ExternalIoStatementState;
   using ListDirectedStatementState<DIR>::GetNextDataEdit;
-  int EndIoStatement();
+  RT_API_ATTRS int EndIoStatement();
 };
 
 template <Direction DIR>
@@ -499,24 +518,25 @@ class ExternalUnformattedIoStatementState
     : public ExternalIoStatementState<DIR> {
 public:
   using ExternalIoStatementState<DIR>::ExternalIoStatementState;
-  bool Receive(char *, std::size_t, std::size_t elementBytes = 0);
+  RT_API_ATTRS bool Receive(char *, std::size_t, std::size_t elementBytes = 0);
 };
 
 template <Direction DIR>
 class ChildIoStatementState : public IoStatementBase,
                               public IoDirectionState<DIR> {
 public:
-  ChildIoStatementState(
+  RT_API_ATTRS ChildIoStatementState(
       ChildIo &, const char *sourceFile = nullptr, int sourceLine = 0);
-  ChildIo &child() { return child_; }
-  MutableModes &mutableModes();
-  ConnectionState &GetConnectionState();
-  ExternalFileUnit *GetExternalFileUnit() const;
-  int EndIoStatement();
-  bool Emit(const char *, std::size_t bytes, std::size_t elementBytes = 0);
-  std::size_t GetNextInputBytes(const char *&);
-  void HandleRelativePosition(std::int64_t);
-  void HandleAbsolutePosition(std::int64_t);
+  RT_API_ATTRS ChildIo &child() { return child_; }
+  RT_API_ATTRS MutableModes &mutableModes();
+  RT_API_ATTRS ConnectionState &GetConnectionState();
+  RT_API_ATTRS ExternalFileUnit *GetExternalFileUnit() const;
+  RT_API_ATTRS int EndIoStatement();
+  RT_API_ATTRS bool Emit(
+      const char *, std::size_t bytes, std::size_t elementBytes = 0);
+  RT_API_ATTRS std::size_t GetNextInputBytes(const char *&);
+  RT_API_ATTRS void HandleRelativePosition(std::int64_t);
+  RT_API_ATTRS void HandleAbsolutePosition(std::int64_t);
 
 private:
   ChildIo &child_;
@@ -527,14 +547,14 @@ class ChildFormattedIoStatementState : public ChildIoStatementState<DIR>,
                                        public FormattedIoStatementState<DIR> {
 public:
   using CharType = CHAR;
-  ChildFormattedIoStatementState(ChildIo &, const CharType *format,
+  RT_API_ATTRS ChildFormattedIoStatementState(ChildIo &, const CharType *format,
       std::size_t formatLength, const Descriptor *formatDescriptor = nullptr,
       const char *sourceFile = nullptr, int sourceLine = 0);
-  MutableModes &mutableModes() { return mutableModes_; }
-  void CompleteOperation();
-  int EndIoStatement();
-  bool AdvanceRecord(int = 1);
-  Fortran::common::optional<DataEdit> GetNextDataEdit(
+  RT_API_ATTRS MutableModes &mutableModes() { return mutableModes_; }
+  RT_API_ATTRS void CompleteOperation();
+  RT_API_ATTRS int EndIoStatement();
+  RT_API_ATTRS bool AdvanceRecord(int = 1);
+  RT_API_ATTRS Fortran::common::optional<DataEdit> GetNextDataEdit(
       IoStatementState &, int maxRepeat = 1) {
     return format_.GetNextDataEdit(*this, maxRepeat);
   }
@@ -550,34 +570,42 @@ class ChildListIoStatementState : public ChildIoStatementState<DIR>,
 public:
   using ChildIoStatementState<DIR>::ChildIoStatementState;
   using ListDirectedStatementState<DIR>::GetNextDataEdit;
-  int EndIoStatement();
+  RT_API_ATTRS int EndIoStatement();
 };
 
 template <Direction DIR>
 class ChildUnformattedIoStatementState : public ChildIoStatementState<DIR> {
 public:
   using ChildIoStatementState<DIR>::ChildIoStatementState;
-  bool Receive(char *, std::size_t, std::size_t elementBytes = 0);
+  RT_API_ATTRS bool Receive(char *, std::size_t, std::size_t elementBytes = 0);
 };
 
 // OPEN
 class OpenStatementState : public ExternalIoStatementBase {
 public:
-  OpenStatementState(ExternalFileUnit &unit, bool wasExtant, bool isNewUnit,
-      const char *sourceFile = nullptr, int sourceLine = 0)
+  RT_API_ATTRS OpenStatementState(ExternalFileUnit &unit, bool wasExtant,
+      bool isNewUnit, const char *sourceFile = nullptr, int sourceLine = 0)
       : ExternalIoStatementBase{unit, sourceFile, sourceLine},
         wasExtant_{wasExtant}, isNewUnit_{isNewUnit} {}
-  bool wasExtant() const { return wasExtant_; }
-  void set_status(OpenStatus status) { status_ = status; } // STATUS=
-  void set_path(const char *, std::size_t); // FILE=
-  void set_position(Position position) { position_ = position; } // POSITION=
-  void set_action(Action action) { action_ = action; } // ACTION=
-  void set_convert(Convert convert) { convert_ = convert; } // CONVERT=
-  void set_access(Access access) { access_ = access; } // ACCESS=
-  void set_isUnformatted(bool yes = true) { isUnformatted_ = yes; } // FORM=
-
-  void CompleteOperation();
-  int EndIoStatement();
+  RT_API_ATTRS bool wasExtant() const { return wasExtant_; }
+  RT_API_ATTRS void set_status(OpenStatus status) {
+    status_ = status;
+  } // STATUS=
+  RT_API_ATTRS void set_path(const char *, std::size_t); // FILE=
+  RT_API_ATTRS void set_position(Position position) {
+    position_ = position;
+  } // POSITION=
+  RT_API_ATTRS void set_action(Action action) { action_ = action; } // ACTION=
+  RT_API_ATTRS void set_convert(Convert convert) {
+    convert_ = convert;
+  } // CONVERT=
+  RT_API_ATTRS void set_access(Access access) { access_ = access; } // ACCESS=
+  RT_API_ATTRS void set_isUnformatted(bool yes = true) {
+    isUnformatted_ = yes;
+  } // FORM=
+
+  RT_API_ATTRS void CompleteOperation();
+  RT_API_ATTRS int EndIoStatement();
 
 private:
   bool wasExtant_;
@@ -594,11 +622,11 @@ private:
 
 class CloseStatementState : public ExternalIoStatementBase {
 public:
-  CloseStatementState(ExternalFileUnit &unit, const char *sourceFile = nullptr,
-      int sourceLine = 0)
+  RT_API_ATTRS CloseStatementState(ExternalFileUnit &unit,
+      const char *sourceFile = nullptr, int sourceLine = 0)
       : ExternalIoStatementBase{unit, sourceFile, sourceLine} {}
-  void set_status(CloseStatus status) { status_ = status; }
-  int EndIoStatement();
+  RT_API_ATTRS void set_status(CloseStatus status) { status_ = status; }
+  RT_API_ATTRS int EndIoStatement();
 
 private:
   CloseStatus status_{CloseStatus::Keep};
@@ -608,16 +636,18 @@ private:
 // and recoverable BACKSPACE(bad unit)
 class NoUnitIoStatementState : public IoStatementBase {
 public:
-  IoStatementState &ioStatementState() { return ioStatementState_; }
-  MutableModes &mutableModes() { return connection_.modes; }
-  ConnectionState &GetConnectionState() { return connection_; }
-  int badUnitNumber() const { return badUnitNumber_; }
-  void CompleteOperation();
-  int EndIoStatement();
+  RT_API_ATTRS IoStatementState &ioStatementState() {
+    return ioStatementState_;
+  }
+  RT_API_ATTRS MutableModes &mutableModes() { return connection_.modes; }
+  RT_API_ATTRS ConnectionState &GetConnectionState() { return connection_; }
+  RT_API_ATTRS int badUnitNumber() const { return badUnitNumber_; }
+  RT_API_ATTRS void CompleteOperation();
+  RT_API_ATTRS int EndIoStatement();
 
 protected:
   template <typename A>
-  NoUnitIoStatementState(A &stmt, const char *sourceFile = nullptr,
+  RT_API_ATTRS NoUnitIoStatementState(A &stmt, const char *sourceFile = nullptr,
       int sourceLine = 0, int badUnitNumber = -1)
       : IoStatementBase{sourceFile, sourceLine}, ioStatementState_{stmt},
         badUnitNumber_{badUnitNumber} {}
@@ -630,10 +660,10 @@ private:
 
 class NoopStatementState : public NoUnitIoStatementState {
 public:
-  NoopStatementState(
+  RT_API_ATTRS NoopStatementState(
       const char *sourceFile = nullptr, int sourceLine = 0, int unitNumber = -1)
       : NoUnitIoStatementState{*this, sourceFile, sourceLine, unitNumber} {}
-  void set_status(CloseStatus) {} // discards
+  RT_API_ATTRS void set_status(CloseStatus) {} // discards
 };
 
 extern template class InternalIoStatementState<Direction::Output>;
@@ -674,32 +704,32 @@ extern template class FormatControl<
 
 class InquireUnitState : public ExternalIoStatementBase {
 public:
-  InquireUnitState(ExternalFileUnit &unit, const char *sourceFile = nullptr,
-      int sourceLine = 0);
-  bool Inquire(InquiryKeywordHash, char *, std::size_t);
-  bool Inquire(InquiryKeywordHash, bool &);
-  bool Inquire(InquiryKeywordHash, std::int64_t, bool &);
-  bool Inquire(InquiryKeywordHash, std::int64_t &);
+  RT_API_ATTRS InquireUnitState(ExternalFileUnit &unit,
+      const char *sourceFile = nullptr, int sourceLine = 0);
+  RT_API_ATTRS bool Inquire(InquiryKeywordHash, char *, std::size_t);
+  RT_API_ATTRS bool Inquire(InquiryKeywordHash, bool &);
+  RT_API_ATTRS bool Inquire(InquiryKeywordHash, std::int64_t, bool &);
+  RT_API_ATTRS bool Inquire(InquiryKeywordHash, std::int64_t &);
 };
 
 class InquireNoUnitState : public NoUnitIoStatementState {
 public:
-  InquireNoUnitState(const char *sourceFile = nullptr, int sourceLine = 0,
-      int badUnitNumber = -1);
-  bool Inquire(InquiryKeywordHash, char *, std::size_t);
-  bool Inquire(InquiryKeywordHash, bool &);
-  bool Inquire(InquiryKeywordHash, std::int64_t, bool &);
-  bool Inquire(InquiryKeywordHash, std::int64_t &);
+  RT_API_ATTRS InquireNoUnitState(const char *sourceFile = nullptr,
+      int sourceLine = 0, int badUnitNumber = -1);
+  RT_API_ATTRS bool Inquire(InquiryKeywordHash, char *, std::size_t);
+  RT_API_ATTRS bool Inquire(InquiryKeywordHash, bool &);
+  RT_API_ATTRS bool Inquire(InquiryKeywordHash, std::int64_t, bool &);
+  RT_API_ATTRS bool Inquire(InquiryKeywordHash, std::int64_t &);
 };
 
 class InquireUnconnectedFileState : public NoUnitIoStatementState {
 public:
-  InquireUnconnectedFileState(OwningPtr<char> &&path,
+  RT_API_ATTRS InquireUnconnectedFileState(OwningPtr<char> &&path,
       const char *sourceFile = nullptr, int sourceLine = 0);
-  bool Inquire(InquiryKeywordHash, char *, std::size_t);
-  bool Inquire(InquiryKeywordHash, bool &);
-  bool Inquire(InquiryKeywordHash, std::int64_t, bool &);
-  bool Inquire(InquiryKeywordHash, std::int64_t &);
+  RT_API_ATTRS bool Inquire(InquiryKeywordHash, char *, std::size_t);
+  RT_API_ATTRS bool Inquire(InquiryKeywordHash, bool &);
+  RT_API_ATTRS bool Inquire(InquiryKeywordHash, std::int64_t, bool &);
+  RT_API_ATTRS bool Inquire(InquiryKeywordHash, std::int64_t &);
 
 private:
   OwningPtr<char> path_; // trimmed and NUL terminated
@@ -708,9 +738,11 @@ private:
 class InquireIOLengthState : public NoUnitIoStatementState,
                              public OutputStatementState {
 public:
-  InquireIOLengthState(const char *sourceFile = nullptr, int sourceLine = 0);
-  std::size_t bytes() const { return bytes_; }
-  bool Emit(const char *, std::size_t bytes, std::size_t elementBytes = 0);
+  RT_API_ATTRS InquireIOLengthState(
+      const char *sourceFile = nullptr, int sourceLine = 0);
+  RT_API_ATTRS std::size_t bytes() const { return bytes_; }
+  RT_API_ATTRS bool Emit(
+      const char *, std::size_t bytes, std::size_t elementBytes = 0);
 
 private:
   std::size_t bytes_{0};
@@ -719,11 +751,11 @@ private:
 class ExternalMiscIoStatementState : public ExternalIoStatementBase {
 public:
   enum Which { Flush, Backspace, Endfile, Rewind, Wait };
-  ExternalMiscIoStatementState(ExternalFileUnit &unit, Which which,
+  RT_API_ATTRS ExternalMiscIoStatementState(ExternalFileUnit &unit, Which which,
       const char *sourceFile = nullptr, int sourceLine = 0)
       : ExternalIoStatementBase{unit, sourceFile, sourceLine}, which_{which} {}
-  void CompleteOperation();
-  int EndIoStatement();
+  RT_API_ATTRS void CompleteOperation();
+  RT_API_ATTRS int EndIoStatement();
 
 private:
   Which which_;
@@ -731,15 +763,15 @@ private:
 
 class ErroneousIoStatementState : public IoStatementBase {
 public:
-  explicit ErroneousIoStatementState(Iostat iostat,
+  explicit RT_API_ATTRS ErroneousIoStatementState(Iostat iostat,
       ExternalFileUnit *unit = nullptr, const char *sourceFile = nullptr,
       int sourceLine = 0)
       : IoStatementBase{sourceFile, sourceLine}, unit_{unit} {
     SetPendingError(iostat);
   }
-  int EndIoStatement();
-  ConnectionState &GetConnectionState() { return connection_; }
-  MutableModes &mutableModes() { return connection_.modes; }
+  RT_API_ATTRS int EndIoStatement();
+  RT_API_ATTRS ConnectionState &GetConnectionState() { return connection_; }
+  RT_API_ATTRS MutableModes &mutableModes() { return connection_.modes; }
 
 private:
   ConnectionState connection_;
diff --git a/flang/runtime/iostat.cpp b/flang/runtime/iostat.cpp
index c993b778e9e1..39e224cb0128 100644
--- a/flang/runtime/iostat.cpp
+++ b/flang/runtime/iostat.cpp
@@ -9,6 +9,8 @@
 #include "flang/Runtime/iostat.h"
 
 namespace Fortran::runtime::io {
+RT_OFFLOAD_API_GROUP_BEGIN
+
 const char *IostatErrorString(int iostat) {
   switch (iostat) {
   case IostatOk:
@@ -122,4 +124,6 @@ const char *IostatErrorString(int iostat) {
   }
 }
 
+RT_OFFLOAD_API_GROUP_END
+
 } // namespace Fortran::runtime::io
diff --git a/flang/runtime/lock.h b/flang/runtime/lock.h
index 61b06a62ff7c..9f27a8295c46 100644
--- a/flang/runtime/lock.h
+++ b/flang/runtime/lock.h
@@ -42,10 +42,10 @@ public:
   // The users of Lock class may use it under
   // USE_PTHREADS and otherwise, so it has to provide
   // all the interfaces.
-  void Take() {}
-  bool Try() { return true; }
-  void Drop() {}
-  bool TakeIfNoDeadlock() { return true; }
+  RT_API_ATTRS void Take() {}
+  RT_API_ATTRS bool Try() { return true; }
+  RT_API_ATTRS void Drop() {}
+  RT_API_ATTRS bool TakeIfNoDeadlock() { return true; }
 #elif USE_PTHREADS
   Lock() { pthread_mutex_init(&mutex_, nullptr); }
   ~Lock() { pthread_mutex_destroy(&mutex_); }
@@ -105,8 +105,10 @@ private:
 
 class CriticalSection {
 public:
-  explicit CriticalSection(Lock &lock) : lock_{lock} { lock_.Take(); }
-  ~CriticalSection() { lock_.Drop(); }
+  explicit RT_API_ATTRS CriticalSection(Lock &lock) : lock_{lock} {
+    lock_.Take();
+  }
+  RT_API_ATTRS ~CriticalSection() { lock_.Drop(); }
 
 private:
   Lock &lock_;
diff --git a/flang/runtime/memory.cpp b/flang/runtime/memory.cpp
index aa6ff9723d1a..de6c4c72fdac 100644
--- a/flang/runtime/memory.cpp
+++ b/flang/runtime/memory.cpp
@@ -7,15 +7,15 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Runtime/memory.h"
+#include "freestanding-tools.h"
 #include "terminator.h"
 #include "tools.h"
 #include <cstdlib>
 
 namespace Fortran::runtime {
-RT_OFFLOAD_VAR_GROUP_BEGIN
+RT_OFFLOAD_API_GROUP_BEGIN
 
-RT_API_ATTRS void *AllocateMemoryOrCrash(
-    const Terminator &terminator, std::size_t bytes) {
+void *AllocateMemoryOrCrash(const Terminator &terminator, std::size_t bytes) {
   if (void *p{std::malloc(bytes)}) {
     return p;
   }
@@ -27,7 +27,7 @@ RT_API_ATTRS void *AllocateMemoryOrCrash(
   return nullptr;
 }
 
-RT_API_ATTRS void *ReallocateMemoryOrCrash(
+void *ReallocateMemoryOrCrash(
     const Terminator &terminator, void *ptr, std::size_t newByteSize) {
   if (void *p{Fortran::runtime::realloc(ptr, newByteSize)}) {
     return p;
@@ -40,7 +40,7 @@ RT_API_ATTRS void *ReallocateMemoryOrCrash(
   return nullptr;
 }
 
-RT_API_ATTRS void FreeMemory(void *p) { std::free(p); }
+void FreeMemory(void *p) { std::free(p); }
 
-RT_OFFLOAD_VAR_GROUP_END
+RT_OFFLOAD_API_GROUP_END
 } // namespace Fortran::runtime
diff --git a/flang/runtime/namelist.cpp b/flang/runtime/namelist.cpp
index ac9234f4af83..b502d41a8d5c 100644
--- a/flang/runtime/namelist.cpp
+++ b/flang/runtime/namelist.cpp
@@ -31,9 +31,12 @@ bool IONAME(OutputNamelist)(Cookie cookie, const NamelistGroup &group) {
   io.CheckFormattedStmtType<Direction::Output>("OutputNamelist");
   io.mutableModes().inNamelist = true;
   ConnectionState &connection{io.GetConnectionState()};
+  // The following lambda definition violates the conding style,
+  // but cuda-11.8 nvcc hits an internal error with the brace initialization.
+
   // Internal function to advance records and convert case
-  const auto EmitUpperCase{[&](const char *prefix, std::size_t prefixLen,
-                               const char *str, char suffix) -> bool {
+  const auto EmitUpperCase = [&](const char *prefix, std::size_t prefixLen,
+                                 const char *str, char suffix) -> bool {
     if ((connection.NeedAdvance(prefixLen) &&
             !(io.AdvanceRecord() && EmitAscii(io, " ", 1))) ||
         !EmitAscii(io, prefix, prefixLen) ||
@@ -49,7 +52,7 @@ bool IONAME(OutputNamelist)(Cookie cookie, const NamelistGroup &group) {
       }
     }
     return suffix == ' ' || EmitAscii(io, &suffix, 1);
-  }};
+  };
   // &GROUP
   if (!EmitUpperCase(" &", 2, group.groupName, ' ')) {
     return false;
@@ -294,7 +297,7 @@ static bool HandleSubstring(
       ch = io.GetNextNonBlank(byteCount);
     }
   }
-  if (ch && ch == ':') {
+  if (ch && *ch == ':') {
     io.HandleRelativePosition(byteCount);
     ch = io.GetNextNonBlank(byteCount);
     if (ch) {
@@ -587,6 +590,8 @@ bool IONAME(InputNamelist)(Cookie cookie, const NamelistGroup &group) {
   return true;
 }
 
+RT_OFFLOAD_API_GROUP_BEGIN
+
 bool IsNamelistNameOrSlash(IoStatementState &io) {
   if (auto *listInput{
           io.get_if<ListDirectedStatementState<Direction::Input>>()}) {
@@ -611,4 +616,6 @@ bool IsNamelistNameOrSlash(IoStatementState &io) {
   return false;
 }
 
+RT_OFFLOAD_API_GROUP_END
+
 } // namespace Fortran::runtime::io
diff --git a/flang/runtime/namelist.h b/flang/runtime/namelist.h
index 9a5da33a907e..25216a75e936 100644
--- a/flang/runtime/namelist.h
+++ b/flang/runtime/namelist.h
@@ -12,6 +12,7 @@
 #define FORTRAN_RUNTIME_NAMELIST_H_
 
 #include "non-tbp-dio.h"
+#include "flang/Common/api-attrs.h"
 
 #include <cstddef>
 
@@ -47,7 +48,7 @@ public:
 // character; for use in disambiguating a name-like value (e.g. F or T) from a
 // NAMELIST group item name and for coping with short arrays.  Always false
 // when not reading a NAMELIST.
-bool IsNamelistNameOrSlash(IoStatementState &);
+RT_API_ATTRS bool IsNamelistNameOrSlash(IoStatementState &);
 
 } // namespace Fortran::runtime::io
 #endif // FORTRAN_RUNTIME_NAMELIST_H_
diff --git a/flang/runtime/non-tbp-dio.h b/flang/runtime/non-tbp-dio.h
index a2030dbfdfe8..05038a264ed9 100644
--- a/flang/runtime/non-tbp-dio.h
+++ b/flang/runtime/non-tbp-dio.h
@@ -39,7 +39,7 @@ struct NonTbpDefinedIo {
 };
 
 struct NonTbpDefinedIoTable {
-  const NonTbpDefinedIo *Find(
+  RT_API_ATTRS const NonTbpDefinedIo *Find(
       const typeInfo::DerivedType &, common::DefinedIo) const;
   std::size_t items{0};
   const NonTbpDefinedIo *item{nullptr};
diff --git a/flang/runtime/numeric-templates.h b/flang/runtime/numeric-templates.h
index 8ea3daaa57bc..af552f9ddfc0 100644
--- a/flang/runtime/numeric-templates.h
+++ b/flang/runtime/numeric-templates.h
@@ -20,8 +20,8 @@
 
 #include "terminator.h"
 #include "tools.h"
+#include "flang/Common/api-attrs.h"
 #include "flang/Common/float128.h"
-#include "flang/Runtime/api-attrs.h"
 #include <cstdint>
 #include <limits>
 
@@ -193,11 +193,6 @@ inline RT_API_ATTRS RESULT Exponent(ARG x) {
   }
 }
 
-// Suppress the warnings about calling __host__-only std::frexp,
-// defined in C++ STD header files, from __device__ code.
-RT_DIAG_PUSH
-RT_DIAG_DISABLE_CALL_HOST_FROM_DEVICE_WARN
-
 // FRACTION (16.9.80)
 template <typename T> inline RT_API_ATTRS T Fraction(T x) {
   if (ISNANTy<T>::compute(x)) {
@@ -212,8 +207,6 @@ template <typename T> inline RT_API_ATTRS T Fraction(T x) {
   }
 }
 
-RT_DIAG_POP
-
 // SET_EXPONENT (16.9.171)
 template <typename T> inline RT_API_ATTRS T SetExponent(T x, std::int64_t p) {
   if (ISNANTy<T>::compute(x)) {
diff --git a/flang/runtime/pointer.cpp b/flang/runtime/pointer.cpp
index b01735dc30e6..08a1223764f3 100644
--- a/flang/runtime/pointer.cpp
+++ b/flang/runtime/pointer.cpp
@@ -185,7 +185,6 @@ int RTDEF(PointerDeallocate)(Descriptor &pointer, bool hasStat,
   if (!pointer.IsAllocated()) {
     return ReturnError(terminator, StatBaseNull, errMsg, hasStat);
   }
-#if !defined(RT_DEVICE_COMPILATION)
   if (executionEnvironment.checkPointerDeallocation) {
     // Validate the footer.  This should fail if the pointer doesn't
     // span the entire object, or the object was not allocated as a
@@ -201,7 +200,6 @@ int RTDEF(PointerDeallocate)(Descriptor &pointer, bool hasStat,
           terminator, StatBadPointerDeallocation, errMsg, hasStat);
     }
   }
-#endif
   return ReturnError(terminator,
       pointer.Destroy(/*finalize=*/true, /*destroyPointers=*/true, &terminator),
       errMsg, hasStat);
diff --git a/flang/runtime/pseudo-unit.cpp b/flang/runtime/pseudo-unit.cpp
index 8b5f36e2233a..a57e3a59efa5 100644
--- a/flang/runtime/pseudo-unit.cpp
+++ b/flang/runtime/pseudo-unit.cpp
@@ -11,12 +11,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "io-error.h"
 #include "tools.h"
+#include "unit.h"
 
+// NOTE: the header files above may define OpenMP declare target
+// variables, so they have to be included unconditionally
+// so that the offload entries are consistent between host and device.
 #if defined(RT_USE_PSEUDO_FILE_UNIT)
-
-#include "io-error.h"
-#include "unit.h"
 #include <cstdio>
 
 namespace Fortran::runtime::io {
diff --git a/flang/runtime/stat.h b/flang/runtime/stat.h
index 55cdac46eb3a..4f46f52ecb29 100644
--- a/flang/runtime/stat.h
+++ b/flang/runtime/stat.h
@@ -11,8 +11,8 @@
 
 #ifndef FORTRAN_RUNTIME_STAT_H_
 #define FORTRAN_RUNTIME_STAT_H_
+#include "flang/Common/api-attrs.h"
 #include "flang/ISO_Fortran_binding_wrapper.h"
-#include "flang/Runtime/api-attrs.h"
 #include "flang/Runtime/magic-numbers.h"
 namespace Fortran::runtime {
 
diff --git a/flang/runtime/terminator.h b/flang/runtime/terminator.h
index 444c68d109ee..59a47ce93e7c 100644
--- a/flang/runtime/terminator.h
+++ b/flang/runtime/terminator.h
@@ -11,7 +11,7 @@
 #ifndef FORTRAN_RUNTIME_TERMINATOR_H_
 #define FORTRAN_RUNTIME_TERMINATOR_H_
 
-#include "flang/Runtime/api-attrs.h"
+#include "flang/Common/api-attrs.h"
 #include <cstdarg>
 #include <cstdio>
 #include <cstdlib>
@@ -67,7 +67,7 @@ public:
 
   template <typename... Args>
   RT_API_ATTRS void PrintCrashArgs(const char *message, Args... args) const {
-#if RT_DEVICE_COMPILATION
+#if defined(RT_DEVICE_COMPILATION)
     std::printf(message, args...);
 #else
     std::fprintf(stderr, message, args...);
diff --git a/flang/runtime/unit.cpp b/flang/runtime/unit.cpp
index 67f4775ae0a9..b5aa307eade8 100644
--- a/flang/runtime/unit.cpp
+++ b/flang/runtime/unit.cpp
@@ -19,17 +19,24 @@
 
 namespace Fortran::runtime::io {
 
-ExternalFileUnit *defaultInput{nullptr}; // unit 5
-ExternalFileUnit *defaultOutput{nullptr}; // unit 6
-ExternalFileUnit *errorOutput{nullptr}; // unit 0 extension
+RT_OFFLOAD_VAR_GROUP_BEGIN
+RT_VAR_ATTRS ExternalFileUnit *defaultInput{nullptr}; // unit 5
+RT_VAR_ATTRS ExternalFileUnit *defaultOutput{nullptr}; // unit 6
+RT_VAR_ATTRS ExternalFileUnit *errorOutput{nullptr}; // unit 0 extension
+RT_OFFLOAD_VAR_GROUP_END
 
-static inline void SwapEndianness(
+RT_OFFLOAD_API_GROUP_BEGIN
+
+static inline RT_API_ATTRS void SwapEndianness(
     char *data, std::size_t bytes, std::size_t elementBytes) {
   if (elementBytes > 1) {
     auto half{elementBytes >> 1};
     for (std::size_t j{0}; j + elementBytes <= bytes; j += elementBytes) {
       for (std::size_t k{0}; k < half; ++k) {
+        RT_DIAG_PUSH
+        RT_DIAG_DISABLE_CALL_HOST_FROM_DEVICE_WARN
         std::swap(data[j + k], data[j + elementBytes - 1 - k]);
+        RT_DIAG_POP
       }
     }
   }
@@ -475,7 +482,10 @@ bool ExternalFileUnit::SetDirectRec(
 
 void ExternalFileUnit::EndIoStatement() {
   io_.reset();
+  RT_DIAG_PUSH
+  RT_DIAG_DISABLE_CALL_HOST_FROM_DEVICE_WARN
   u_.emplace<std::monostate>();
+  RT_DIAG_POP
   lock_.Drop();
 }
 
@@ -600,7 +610,8 @@ void ExternalFileUnit::BackspaceVariableUnformattedRecord(
 
 // There's no portable memrchr(), unfortunately, and strrchr() would
 // fail on a record with a NUL, so we have to do it the hard way.
-static const char *FindLastNewline(const char *str, std::size_t length) {
+static RT_API_ATTRS const char *FindLastNewline(
+    const char *str, std::size_t length) {
   for (const char *p{str + length}; p >= str; p--) {
     if (*p == '\n') {
       return p;
@@ -741,7 +752,10 @@ std::int32_t ExternalFileUnit::ReadHeaderOrFooter(std::int64_t frameOffset) {
 
 void ChildIo::EndIoStatement() {
   io_.reset();
+  RT_DIAG_PUSH
+  RT_DIAG_DISABLE_CALL_HOST_FROM_DEVICE_WARN
   u_.emplace<std::monostate>();
+  RT_DIAG_POP
 }
 
 Iostat ChildIo::CheckFormattingAndDirection(
@@ -764,4 +778,5 @@ Iostat ChildIo::CheckFormattingAndDirection(
   }
 }
 
+RT_OFFLOAD_API_GROUP_END
 } // namespace Fortran::runtime::io
diff --git a/flang/runtime/unit.h b/flang/runtime/unit.h
index 5f854abd42f6..8b7db5cbc90b 100644
--- a/flang/runtime/unit.h
+++ b/flang/runtime/unit.h
@@ -33,10 +33,12 @@ class UnitMap;
 class ChildIo;
 class ExternalFileUnit;
 
+RT_OFFLOAD_VAR_GROUP_BEGIN
 // Predefined file units.
-extern ExternalFileUnit *defaultInput; // unit 5
-extern ExternalFileUnit *defaultOutput; // unit 6
-extern ExternalFileUnit *errorOutput; // unit 0 extension
+extern RT_VAR_ATTRS ExternalFileUnit *defaultInput; // unit 5
+extern RT_VAR_ATTRS ExternalFileUnit *defaultOutput; // unit 6
+extern RT_VAR_ATTRS ExternalFileUnit *errorOutput; // unit 0 extension
+RT_OFFLOAD_VAR_GROUP_END
 
 #if defined(RT_USE_PSEUDO_FILE_UNIT)
 // A flavor of OpenFile class that pretends to be a terminal,
@@ -49,34 +51,36 @@ class PseudoOpenFile {
 public:
   using FileOffset = std::int64_t;
 
-  const char *path() const { return nullptr; }
-  std::size_t pathLength() const { return 0; }
-  void set_path(OwningPtr<char> &&, std::size_t bytes) {}
-  bool mayRead() const { return false; }
-  bool mayWrite() const { return true; }
-  bool mayPosition() const { return false; }
-  bool mayAsynchronous() const { return false; }
-  void set_mayAsynchronous(bool yes);
+  RT_API_ATTRS const char *path() const { return nullptr; }
+  RT_API_ATTRS std::size_t pathLength() const { return 0; }
+  RT_API_ATTRS void set_path(OwningPtr<char> &&, std::size_t bytes) {}
+  RT_API_ATTRS bool mayRead() const { return false; }
+  RT_API_ATTRS bool mayWrite() const { return true; }
+  RT_API_ATTRS bool mayPosition() const { return false; }
+  RT_API_ATTRS bool mayAsynchronous() const { return false; }
+  RT_API_ATTRS void set_mayAsynchronous(bool yes);
   // Pretend to be a terminal to force the output
   // at the end of IO statement.
-  bool isTerminal() const { return true; }
-  bool isWindowsTextFile() const { return false; }
-  Fortran::common::optional<FileOffset> knownSize() const;
-  bool IsConnected() const { return false; }
-  void Open(OpenStatus, Fortran::common::optional<Action>, Position,
-      IoErrorHandler &);
-  void Predefine(int fd) {}
-  void Close(CloseStatus, IoErrorHandler &);
-  std::size_t Read(FileOffset, char *, std::size_t minBytes,
+  RT_API_ATTRS bool isTerminal() const { return true; }
+  RT_API_ATTRS bool isWindowsTextFile() const { return false; }
+  RT_API_ATTRS Fortran::common::optional<FileOffset> knownSize() const;
+  RT_API_ATTRS bool IsConnected() const { return false; }
+  RT_API_ATTRS void Open(OpenStatus, Fortran::common::optional<Action>,
+      Position, IoErrorHandler &);
+  RT_API_ATTRS void Predefine(int fd) {}
+  RT_API_ATTRS void Close(CloseStatus, IoErrorHandler &);
+  RT_API_ATTRS std::size_t Read(FileOffset, char *, std::size_t minBytes,
       std::size_t maxBytes, IoErrorHandler &);
-  std::size_t Write(FileOffset, const char *, std::size_t, IoErrorHandler &);
-  void Truncate(FileOffset, IoErrorHandler &);
-  int ReadAsynchronously(FileOffset, char *, std::size_t, IoErrorHandler &);
-  int WriteAsynchronously(
+  RT_API_ATTRS std::size_t Write(
+      FileOffset, const char *, std::size_t, IoErrorHandler &);
+  RT_API_ATTRS void Truncate(FileOffset, IoErrorHandler &);
+  RT_API_ATTRS int ReadAsynchronously(
+      FileOffset, char *, std::size_t, IoErrorHandler &);
+  RT_API_ATTRS int WriteAsynchronously(
       FileOffset, const char *, std::size_t, IoErrorHandler &);
-  void Wait(int id, IoErrorHandler &);
-  void WaitAll(IoErrorHandler &);
-  Position InquirePosition() const;
+  RT_API_ATTRS void Wait(int id, IoErrorHandler &);
+  RT_API_ATTRS void WaitAll(IoErrorHandler &);
+  RT_API_ATTRS Position InquirePosition() const;
 };
 #endif // defined(RT_USE_PSEUDO_FILE_UNIT)
 
@@ -95,44 +99,51 @@ class ExternalFileUnit : public ConnectionState,
 public:
   static constexpr int maxAsyncIds{64 * 16};
 
-  explicit ExternalFileUnit(int unitNumber) : unitNumber_{unitNumber} {
+  explicit RT_API_ATTRS ExternalFileUnit(int unitNumber)
+      : unitNumber_{unitNumber} {
     isUTF8 = executionEnvironment.defaultUTF8;
     for (int j{0}; 64 * j < maxAsyncIds; ++j) {
       asyncIdAvailable_[j].set();
     }
     asyncIdAvailable_[0].reset(0);
   }
-  ~ExternalFileUnit() {}
+  RT_API_ATTRS ~ExternalFileUnit() {}
 
-  int unitNumber() const { return unitNumber_; }
-  bool swapEndianness() const { return swapEndianness_; }
-  bool createdForInternalChildIo() const { return createdForInternalChildIo_; }
+  RT_API_ATTRS int unitNumber() const { return unitNumber_; }
+  RT_API_ATTRS bool swapEndianness() const { return swapEndianness_; }
+  RT_API_ATTRS bool createdForInternalChildIo() const {
+    return createdForInternalChildIo_;
+  }
 
-  static ExternalFileUnit *LookUp(int unit);
-  static ExternalFileUnit *LookUpOrCreate(
+  static RT_API_ATTRS ExternalFileUnit *LookUp(int unit);
+  static RT_API_ATTRS ExternalFileUnit *LookUpOrCreate(
       int unit, const Terminator &, bool &wasExtant);
-  static ExternalFileUnit *LookUpOrCreateAnonymous(int unit, Direction,
-      Fortran::common::optional<bool> isUnformatted, const Terminator &);
-  static ExternalFileUnit *LookUp(const char *path, std::size_t pathLen);
-  static ExternalFileUnit &CreateNew(int unit, const Terminator &);
-  static ExternalFileUnit *LookUpForClose(int unit);
-  static ExternalFileUnit &NewUnit(const Terminator &, bool forChildIo);
-  static void CloseAll(IoErrorHandler &);
-  static void FlushAll(IoErrorHandler &);
+  static RT_API_ATTRS ExternalFileUnit *LookUpOrCreateAnonymous(int unit,
+      Direction, Fortran::common::optional<bool> isUnformatted,
+      const Terminator &);
+  static RT_API_ATTRS ExternalFileUnit *LookUp(
+      const char *path, std::size_t pathLen);
+  static RT_API_ATTRS ExternalFileUnit &CreateNew(int unit, const Terminator &);
+  static RT_API_ATTRS ExternalFileUnit *LookUpForClose(int unit);
+  static RT_API_ATTRS ExternalFileUnit &NewUnit(
+      const Terminator &, bool forChildIo);
+  static RT_API_ATTRS void CloseAll(IoErrorHandler &);
+  static RT_API_ATTRS void FlushAll(IoErrorHandler &);
 
   // Returns true if an existing unit was closed
-  bool OpenUnit(Fortran::common::optional<OpenStatus>,
+  RT_API_ATTRS bool OpenUnit(Fortran::common::optional<OpenStatus>,
       Fortran::common::optional<Action>, Position, OwningPtr<char> &&path,
       std::size_t pathLength, Convert, IoErrorHandler &);
-  void OpenAnonymousUnit(Fortran::common::optional<OpenStatus>,
+  RT_API_ATTRS void OpenAnonymousUnit(Fortran::common::optional<OpenStatus>,
       Fortran::common::optional<Action>, Position, Convert, IoErrorHandler &);
-  void CloseUnit(CloseStatus, IoErrorHandler &);
-  void DestroyClosed();
+  RT_API_ATTRS void CloseUnit(CloseStatus, IoErrorHandler &);
+  RT_API_ATTRS void DestroyClosed();
 
-  Iostat SetDirection(Direction);
+  RT_API_ATTRS Iostat SetDirection(Direction);
 
   template <typename A, typename... X>
-  IoStatementState &BeginIoStatement(const Terminator &terminator, X &&...xs) {
+  RT_API_ATTRS IoStatementState &BeginIoStatement(
+      const Terminator &terminator, X &&...xs) {
     // Take lock_ and hold it until EndIoStatement().
 #if USE_PTHREADS
     if (!lock_.TakeIfNoDeadlock()) {
@@ -141,7 +152,10 @@ public:
 #else
     lock_.Take();
 #endif
+    RT_DIAG_PUSH
+    RT_DIAG_DISABLE_CALL_HOST_FROM_DEVICE_WARN
     A &state{u_.emplace<A>(std::forward<X>(xs)...)};
+    RT_DIAG_POP
     if constexpr (!std::is_same_v<A, OpenStatementState>) {
       state.mutableModes() = ConnectionState::modes;
     }
@@ -150,50 +164,54 @@ public:
     return *io_;
   }
 
-  bool Emit(
+  RT_API_ATTRS bool Emit(
       const char *, std::size_t, std::size_t elementBytes, IoErrorHandler &);
-  bool Receive(char *, std::size_t, std::size_t elementBytes, IoErrorHandler &);
-  std::size_t GetNextInputBytes(const char *&, IoErrorHandler &);
-  bool BeginReadingRecord(IoErrorHandler &);
-  void FinishReadingRecord(IoErrorHandler &);
-  bool AdvanceRecord(IoErrorHandler &);
-  void BackspaceRecord(IoErrorHandler &);
-  void FlushOutput(IoErrorHandler &);
-  void FlushIfTerminal(IoErrorHandler &);
-  void Endfile(IoErrorHandler &);
-  void Rewind(IoErrorHandler &);
-  void EndIoStatement();
-  bool SetStreamPos(std::int64_t, IoErrorHandler &); // one-based, for POS=
-  bool SetDirectRec(std::int64_t, IoErrorHandler &); // one-based, for REC=
-  std::int64_t InquirePos() const {
+  RT_API_ATTRS bool Receive(
+      char *, std::size_t, std::size_t elementBytes, IoErrorHandler &);
+  RT_API_ATTRS std::size_t GetNextInputBytes(const char *&, IoErrorHandler &);
+  RT_API_ATTRS bool BeginReadingRecord(IoErrorHandler &);
+  RT_API_ATTRS void FinishReadingRecord(IoErrorHandler &);
+  RT_API_ATTRS bool AdvanceRecord(IoErrorHandler &);
+  RT_API_ATTRS void BackspaceRecord(IoErrorHandler &);
+  RT_API_ATTRS void FlushOutput(IoErrorHandler &);
+  RT_API_ATTRS void FlushIfTerminal(IoErrorHandler &);
+  RT_API_ATTRS void Endfile(IoErrorHandler &);
+  RT_API_ATTRS void Rewind(IoErrorHandler &);
+  RT_API_ATTRS void EndIoStatement();
+  RT_API_ATTRS bool SetStreamPos(
+      std::int64_t, IoErrorHandler &); // one-based, for POS=
+  RT_API_ATTRS bool SetDirectRec(
+      std::int64_t, IoErrorHandler &); // one-based, for REC=
+  RT_API_ATTRS std::int64_t InquirePos() const {
     // 12.6.2.11 defines POS=1 as the beginning of file
     return frameOffsetInFile_ + recordOffsetInFrame_ + positionInRecord + 1;
   }
 
-  ChildIo *GetChildIo() { return child_.get(); }
-  ChildIo &PushChildIo(IoStatementState &);
-  void PopChildIo(ChildIo &);
+  RT_API_ATTRS ChildIo *GetChildIo() { return child_.get(); }
+  RT_API_ATTRS ChildIo &PushChildIo(IoStatementState &);
+  RT_API_ATTRS void PopChildIo(ChildIo &);
 
-  int GetAsynchronousId(IoErrorHandler &);
-  bool Wait(int);
+  RT_API_ATTRS int GetAsynchronousId(IoErrorHandler &);
+  RT_API_ATTRS bool Wait(int);
 
 private:
-  static UnitMap &CreateUnitMap();
-  static UnitMap &GetUnitMap();
-  const char *FrameNextInput(IoErrorHandler &, std::size_t);
-  void SetPosition(std::int64_t, IoErrorHandler &); // zero-based
-  void BeginSequentialVariableUnformattedInputRecord(IoErrorHandler &);
-  void BeginVariableFormattedInputRecord(IoErrorHandler &);
-  void BackspaceFixedRecord(IoErrorHandler &);
-  void BackspaceVariableUnformattedRecord(IoErrorHandler &);
-  void BackspaceVariableFormattedRecord(IoErrorHandler &);
-  bool SetVariableFormattedRecordLength();
-  void DoImpliedEndfile(IoErrorHandler &);
-  void DoEndfile(IoErrorHandler &);
-  void CommitWrites();
-  bool CheckDirectAccess(IoErrorHandler &);
-  void HitEndOnRead(IoErrorHandler &);
-  std::int32_t ReadHeaderOrFooter(std::int64_t frameOffset);
+  static RT_API_ATTRS UnitMap &CreateUnitMap();
+  static RT_API_ATTRS UnitMap &GetUnitMap();
+  RT_API_ATTRS const char *FrameNextInput(IoErrorHandler &, std::size_t);
+  RT_API_ATTRS void SetPosition(std::int64_t, IoErrorHandler &); // zero-based
+  RT_API_ATTRS void BeginSequentialVariableUnformattedInputRecord(
+      IoErrorHandler &);
+  RT_API_ATTRS void BeginVariableFormattedInputRecord(IoErrorHandler &);
+  RT_API_ATTRS void BackspaceFixedRecord(IoErrorHandler &);
+  RT_API_ATTRS void BackspaceVariableUnformattedRecord(IoErrorHandler &);
+  RT_API_ATTRS void BackspaceVariableFormattedRecord(IoErrorHandler &);
+  RT_API_ATTRS bool SetVariableFormattedRecordLength();
+  RT_API_ATTRS void DoImpliedEndfile(IoErrorHandler &);
+  RT_API_ATTRS void DoEndfile(IoErrorHandler &);
+  RT_API_ATTRS void CommitWrites();
+  RT_API_ATTRS bool CheckDirectAccess(IoErrorHandler &);
+  RT_API_ATTRS void HitEndOnRead(IoErrorHandler &);
+  RT_API_ATTRS std::int32_t ReadHeaderOrFooter(std::int64_t frameOffset);
 
   Lock lock_;
 
@@ -238,23 +256,28 @@ private:
 // be a child I/O statement.
 class ChildIo {
 public:
-  ChildIo(IoStatementState &parent, OwningPtr<ChildIo> &&previous)
+  RT_API_ATTRS ChildIo(IoStatementState &parent, OwningPtr<ChildIo> &&previous)
       : parent_{parent}, previous_{std::move(previous)} {}
 
-  IoStatementState &parent() const { return parent_; }
+  RT_API_ATTRS IoStatementState &parent() const { return parent_; }
 
-  void EndIoStatement();
+  RT_API_ATTRS void EndIoStatement();
 
   template <typename A, typename... X>
-  IoStatementState &BeginIoStatement(X &&...xs) {
+  RT_API_ATTRS IoStatementState &BeginIoStatement(X &&...xs) {
+    RT_DIAG_PUSH
+    RT_DIAG_DISABLE_CALL_HOST_FROM_DEVICE_WARN
     A &state{u_.emplace<A>(std::forward<X>(xs)...)};
+    RT_DIAG_POP
     io_.emplace(state);
     return *io_;
   }
 
-  OwningPtr<ChildIo> AcquirePrevious() { return std::move(previous_); }
+  RT_API_ATTRS OwningPtr<ChildIo> AcquirePrevious() {
+    return std::move(previous_);
+  }
 
-  Iostat CheckFormattingAndDirection(bool unformatted, Direction);
+  RT_API_ATTRS Iostat CheckFormattingAndDirection(bool unformatted, Direction);
 
 private:
   IoStatementState &parent_;
diff --git a/flang/runtime/utf.cpp b/flang/runtime/utf.cpp
index e9ccc2c04b6b..9945dc6509ec 100644
--- a/flang/runtime/utf.cpp
+++ b/flang/runtime/utf.cpp
@@ -11,7 +11,8 @@
 namespace Fortran::runtime {
 
 // clang-format off
-const std::uint8_t UTF8FirstByteTable[256]{
+RT_OFFLOAD_VAR_GROUP_BEGIN
+const RT_CONST_VAR_ATTRS std::uint8_t UTF8FirstByteTable[256]{
   /* 00 - 7F:  7 bit payload in single byte */
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
@@ -37,8 +38,10 @@ const std::uint8_t UTF8FirstByteTable[256]{
   /* FE:      32 bit payload */ 7,
   /* FF:      invalid */ 0
 };
+RT_OFFLOAD_VAR_GROUP_END
 // clang-format on
 
+RT_OFFLOAD_API_GROUP_BEGIN
 // Non-minimal encodings are accepted.
 Fortran::common::optional<char32_t> DecodeUTF8(const char *p0) {
   const std::uint8_t *p{reinterpret_cast<const std::uint8_t *>(p0)};
@@ -107,5 +110,6 @@ std::size_t EncodeUTF8(char *p0, char32_t ucs) {
     return 7;
   }
 }
+RT_OFFLOAD_API_GROUP_END
 
 } // namespace Fortran::runtime
diff --git a/flang/runtime/utf.h b/flang/runtime/utf.h
index 2b4e4f9a1887..29670d54b3eb 100644
--- a/flang/runtime/utf.h
+++ b/flang/runtime/utf.h
@@ -49,20 +49,22 @@ namespace Fortran::runtime {
 
 // Derive the length of a UTF-8 character encoding from its first byte.
 // A zero result signifies an invalid encoding.
-extern const std::uint8_t UTF8FirstByteTable[256];
-static inline std::size_t MeasureUTF8Bytes(char first) {
+RT_OFFLOAD_VAR_GROUP_BEGIN
+extern const RT_CONST_VAR_ATTRS std::uint8_t UTF8FirstByteTable[256];
+static constexpr std::size_t maxUTF8Bytes{7};
+RT_OFFLOAD_VAR_GROUP_END
+
+static inline RT_API_ATTRS std::size_t MeasureUTF8Bytes(char first) {
   return UTF8FirstByteTable[static_cast<std::uint8_t>(first)];
 }
 
-static constexpr std::size_t maxUTF8Bytes{7};
-
 // Ensure that all bytes are present in sequence in the input buffer
 // before calling; use MeasureUTF8Bytes(first byte) to count them.
-Fortran::common::optional<char32_t> DecodeUTF8(const char *);
+RT_API_ATTRS Fortran::common::optional<char32_t> DecodeUTF8(const char *);
 
 // Ensure that at least maxUTF8Bytes remain in the output
 // buffer before calling.
-std::size_t EncodeUTF8(char *, char32_t);
+RT_API_ATTRS std::size_t EncodeUTF8(char *, char32_t);
 
 } // namespace Fortran::runtime
 #endif // FORTRAN_RUNTIME_UTF_H_
-- 
GitLab


From f1f06f31b89683e76b87c7665c3318df37ebb6c7 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 25 Mar 2024 16:02:34 -0700
Subject: [PATCH 219/404] [ELF] Move parse files from Driver.cpp to
 InputFiles.cpp. NFC

Fixes: 36146d2b6be53e5e98dee3c1fce8699db9615728

When `doParseFile template defintion` in InputFiles.cpp is optimized
out, we will get a link failure. Actually, we can move the file parsing
loop from Driver.too to InputFiles.cpp and merge it with
parseArmCMSEImportLib.
---
 lld/ELF/Driver.cpp     | 14 +-------------
 lld/ELF/InputFiles.cpp | 22 +++++++++++++++++-----
 lld/ELF/InputFiles.h   |  5 ++---
 3 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 7257ebd0fac9..65d0fa9c4a78 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -2725,19 +2725,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   for (StringRef name : config->undefined)
     addUnusedUndefined(name)->referenced = true;
 
-  // Add all files to the symbol table. This will add almost all
-  // symbols that we need to the symbol table. This process might
-  // add files to the link, via autolinking, these files are always
-  // appended to the Files vector.
-  {
-    llvm::TimeTraceScope timeScope("Parse input files");
-    for (size_t i = 0; i < files.size(); ++i) {
-      llvm::TimeTraceScope timeScope("Parse input files", files[i]->getName());
-      doParseFile<ELFT>(files[i]);
-    }
-    if (armCmseImpLib)
-      parseArmCMSEImportLib(*armCmseImpLib);
-  }
+  parseFiles(files, armCmseImpLib);
 
   // Now that we have every file, we can decide if we will need a
   // dynamic symbol table.
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index 1104433a5047..42761b6e1209 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -288,7 +288,7 @@ static bool isCompatible(InputFile *file) {
   return false;
 }
 
-template <class ELFT> void elf::doParseFile(InputFile *file) {
+template <class ELFT> static void doParseFile(InputFile *file) {
   if (!isCompatible(file))
     return;
 
@@ -330,12 +330,24 @@ extern template void ObjFile<ELF32BE>::importCmseSymbols();
 extern template void ObjFile<ELF64LE>::importCmseSymbols();
 extern template void ObjFile<ELF64BE>::importCmseSymbols();
 
-template <class ELFT> static void doParseArmCMSEImportLib(InputFile &file) {
-  cast<ObjFile<ELFT>>(file).importCmseSymbols();
+template <class ELFT>
+static void doParseFiles(const std::vector<InputFile *> &files,
+                         InputFile *armCmseImpLib) {
+  // Add all files to the symbol table. This will add almost all symbols that we
+  // need to the symbol table. This process might add files to the link due to
+  // addDependentLibrary.
+  for (size_t i = 0; i < files.size(); ++i) {
+    llvm::TimeTraceScope timeScope("Parse input files", files[i]->getName());
+    doParseFile<ELFT>(files[i]);
+  }
+  if (armCmseImpLib)
+    cast<ObjFile<ELFT>>(*armCmseImpLib).importCmseSymbols();
 }
 
-void elf::parseArmCMSEImportLib(InputFile &file) {
-  invokeELFT(doParseArmCMSEImportLib, file);
+void elf::parseFiles(const std::vector<InputFile *> &files,
+                     InputFile *armCmseImpLib) {
+  llvm::TimeTraceScope timeScope("Parse input files");
+  invokeELFT(doParseFiles, files, armCmseImpLib);
 }
 
 // Concatenates arguments to construct a string representing an error location.
diff --git a/lld/ELF/InputFiles.h b/lld/ELF/InputFiles.h
index 7d5dd3428275..95197599a2e1 100644
--- a/lld/ELF/InputFiles.h
+++ b/lld/ELF/InputFiles.h
@@ -46,10 +46,9 @@ extern std::unique_ptr<llvm::TarWriter> tar;
 std::optional<MemoryBufferRef> readFile(StringRef path);
 
 // Add symbols in File to the symbol table.
-template <class ELFT> void doParseFile(InputFile *file);
 void parseFile(InputFile *file);
-
-void parseArmCMSEImportLib(InputFile &file);
+void parseFiles(const std::vector<InputFile *> &files,
+                InputFile *armCmseImpLib);
 
 // The root class of input files.
 class InputFile {
-- 
GitLab


From b6dfaf4c291ee186481f6c1dcab03874d931c307 Mon Sep 17 00:00:00 2001
From: jimingham <jingham@apple.com>
Date: Mon, 25 Mar 2024 16:06:51 -0700
Subject: [PATCH 220/404] Make the correct (5 argument) form of the command
 definition be the primary one suggested in the docs (#86593)

This has been available for years now, so it should be safe to always
use it.
---
 lldb/docs/use/python-reference.rst | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/lldb/docs/use/python-reference.rst b/lldb/docs/use/python-reference.rst
index e5195a2471d9..795e38fab379 100644
--- a/lldb/docs/use/python-reference.rst
+++ b/lldb/docs/use/python-reference.rst
@@ -491,14 +491,17 @@ which will work like all the natively defined lldb commands. This provides a
 very flexible and easy way to extend LLDB to meet your debugging requirements.
 
 To write a python function that implements a new LLDB command define the
-function to take four arguments as follows:
+function to take five arguments as follows:
 
 ::
 
-  def command_function(debugger, command, result, internal_dict):
+  def command_function(debugger, command, exe_ctx, result, internal_dict):
       # Your code goes here
 
-Optionally, you can also provide a Python docstring, and LLDB will use it when providing help for your command, as in:
+The meaning of the arguments is given in the table below.
+
+If you provide a Python docstring in your command function LLDB will use it
+when providing "long help" for your command, as in:
 
 ::
 
@@ -506,19 +509,24 @@ Optionally, you can also provide a Python docstring, and LLDB will use it when p
       """This command takes a lot of options and does many fancy things"""
       # Your code goes here
 
-Since lldb 3.5.2, LLDB Python commands can also take an SBExecutionContext as an
-argument. This is useful in cases where the command's notion of where to act is
-independent of the currently-selected entities in the debugger.
+though providing help can also be done programmatically (see below).
 
-This feature is enabled if the command-implementing function can be recognized
-as taking 5 arguments, or a variable number of arguments, and it alters the
-signature as such:
+Prior to lldb 3.5.2 (April 2015), LLDB Python command definitions didn't take the SBExecutionContext
+argument. So you may still see commands where the command definition is:
 
 ::
 
-  def command_function(debugger, command, exe_ctx, result, internal_dict):
+  def command_function(debugger, command, result, internal_dict):
       # Your code goes here
 
+Using this form is strongly discouraged because it can only operate on the "currently selected"
+target, process, thread, frame.  The command will behave as expected when run
+directly on the command line.  But if the command is used in a stop-hook, breakpoint
+callback, etc. where the response to the callback determines whether we will select
+this or that particular process/frame/thread, the global "currently selected"
+entity is not necessarily the one the callback is meant to handle.  In that case, this
+command definition form can't do the right thing.
+
 +-------------------+--------------------------------+----------------------------------------------------------------------------------------------------------------------------------+
 | Argument          | Type                           | Description                                                                                                                      |
 +-------------------+--------------------------------+----------------------------------------------------------------------------------------------------------------------------------+
-- 
GitLab


From ebb326a51fec37b5a47e5702e8ea157cd4f835cd Mon Sep 17 00:00:00 2001
From: Parth Arora <quic_partaror@quicinc.com>
Date: Mon, 25 Mar 2024 16:11:21 -0700
Subject: [PATCH 221/404] [ELF] Fix unnecessary inclusion of unreferenced
 provide symbols

Previously, linker was unnecessarily including a PROVIDE symbol which
was referenced by another unused PROVIDE symbol. For example, if a
linker script contained the below code and 'not_used_sym' provide symbol
is not included, then linker was still unnecessarily including 'foo' PROVIDE
symbol because it was referenced by 'not_used_sym'. This commit fixes
this behavior.

PROVIDE(not_used_sym = foo)
PROVIDE(foo = 0x1000)

This commit fixes this behavior by using dfs-like algorithm to find
all the symbols referenced in provide expressions of included provide
symbols.

This commit also fixes the issue of unused section not being garbage-collected
if a symbol of the section is referenced by an unused PROVIDE symbol.

Closes #74771
Closes #84730

Co-authored-by: Fangrui Song <i@maskray.me>
---
 lld/ELF/Driver.cpp                           | 22 ++-----
 lld/ELF/LinkerScript.cpp                     | 48 +++++++++++++---
 lld/ELF/LinkerScript.h                       | 21 +++++++
 lld/ELF/ScriptParser.cpp                     | 13 ++++-
 lld/ELF/SymbolTable.cpp                      |  4 ++
 lld/ELF/SymbolTable.h                        |  3 +
 lld/test/ELF/gc-sections-with-provide.s      | 60 ++++++++++++++++++++
 lld/test/ELF/linkerscript/symbolreferenced.s | 51 ++++++++++++++---
 8 files changed, 188 insertions(+), 34 deletions(-)
 create mode 100644 lld/test/ELF/gc-sections-with-provide.s

diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 65d0fa9c4a78..048ef0a60618 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -2393,12 +2393,6 @@ static void readSymbolPartitionSection(InputSectionBase *s) {
   sym->partition = newPart.getNumber();
 }
 
-static Symbol *addUnusedUndefined(StringRef name,
-                                  uint8_t binding = STB_GLOBAL) {
-  return symtab.addSymbol(
-      Undefined{ctx.internalFile, name, binding, STV_DEFAULT, 0});
-}
-
 static void markBuffersAsDontNeed(bool skipLinkedOutput) {
   // With --thinlto-index-only, all buffers are nearly unused from now on
   // (except symbol/section names used by infrequent passes). Mark input file
@@ -2485,15 +2479,15 @@ static std::vector<WrappedSymbol> addWrappedSymbols(opt::InputArgList &args) {
       continue;
 
     Symbol *wrap =
-        addUnusedUndefined(saver().save("__wrap_" + name), sym->binding);
+        symtab.addUnusedUndefined(saver().save("__wrap_" + name), sym->binding);
 
     // If __real_ is referenced, pull in the symbol if it is lazy. Do this after
     // processing __wrap_ as that may have referenced __real_.
     StringRef realName = saver().save("__real_" + name);
     if (symtab.find(realName))
-      addUnusedUndefined(name, sym->binding);
+      symtab.addUnusedUndefined(name, sym->binding);
 
-    Symbol *real = addUnusedUndefined(realName);
+    Symbol *real = symtab.addUnusedUndefined(realName);
     v.push_back({sym, real, wrap});
 
     // We want to tell LTO not to inline symbols to be overwritten
@@ -2723,7 +2717,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   // Handle -u/--undefined before input files. If both a.a and b.so define foo,
   // -u foo a.a b.so will extract a.a.
   for (StringRef name : config->undefined)
-    addUnusedUndefined(name)->referenced = true;
+    symtab.addUnusedUndefined(name)->referenced = true;
 
   parseFiles(files, armCmseImpLib);
 
@@ -2736,13 +2730,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   config->hasDynSymTab =
       !ctx.sharedFiles.empty() || config->isPic || config->exportDynamic;
 
-  // Some symbols (such as __ehdr_start) are defined lazily only when there
-  // are undefined symbols for them, so we add these to trigger that logic.
-  for (StringRef name : script->referencedSymbols) {
-    Symbol *sym = addUnusedUndefined(name);
-    sym->isUsedInRegularObj = true;
-    sym->referenced = true;
-  }
+  script->addScriptReferencedSymbolsToSymTable();
 
   // Prevent LTO from removing any definition referenced by -u.
   for (StringRef name : config->undefined)
diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp
index 3af09a32b651..f815b3ac6fee 100644
--- a/lld/ELF/LinkerScript.cpp
+++ b/lld/ELF/LinkerScript.cpp
@@ -198,15 +198,7 @@ static bool shouldDefineSym(SymbolAssignment *cmd) {
   if (cmd->name == ".")
     return false;
 
-  if (!cmd->provide)
-    return true;
-
-  // If a symbol was in PROVIDE(), we need to define it only
-  // when it is a referenced undefined symbol.
-  Symbol *b = symtab.find(cmd->name);
-  if (b && !b->isDefined() && !b->isCommon())
-    return true;
-  return false;
+  return !cmd->provide || LinkerScript::shouldAddProvideSym(cmd->name);
 }
 
 // Called by processSymbolAssignments() to assign definitions to
@@ -1517,3 +1509,41 @@ void LinkerScript::checkFinalScriptConditions() const {
       checkMemoryRegion(lmaRegion, sec, sec->getLMA());
   }
 }
+
+void LinkerScript::addScriptReferencedSymbolsToSymTable() {
+  // Some symbols (such as __ehdr_start) are defined lazily only when there
+  // are undefined symbols for them, so we add these to trigger that logic.
+  auto reference = [](StringRef name) {
+    Symbol *sym = symtab.addUnusedUndefined(name);
+    sym->isUsedInRegularObj = true;
+    sym->referenced = true;
+  };
+  for (StringRef name : referencedSymbols)
+    reference(name);
+
+  // Keeps track of references from which PROVIDE symbols have been added to the
+  // symbol table.
+  DenseSet<StringRef> added;
+  SmallVector<const SmallVector<StringRef, 0> *, 0> symRefsVec;
+  for (const auto &[name, symRefs] : provideMap)
+    if (LinkerScript::shouldAddProvideSym(name) && added.insert(name).second)
+      symRefsVec.push_back(&symRefs);
+  while (symRefsVec.size()) {
+    for (StringRef name : *symRefsVec.pop_back_val()) {
+      reference(name);
+      // Prevent the symbol from being discarded by --gc-sections.
+      script->referencedSymbols.push_back(name);
+      auto it = script->provideMap.find(name);
+      if (it != script->provideMap.end() &&
+          LinkerScript::shouldAddProvideSym(name) &&
+          added.insert(name).second) {
+        symRefsVec.push_back(&it->second);
+      }
+    }
+  }
+}
+
+bool LinkerScript::shouldAddProvideSym(StringRef symName) {
+  Symbol *sym = symtab.find(symName);
+  return sym && !sym->isDefined() && !sym->isCommon();
+}
diff --git a/lld/ELF/LinkerScript.h b/lld/ELF/LinkerScript.h
index 18eaf58b785e..fa7c6eb9c0d8 100644
--- a/lld/ELF/LinkerScript.h
+++ b/lld/ELF/LinkerScript.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Compiler.h"
 #include <cstddef>
@@ -348,6 +349,18 @@ public:
   // Check backward location counter assignment and memory region/LMA overflows.
   void checkFinalScriptConditions() const;
 
+  // Add symbols that are referenced in the linker script to the symbol table.
+  // Symbols referenced in a PROVIDE command are only added to the symbol table
+  // if the PROVIDE command actually provides the symbol.
+  // It also adds the symbols referenced by the used PROVIDE symbols to the
+  // linker script referenced symbols list.
+  void addScriptReferencedSymbolsToSymTable();
+
+  // Returns true if the PROVIDE symbol should be added to the link.
+  // A PROVIDE symbol is added to the link only if it satisfies an
+  // undefined reference.
+  static bool shouldAddProvideSym(StringRef symName);
+
   // SECTIONS command list.
   SmallVector<SectionCommand *, 0> sectionCommands;
 
@@ -379,6 +392,14 @@ public:
 
   // Sections that will be warned/errored by --orphan-handling.
   SmallVector<const InputSectionBase *, 0> orphanSections;
+
+  // Stores the mapping: PROVIDE symbol -> symbols referred in the PROVIDE
+  // expression. For example, if the PROVIDE command is:
+  //
+  // PROVIDE(v = a + b + c);
+  //
+  // then provideMap should contain the mapping: 'v' -> ['a', 'b', 'c']
+  llvm::MapVector<StringRef, SmallVector<StringRef, 0>> provideMap;
 };
 
 LLVM_LIBRARY_VISIBILITY extern std::unique_ptr<LinkerScript> script;
diff --git a/lld/ELF/ScriptParser.cpp b/lld/ELF/ScriptParser.cpp
index 3bb1de99480f..f90ce6fa7407 100644
--- a/lld/ELF/ScriptParser.cpp
+++ b/lld/ELF/ScriptParser.cpp
@@ -36,6 +36,7 @@
 #include "llvm/Support/TimeProfiler.h"
 #include <cassert>
 #include <limits>
+#include <optional>
 #include <vector>
 
 using namespace llvm;
@@ -138,6 +139,10 @@ private:
 
   // A set to detect an INCLUDE() cycle.
   StringSet<> seen;
+
+  // If we are currently parsing a PROVIDE|PROVIDE_HIDDEN command,
+  // then this member is set to the PROVIDE symbol name.
+  std::optional<llvm::StringRef> activeProvideSym;
 };
 } // namespace
 
@@ -1055,6 +1060,9 @@ SymbolAssignment *ScriptParser::readProvideHidden(bool provide, bool hidden) {
       ;
     return nullptr;
   }
+  llvm::SaveAndRestore saveActiveProvideSym(activeProvideSym);
+  if (provide)
+    activeProvideSym = name;
   SymbolAssignment *cmd = readSymbolAssignment(name);
   cmd->provide = provide;
   cmd->hidden = hidden;
@@ -1570,7 +1578,10 @@ Expr ScriptParser::readPrimary() {
     tok = unquote(tok);
   else if (!isValidSymbolName(tok))
     setError("malformed number: " + tok);
-  script->referencedSymbols.push_back(tok);
+  if (activeProvideSym)
+    script->provideMap[*activeProvideSym].push_back(tok);
+  else
+    script->referencedSymbols.push_back(tok);
   return [=] { return script->getSymbolValue(tok, location); };
 }
 
diff --git a/lld/ELF/SymbolTable.cpp b/lld/ELF/SymbolTable.cpp
index b3d97e4de779..258a78ab40bb 100644
--- a/lld/ELF/SymbolTable.cpp
+++ b/lld/ELF/SymbolTable.cpp
@@ -333,3 +333,7 @@ void SymbolTable::scanVersionScript() {
   // --dynamic-list.
   handleDynamicList();
 }
+
+Symbol *SymbolTable::addUnusedUndefined(StringRef name, uint8_t binding) {
+  return addSymbol(Undefined{ctx.internalFile, name, binding, STV_DEFAULT, 0});
+}
diff --git a/lld/ELF/SymbolTable.h b/lld/ELF/SymbolTable.h
index 37e31d237296..269f7f284bc7 100644
--- a/lld/ELF/SymbolTable.h
+++ b/lld/ELF/SymbolTable.h
@@ -57,6 +57,9 @@ public:
 
   void handleDynamicList();
 
+  Symbol *addUnusedUndefined(StringRef name,
+                             uint8_t binding = llvm::ELF::STB_GLOBAL);
+
   // Set of .so files to not link the same shared object file more than once.
   llvm::DenseMap<llvm::CachedHashStringRef, SharedFile *> soNames;
 
diff --git a/lld/test/ELF/gc-sections-with-provide.s b/lld/test/ELF/gc-sections-with-provide.s
new file mode 100644
index 000000000000..3e5b1b1efe6c
--- /dev/null
+++ b/lld/test/ELF/gc-sections-with-provide.s
@@ -0,0 +1,60 @@
+# REQUIRES: x86
+
+# This test verifies that garbage-collection is correctly garbage collecting
+# unused sections when the symbol of the unused section is only referred by
+# an unused PROVIDE symbol.
+
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64 a.s -o a.o
+# RUN: ld.lld -o a_nogc a.o -T script.t
+# RUN: llvm-nm a_nogc | FileCheck -check-prefix=NOGC %s
+# RUN: ld.lld -o a_gc a.o --gc-sections --print-gc-sections -T script.t | FileCheck --check-prefix=GC_LINK %s
+# RUN: llvm-nm a_gc | FileCheck -check-prefix=GC %s
+
+NOGC-NOT: another_unused
+NOGC: another_used
+NOGC: bar
+NOGC: baz
+NOGC: baz_ref
+NOGC: foo
+NOGC-NOT: unused
+NOGC: used
+
+GC_LINK: removing unused section a.o:(.text.bar)
+
+GC-NOT: another_unused
+GC: another_used
+GC-NOT: bar
+GC: baz
+GC: baz_ref
+GC: foo
+GC-NOT: unused
+GC: used
+
+#--- a.s
+.global _start
+_start:
+ call foo
+ call used
+
+.section .text.foo,"ax",@progbits
+foo:
+ nop
+
+.section .text.bar,"ax",@progbits
+.global bar
+bar:
+ nop
+
+.section .text.baz,"ax",@progbits
+.global baz
+baz:
+ nop
+
+
+#--- script.t
+PROVIDE(unused = bar + used);
+PROVIDE(used = another_used);
+PROVIDE(baz_ref = baz);
+PROVIDE(another_used = baz_ref);
+PROVIDE(another_unused = unused + bar + 0x1);
diff --git a/lld/test/ELF/linkerscript/symbolreferenced.s b/lld/test/ELF/linkerscript/symbolreferenced.s
index ba7a7721ea96..6f583d20e276 100644
--- a/lld/test/ELF/linkerscript/symbolreferenced.s
+++ b/lld/test/ELF/linkerscript/symbolreferenced.s
@@ -21,11 +21,31 @@
 # RUN: ld.lld -o chain -T chain.t a.o
 # RUN: llvm-nm chain | FileCheck %s
 
-# CHECK:      0000000000001000 a f1
-# CHECK-NEXT: 0000000000001000 A f2
-# CHECK-NEXT: 0000000000001000 a g1
-# CHECK-NEXT: 0000000000001000 A g2
-# CHECK-NEXT: 0000000000001000 A newsym
+# CHECK-NOT: another_unused
+# CHECK:      0000000000007000 a f1
+# CHECK-NEXT: 0000000000007000 A f2
+# CHECK-NEXT: 0000000000007000 A f3
+# CHECK-NEXT: 0000000000007000 A f4
+# CHECK-NEXT: 0000000000006000 A f5
+# CHECK-NEXT: 0000000000003000 A f6
+# CHECK-NEXT: 0000000000001000 A f7
+# CHECK-NOT: g1
+# CHECK-NOT: g2
+# CHECK-NEXT: 0000000000007500 A newsym
+# CHECK: 0000000000002000 A u
+# CHECK-NOT: unused
+# CHECK-NEXT: 0000000000002000 A v
+# CHECK-NEXT: 0000000000002000 A w
+
+
+# RUN: ld.lld -o chain_with_cycle -T chain_with_cycle.t a.o
+# RUN: llvm-nm chain_with_cycle | FileCheck %s --check-prefix=CHAIN_WITH_CYCLE
+
+# CHAIN_WITH_CYCLE: 000 A f1
+# CHAIN_WITH_CYCLE: 000 A f2
+# CHAIN_WITH_CYCLE: 000 A f3
+# CHAIN_WITH_CYCLE: 000 A f4
+# CHAIN_WITH_CYCLE: 000 A newsym
 
 # RUN: not ld.lld -T chain2.t a.o 2>&1 | FileCheck %s --check-prefix=ERR --implicit-check-not=error:
 # ERR-COUNT-3: error: chain2.t:1: symbol not found: undef
@@ -40,13 +60,30 @@ patatino:
   movl newsym, %eax
 
 #--- chain.t
-PROVIDE(f2 = 0x1000);
+PROVIDE(f7 = 0x1000);
+PROVIDE(f5 = f6 + 0x3000);
+PROVIDE(f6 = f7 + 0x2000);
+PROVIDE(f4 = f5 + 0x1000);
+PROVIDE(f3 = f4);
+PROVIDE(f2 = f3);
 PROVIDE_HIDDEN(f1 = f2);
-PROVIDE(newsym = f1);
+PROVIDE(newsym = f1 + 0x500);
+
+u = v;
+PROVIDE(w = 0x2000);
+PROVIDE(v = w);
 
 PROVIDE(g2 = 0x1000);
 PROVIDE_HIDDEN(g1 = g2);
 PROVIDE(unused = g1);
+PROVIDE_HIDDEN(another_unused = g1);
+
+#--- chain_with_cycle.t
+PROVIDE(f1 = f2 + f3);
+PROVIDE(f2 = f3 + f4);
+PROVIDE(f3 = f4);
+PROVIDE(f4 = f1);
+PROVIDE(newsym = f1);
 
 #--- chain2.t
 PROVIDE(f2 = undef);
-- 
GitLab


From 4fc8df9c6afa5ddcaaab91aad87bf713c592545d Mon Sep 17 00:00:00 2001
From: Ed Maste <emaste@freebsd.org>
Date: Mon, 25 Mar 2024 19:17:30 -0400
Subject: [PATCH 222/404] [libc++] Switch FreeBSD CI job to Clang 17 (#86320)

libc++ will drop support for Clang 16 before long.
---
 libcxx/utils/ci/buildkite-pipeline.yml | 4 ++--
 libcxx/utils/libcxx/test/features.py   | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/libcxx/utils/ci/buildkite-pipeline.yml b/libcxx/utils/ci/buildkite-pipeline.yml
index e42262620d5f..31e794e67d33 100644
--- a/libcxx/utils/ci/buildkite-pipeline.yml
+++ b/libcxx/utils/ci/buildkite-pipeline.yml
@@ -209,8 +209,8 @@ steps:
   - label: FreeBSD 13 amd64
     command: libcxx/utils/ci/run-buildbot generic-cxx23
     env:
-      CC: clang16
-      CXX: clang++16
+      CC: clang17
+      CXX: clang++17
     agents:
       queue: libcxx-builders
       os: freebsd
diff --git a/libcxx/utils/libcxx/test/features.py b/libcxx/utils/libcxx/test/features.py
index 307d35349f3a..0793c34fd7f0 100644
--- a/libcxx/utils/libcxx/test/features.py
+++ b/libcxx/utils/libcxx/test/features.py
@@ -280,6 +280,7 @@ DEFAULT_FEATURES = [
         # This is not allowed per C11 7.1.2 Standard headers/6
         #  Any declaration of a library function shall have external linkage.
         when=lambda cfg: "__ANDROID__" in compilerMacros(cfg)
+        or "__FreeBSD__" in compilerMacros(cfg)
         or "_WIN32" in compilerMacros(cfg)
         or platform.system().lower().startswith("aix")
         # Avoid building on platforms that don't support modules properly.
-- 
GitLab


From 213eda157a5ffb0985d7da56a9c78aa0c345d84d Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Mon, 25 Mar 2024 16:23:21 -0700
Subject: [PATCH 223/404] [BOLT] Add CallSiteInfo entries in YAMLBAT (#76896)

Attach call counters to YAML profile, covering inter-function control
flow.

Depends on: https://github.com/llvm/llvm-project/pull/86218

Test Plan:
Updated bolt/test/X86/bolt-address-translation-yaml.test
---
 .../bolt/Profile/BoltAddressTranslation.h     |  5 ++
 bolt/include/bolt/Profile/DataAggregator.h    |  3 +
 bolt/lib/Profile/BoltAddressTranslation.cpp   | 20 +++++
 bolt/lib/Profile/DataAggregator.cpp           | 86 ++++++++++++++++++-
 .../X86/bolt-address-translation-yaml.test    | 12 +++
 5 files changed, 125 insertions(+), 1 deletion(-)

diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h
index bade0b57bf5c..caf907cc43da 100644
--- a/bolt/include/bolt/Profile/BoltAddressTranslation.h
+++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h
@@ -118,6 +118,11 @@ public:
   /// True if a given \p Address is a function with translation table entry.
   bool isBATFunction(uint64_t Address) const { return Maps.count(Address); }
 
+  /// Returns branch offsets grouped by containing basic block in a given
+  /// function.
+  std::unordered_map<uint32_t, std::vector<uint32_t>>
+  getBFBranches(uint64_t FuncOutputAddress) const;
+
 private:
   /// Helper to update \p Map by inserting one or more BAT entries reflecting
   /// \p BB for function located at \p FuncAddress. At least one entry will be
diff --git a/bolt/include/bolt/Profile/DataAggregator.h b/bolt/include/bolt/Profile/DataAggregator.h
index f7840b49199f..4fbe524b1c38 100644
--- a/bolt/include/bolt/Profile/DataAggregator.h
+++ b/bolt/include/bolt/Profile/DataAggregator.h
@@ -467,6 +467,9 @@ private:
   std::error_code writeBATYAML(BinaryContext &BC,
                                StringRef OutputFilename) const;
 
+  /// Fixup profile collected on BOLTed binary, namely handle split functions.
+  void fixupBATProfile(BinaryContext &BC);
+
   /// Filter out binaries based on PID
   void filterBinaryMMapInfo();
 
diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp
index aa4cde27e75c..bcd4a457ce3b 100644
--- a/bolt/lib/Profile/BoltAddressTranslation.cpp
+++ b/bolt/lib/Profile/BoltAddressTranslation.cpp
@@ -577,5 +577,25 @@ void BoltAddressTranslation::saveMetadata(BinaryContext &BC) {
   }
 }
 
+std::unordered_map<uint32_t, std::vector<uint32_t>>
+BoltAddressTranslation::getBFBranches(uint64_t OutputAddress) const {
+  std::unordered_map<uint32_t, std::vector<uint32_t>> Branches;
+  auto FuncIt = Maps.find(OutputAddress);
+  assert(FuncIt != Maps.end());
+  std::vector<uint32_t> InputOffsets;
+  for (const auto &KV : FuncIt->second)
+    InputOffsets.emplace_back(KV.second);
+  // Sort with LSB BRANCHENTRY bit.
+  llvm::sort(InputOffsets);
+  uint32_t BBOffset{0};
+  for (uint32_t InOffset : InputOffsets) {
+    if (InOffset & BRANCHENTRY)
+      Branches[BBOffset].push_back(InOffset >> 1);
+    else
+      BBOffset = InOffset >> 1;
+  }
+  return Branches;
+}
+
 } // namespace bolt
 } // namespace llvm
diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index b0f1cccc3256..05099aa25ce2 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -604,6 +604,8 @@ Error DataAggregator::readProfile(BinaryContext &BC) {
     // BAT YAML is handled by DataAggregator since normal YAML output requires
     // CFG which is not available in BAT mode.
     if (usesBAT()) {
+      // Postprocess split function profile for BAT
+      fixupBATProfile(BC);
       if (opts::ProfileFormat == opts::ProfileFormatKind::PF_YAML)
         if (std::error_code EC = writeBATYAML(BC, opts::OutputFilename))
           report_error("cannot create output data file", EC);
@@ -2271,6 +2273,29 @@ DataAggregator::writeAggregatedFile(StringRef OutputFilename) const {
   return std::error_code();
 }
 
+void DataAggregator::fixupBATProfile(BinaryContext &BC) {
+  for (auto &[FuncName, Branches] : NamesToBranches) {
+    BinaryData *BD = BC.getBinaryDataByName(FuncName);
+    assert(BD);
+    uint64_t FuncAddress = BD->getAddress();
+    if (!BAT->isBATFunction(FuncAddress))
+      continue;
+    // Filter out cold fragments
+    if (!BD->getSectionName().equals(BC.getMainCodeSectionName()))
+      continue;
+    // Convert inter-branches between hot and cold fragments into
+    // intra-branches.
+    for (auto &[OffsetFrom, CallToMap] : Branches.InterIndex) {
+      for (auto &[CallToLoc, CallToIdx] : CallToMap) {
+        if (CallToLoc.Name != FuncName)
+          continue;
+        Branches.IntraIndex[OffsetFrom][CallToLoc.Offset] = CallToIdx;
+        Branches.InterIndex[OffsetFrom].erase(CallToLoc);
+      }
+    }
+  }
+}
+
 std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
                                              StringRef OutputFilename) const {
   std::error_code EC;
@@ -2343,6 +2368,63 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
         YamlBB.Successors.emplace_back(SI);
       };
 
+      std::unordered_map<uint32_t, std::vector<uint32_t>> BFBranches =
+          BAT->getBFBranches(FuncAddress);
+
+      auto addCallsProfile = [&](yaml::bolt::BinaryBasicBlockProfile &YamlBB,
+                                 uint64_t Offset) {
+        // Iterate over BRANCHENTRY records in the current block
+        for (uint32_t BranchOffset : BFBranches[Offset]) {
+          if (!Branches.InterIndex.contains(BranchOffset))
+            continue;
+          for (const auto &[CallToLoc, CallToIdx] :
+               Branches.InterIndex.at(BranchOffset)) {
+            const llvm::bolt::BranchInfo &BI = Branches.Data.at(CallToIdx);
+            yaml::bolt::CallSiteInfo YamlCSI;
+            YamlCSI.DestId = 0; // designated for unknown functions
+            YamlCSI.EntryDiscriminator = 0;
+            YamlCSI.Count = BI.Branches;
+            YamlCSI.Mispreds = BI.Mispreds;
+            YamlCSI.Offset = BranchOffset - Offset;
+            BinaryData *CallTargetBD = BC.getBinaryDataByName(CallToLoc.Name);
+            if (!CallTargetBD) {
+              YamlBB.CallSites.emplace_back(YamlCSI);
+              continue;
+            }
+            uint64_t CallTargetAddress = CallTargetBD->getAddress();
+            BinaryFunction *CallTargetBF =
+                BC.getBinaryFunctionAtAddress(CallTargetAddress);
+            if (!CallTargetBF) {
+              YamlBB.CallSites.emplace_back(YamlCSI);
+              continue;
+            }
+            // Calls between hot and cold fragments must be handled in
+            // fixupBATProfile.
+            assert(CallTargetBF != BF && "invalid CallTargetBF");
+            YamlCSI.DestId = CallTargetBF->getFunctionNumber();
+            if (CallToLoc.Offset) {
+              if (BAT->isBATFunction(CallTargetAddress)) {
+                LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Unsupported secondary "
+                                     "entry point in BAT function "
+                                  << CallToLoc.Name << '\n');
+              } else if (const BinaryBasicBlock *CallTargetBB =
+                             CallTargetBF->getBasicBlockAtOffset(
+                                 CallToLoc.Offset)) {
+                // Only record true call information, ignoring returns (normally
+                // won't have a target basic block) and jumps to the landing
+                // pads (not an entry point).
+                if (CallTargetBB->isEntryPoint()) {
+                  YamlCSI.EntryDiscriminator =
+                      CallTargetBF->getEntryIDForSymbol(
+                          CallTargetBB->getLabel());
+                }
+              }
+            }
+            YamlBB.CallSites.emplace_back(YamlCSI);
+          }
+        }
+      };
+
       for (const auto &[FromOffset, SuccKV] : Branches.IntraIndex) {
         yaml::bolt::BinaryBasicBlockProfile YamlBB;
         if (!BlockMap.isInputBlock(FromOffset))
@@ -2351,7 +2433,9 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
         YamlBB.Hash = BlockMap.getBBHash(FromOffset);
         for (const auto &[SuccOffset, SuccDataIdx] : SuccKV)
           addSuccProfile(YamlBB, SuccOffset, SuccDataIdx);
-        if (YamlBB.ExecCount || !YamlBB.Successors.empty())
+        addCallsProfile(YamlBB, FromOffset);
+        if (YamlBB.ExecCount || !YamlBB.Successors.empty() ||
+            !YamlBB.CallSites.empty())
           YamlBF.Blocks.emplace_back(YamlBB);
       }
       BP.Functions.emplace_back(YamlBF);
diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test
index 306050a1c97e..7fdf7709a8b9 100644
--- a/bolt/test/X86/bolt-address-translation-yaml.test
+++ b/bolt/test/X86/bolt-address-translation-yaml.test
@@ -36,6 +36,18 @@ YAML-BAT-CHECK-NEXT:   - bid:   0
 YAML-BAT-CHECK-NEXT:     insns: 26
 YAML-BAT-CHECK-NEXT:     hash:  0xA900AE79CFD40000
 YAML-BAT-CHECK-NEXT:     succ:  [ { bid: 3, cnt: 0 }, { bid: 1, cnt: 0 } ]
+# Function covered by BAT with calls
+YAML-BAT-CHECK:      - name:    SolveCubic
+YAML-BAT-CHECK-NEXT:   fid:     [[#]]
+YAML-BAT-CHECK-NEXT:   hash:    0x6AF7E61EA3966722
+YAML-BAT-CHECK-NEXT:   exec:    25
+YAML-BAT-CHECK-NEXT:   nblocks: 15
+YAML-BAT-CHECK-NEXT:   blocks:
+YAML-BAT-CHECK:        - bid:   3
+YAML-BAT-CHECK-NEXT:     insns: [[#]]
+YAML-BAT-CHECK-NEXT:     hash:  0xDDA1DC5F69F900AC
+YAML-BAT-CHECK-NEXT:     calls: [ { off: 0x26, fid: [[#]], cnt: [[#]] } ]
+YAML-BAT-CHECK-NEXT:     succ:  [ { bid: 5, cnt: [[#]] }
 # Function covered by BAT - doesn't have insns in basic block
 YAML-BAT-CHECK:      - name:    usqrt
 YAML-BAT-CHECK-NEXT:   fid:     [[#]]
-- 
GitLab


From 8f86c6bf9524a5f8c9ae26a6599678008d8c9386 Mon Sep 17 00:00:00 2001
From: NAKAMURA Takumi <geek4civic@gmail.com>
Date: Tue, 26 Mar 2024 08:42:47 +0900
Subject: [PATCH 224/404] compiler-rt: Fix variadic macro warnings
 [-Wc++20-extensions]

They began complaining since #84520.
---
 compiler-rt/lib/asan/asan_interceptors.cpp           |  4 ++--
 .../lib/hwasan/hwasan_allocation_functions.cpp       |  2 +-
 compiler-rt/lib/hwasan/hwasan_interceptors.cpp       |  4 ++--
 compiler-rt/lib/msan/msan_interceptors.cpp           |  2 +-
 .../sanitizer_common_interceptors.inc                | 12 ++++++------
 .../lib/sanitizer_common/sanitizer_printf.cpp        |  2 +-
 6 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/compiler-rt/lib/asan/asan_interceptors.cpp b/compiler-rt/lib/asan/asan_interceptors.cpp
index da47317c0e57..6d1360e10497 100644
--- a/compiler-rt/lib/asan/asan_interceptors.cpp
+++ b/compiler-rt/lib/asan/asan_interceptors.cpp
@@ -761,8 +761,8 @@ INTERCEPTOR(int, pthread_atfork, void (*prepare)(), void (*parent)(),
 #endif
 
 #if ASAN_INTERCEPT_VFORK
-DEFINE_REAL(int, vfork)
-DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork)
+DEFINE_REAL(int, vfork,)
+DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork,)
 #endif
 
 // ---------------------- InitializeAsanInterceptors ---------------- {{{1
diff --git a/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp b/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp
index 75d91ed09ce1..9af09e2a4bed 100644
--- a/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp
@@ -184,7 +184,7 @@ INTERCEPTOR_ALIAS(void *, malloc, SIZE_T size);
 INTERCEPTOR_ALIAS(void *, memalign, SIZE_T alignment, SIZE_T size);
 INTERCEPTOR_ALIAS(void *, pvalloc, SIZE_T size);
 INTERCEPTOR_ALIAS(void, cfree, void *ptr);
-INTERCEPTOR_ALIAS(__sanitizer_struct_mallinfo, mallinfo);
+INTERCEPTOR_ALIAS(__sanitizer_struct_mallinfo, mallinfo,);
 INTERCEPTOR_ALIAS(int, mallopt, int cmd, int value);
 INTERCEPTOR_ALIAS(void, malloc_stats, void);
 #  endif
diff --git a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
index d519ac3a459b..08ae435b8214 100644
--- a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
@@ -336,8 +336,8 @@ INTERCEPTOR(int, pthread_timedjoin_np, void *thread, void **ret,
 
 DEFINE_REAL_PTHREAD_FUNCTIONS
 
-DEFINE_REAL(int, vfork)
-DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork)
+DEFINE_REAL(int, vfork,)
+DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork,)
 
 // Get and/or change the set of blocked signals.
 extern "C" int sigprocmask(int __how, const __hw_sigset_t *__restrict __set,
diff --git a/compiler-rt/lib/msan/msan_interceptors.cpp b/compiler-rt/lib/msan/msan_interceptors.cpp
index 6e0b2bf2ef5b..9abf24063325 100644
--- a/compiler-rt/lib/msan/msan_interceptors.cpp
+++ b/compiler-rt/lib/msan/msan_interceptors.cpp
@@ -255,7 +255,7 @@ static NOINLINE void clear_mallinfo(T *sret) {
 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
 // Interceptors use NRVO and assume that sret will be pre-allocated in
 // caller frame.
-INTERCEPTOR(__sanitizer_struct_mallinfo, mallinfo) {
+INTERCEPTOR(__sanitizer_struct_mallinfo, mallinfo,) {
   __sanitizer_struct_mallinfo sret;
   clear_mallinfo(&sret);
   return sret;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
index a1be676730a7..1df61e79f7d8 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -7650,9 +7650,9 @@ static void write_protoent(void *ctx, struct __sanitizer_protoent *p) {
   COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases, pp_size * sizeof(char *));
 }
 
-INTERCEPTOR(struct __sanitizer_protoent *, getprotoent) {
+INTERCEPTOR(struct __sanitizer_protoent *, getprotoent,) {
   void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, getprotoent);
+  COMMON_INTERCEPTOR_ENTER(ctx, getprotoent,);
   struct __sanitizer_protoent *p = REAL(getprotoent)();
   if (p)
     write_protoent(ctx, p);
@@ -7739,9 +7739,9 @@ INTERCEPTOR(int, getprotobynumber_r, int num,
 #endif
 
 #if SANITIZER_INTERCEPT_NETENT
-INTERCEPTOR(struct __sanitizer_netent *, getnetent) {
+INTERCEPTOR(struct __sanitizer_netent *, getnetent,) {
   void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, getnetent);
+  COMMON_INTERCEPTOR_ENTER(ctx, getnetent,);
   struct __sanitizer_netent *n = REAL(getnetent)();
   if (n) {
     COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));
@@ -9903,9 +9903,9 @@ INTERCEPTOR(char *, fdevname_r,  int fd, char *buf, SIZE_T len) {
 #endif
 
 #if SANITIZER_INTERCEPT_GETUSERSHELL
-INTERCEPTOR(char *, getusershell) {
+INTERCEPTOR(char *, getusershell,) {
   void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, getusershell);
+  COMMON_INTERCEPTOR_ENTER(ctx, getusershell,);
   char *res = REAL(getusershell)();
   if (res)
     COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_printf.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_printf.cpp
index 62c1cf4abe42..24511720bd99 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_printf.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_printf.cpp
@@ -54,7 +54,7 @@ static int AppendNumber(char **buff, const char *buff_end, u64 absolute_value,
   uptr num_buffer[kMaxLen];
   int pos = 0;
   do {
-    RAW_CHECK_MSG((uptr)pos < kMaxLen, "AppendNumber buffer overflow");
+    RAW_CHECK_MSG((uptr)pos < kMaxLen, "AppendNumber buffer overflow",);
     num_buffer[pos++] = absolute_value % base;
     absolute_value /= base;
   } while (absolute_value > 0);
-- 
GitLab


From 350bda4419e15e5d68a87667988458546fa2e0c2 Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang@amd.com>
Date: Mon, 25 Mar 2024 16:55:22 -0700
Subject: [PATCH 225/404] AMDGPU: Rename intrinsics and remove f16/bf16
 versions for load transpose (#86313)

Rename the intrinsics to close to the instruction mnemonic names:
Use global_load_tr_b64 and global_load_tr_b128 instead of
global_load_tr.

This patch also removes f16/bf16 versions of builtins/intrinsics. To
simplify the design, we should avoid enumerating all possible types in
implementing builtins. We can always use bitcast.
---
 clang/include/clang/Basic/BuiltinsAMDGPU.def  |   5 -
 clang/lib/CodeGen/CGBuiltin.cpp               |  28 +---
 ...uiltins-amdgcn-global-load-tr-gfx11-err.cl |  16 +-
 ...ins-amdgcn-global-load-tr-gfx12-w32-err.cl |   6 +-
 ...ins-amdgcn-global-load-tr-gfx12-w64-err.cl |   6 +-
 .../builtins-amdgcn-global-load-tr-w32.cl     |  26 +---
 .../builtins-amdgcn-global-load-tr-w64.cl     |  26 +---
 llvm/include/llvm/IR/IntrinsicsAMDGPU.td      |  15 +-
 .../Target/AMDGPU/AMDGPURegisterBankInfo.cpp  |   3 +-
 .../Target/AMDGPU/AMDGPUSearchableTables.td   |   3 +-
 llvm/lib/Target/AMDGPU/FLATInstructions.td    |  12 +-
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |   6 +-
 .../UniformityAnalysis/AMDGPU/intrinsics.ll   |  70 ++-------
 .../AMDGPU/llvm.amdgcn.global.load.tr-w32.ll  | 146 ++++--------------
 .../AMDGPU/llvm.amdgcn.global.load.tr-w64.ll  | 146 ++++--------------
 15 files changed, 104 insertions(+), 410 deletions(-)

diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index 4153b316c22b..c660582cc98e 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -434,13 +434,8 @@ TARGET_BUILTIN(__builtin_amdgcn_s_get_barrier_state, "Uii", "n", "gfx12-insts")
 
 TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b64_v2i32, "V2iV2i*1", "nc", "gfx12-insts,wavefrontsize32")
 TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v8i16, "V8sV8s*1", "nc", "gfx12-insts,wavefrontsize32")
-TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v8f16, "V8hV8h*1", "nc", "gfx12-insts,wavefrontsize32")
-TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v8bf16, "V8yV8y*1", "nc", "gfx12-insts,wavefrontsize32")
-
 TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b64_i32, "ii*1", "nc", "gfx12-insts,wavefrontsize64")
 TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v4i16, "V4sV4s*1", "nc", "gfx12-insts,wavefrontsize64")
-TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v4f16, "V4hV4h*1", "nc", "gfx12-insts,wavefrontsize64")
-TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v4bf16, "V4yV4y*1", "nc", "gfx12-insts,wavefrontsize64")
 
 //===----------------------------------------------------------------------===//
 // WMMA builtins.
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 8f4817258e3b..46a815155e7b 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -18540,51 +18540,35 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
   }
   case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
   case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
-  case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16:
-  case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16:
   case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
-  case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16:
-  case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16:
   case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16: {
 
+    Intrinsic::ID IID;
     llvm::Type *ArgTy;
     switch (BuiltinID) {
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
       ArgTy = llvm::Type::getInt32Ty(getLLVMContext());
+      IID = Intrinsic::amdgcn_global_load_tr_b64;
       break;
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
       ArgTy = llvm::FixedVectorType::get(
           llvm::Type::getInt32Ty(getLLVMContext()), 2);
-      break;
-    case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16:
-      ArgTy = llvm::FixedVectorType::get(
-          llvm::Type::getBFloatTy(getLLVMContext()), 4);
-      break;
-    case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16:
-      ArgTy = llvm::FixedVectorType::get(
-          llvm::Type::getHalfTy(getLLVMContext()), 4);
+      IID = Intrinsic::amdgcn_global_load_tr_b64;
       break;
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
       ArgTy = llvm::FixedVectorType::get(
           llvm::Type::getInt16Ty(getLLVMContext()), 4);
-      break;
-    case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16:
-      ArgTy = llvm::FixedVectorType::get(
-          llvm::Type::getBFloatTy(getLLVMContext()), 8);
-      break;
-    case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16:
-      ArgTy = llvm::FixedVectorType::get(
-          llvm::Type::getHalfTy(getLLVMContext()), 8);
+      IID = Intrinsic::amdgcn_global_load_tr_b128;
       break;
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
       ArgTy = llvm::FixedVectorType::get(
           llvm::Type::getInt16Ty(getLLVMContext()), 8);
+      IID = Intrinsic::amdgcn_global_load_tr_b128;
       break;
     }
 
     llvm::Value *Addr = EmitScalarExpr(E->getArg(0));
-    llvm::Function *F =
-        CGM.getIntrinsic(Intrinsic::amdgcn_global_load_tr, {ArgTy});
+    llvm::Function *F = CGM.getIntrinsic(IID, {ArgTy});
     return Builder.CreateCall(F, {Addr});
   }
   case AMDGPU::BI__builtin_amdgcn_get_fpenv: {
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl
index 4363769b8645..1e78ab283486 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl
@@ -4,24 +4,14 @@
 // REQUIRES: amdgpu-registered-target
 
 typedef int    v2i   __attribute__((ext_vector_type(2)));
-typedef half   v8h   __attribute__((ext_vector_type(8)));
 typedef short  v8s   __attribute__((ext_vector_type(8)));
-typedef __bf16 v8bf16 __attribute__((ext_vector_type(8)));
-
-typedef half   v4h   __attribute__((ext_vector_type(4)));
 typedef short  v4s   __attribute__((ext_vector_type(4)));
-typedef __bf16 v4bf16 __attribute__((ext_vector_type(4)));
 
-void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr, global v8h* v8h_inptr, global v8bf16* v8bf16_inptr,
-                           global int* int_inptr, global v4s* v4s_inptr, global v4h* v4h_inptr, global v4bf16* v4bf16_inptr)
+void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr, global int* int_inptr, global v4s* v4s_inptr)
 {
   v2i out_1 = __builtin_amdgcn_global_load_tr_b64_v2i32(v2i_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_v2i32' needs target feature gfx12-insts,wavefrontsize32}}
   v8s out_2 = __builtin_amdgcn_global_load_tr_b128_v8i16(v8s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8i16' needs target feature gfx12-insts,wavefrontsize32}}
-  v8h out_3 = __builtin_amdgcn_global_load_tr_b128_v8f16(v8h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8f16' needs target feature gfx12-insts,wavefrontsize32}}
-  v8bf16 o4 = __builtin_amdgcn_global_load_tr_b128_v8bf16(v8bf16_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8bf16' needs target feature gfx12-insts,wavefrontsize32}}
 
-  int out_5 = __builtin_amdgcn_global_load_tr_b64_i32(int_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_i32' needs target feature gfx12-insts,wavefrontsize64}}
-  v4s out_6 = __builtin_amdgcn_global_load_tr_b128_v4i16(v4s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4i16' needs target feature gfx12-insts,wavefrontsize64}}
-  v4h out_7 = __builtin_amdgcn_global_load_tr_b128_v4f16(v4h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4f16' needs target feature gfx12-insts,wavefrontsize64}}
-  v4bf16 o8 = __builtin_amdgcn_global_load_tr_b128_v4bf16(v4bf16_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4bf16' needs target feature gfx12-insts,wavefrontsize64}}
+  int out_3 = __builtin_amdgcn_global_load_tr_b64_i32(int_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_i32' needs target feature gfx12-insts,wavefrontsize64}}
+  v4s out_4 = __builtin_amdgcn_global_load_tr_b128_v4i16(v4s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4i16' needs target feature gfx12-insts,wavefrontsize64}}
 }
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl
index 208f92fc5d44..1acc4cd7adc9 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl
@@ -3,14 +3,10 @@
 
 // REQUIRES: amdgpu-registered-target
 
-typedef half   v4h   __attribute__((ext_vector_type(4)));
 typedef short  v4s   __attribute__((ext_vector_type(4)));
-typedef __bf16 v4bf16 __attribute__((ext_vector_type(4)));
 
-void amdgcn_global_load_tr(global int* int_inptr, global v4s* v4s_inptr, global v4h* v4h_inptr, global v4bf16* v4bf16_inptr)
+void amdgcn_global_load_tr(global int* int_inptr, global v4s* v4s_inptr)
 {
   int out_1 = __builtin_amdgcn_global_load_tr_b64_i32(int_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_i32' needs target feature gfx12-insts,wavefrontsize64}}
   v4s out_2 = __builtin_amdgcn_global_load_tr_b128_v4i16(v4s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4i16' needs target feature gfx12-insts,wavefrontsize64}}
-  v4h out_3 = __builtin_amdgcn_global_load_tr_b128_v4f16(v4h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4f16' needs target feature gfx12-insts,wavefrontsize64}}
-  v4bf16 o4 = __builtin_amdgcn_global_load_tr_b128_v4bf16(v4bf16_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4bf16' needs target feature gfx12-insts,wavefrontsize64}}
 }
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl
index 199146a9715d..96b0e4c3993a 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl
@@ -4,14 +4,10 @@
 // REQUIRES: amdgpu-registered-target
 
 typedef int    v2i   __attribute__((ext_vector_type(2)));
-typedef half   v8h   __attribute__((ext_vector_type(8)));
 typedef short  v8s   __attribute__((ext_vector_type(8)));
-typedef __bf16 v8bf16 __attribute__((ext_vector_type(8)));
 
-void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr, global v8h* v8h_inptr, global v8bf16* v8bf16_inptr)
+void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr)
 {
   v2i out_1 = __builtin_amdgcn_global_load_tr_b64_v2i32(v2i_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_v2i32' needs target feature gfx12-insts,wavefrontsize32}}
   v8s out_2 = __builtin_amdgcn_global_load_tr_b128_v8i16(v8s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8i16' needs target feature gfx12-insts,wavefrontsize32}}
-  v8h out_3 = __builtin_amdgcn_global_load_tr_b128_v8f16(v8h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8f16' needs target feature gfx12-insts,wavefrontsize32}}
-  v8bf16 o4 = __builtin_amdgcn_global_load_tr_b128_v8bf16(v8bf16_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8bf16' needs target feature gfx12-insts,wavefrontsize32}}
 }
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl
index 0035b16b902b..126d7d6fb7b0 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl
@@ -3,13 +3,11 @@
 // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1200 -target-feature +wavefrontsize32 -S -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-GFX1200
 
 typedef int    v2i   __attribute__((ext_vector_type(2)));
-typedef half   v8h   __attribute__((ext_vector_type(8)));
 typedef short  v8s   __attribute__((ext_vector_type(8)));
-typedef __bf16 v8bf16 __attribute__((ext_vector_type(8)));
 
 // CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b64_v2i32(
 // CHECK-GFX1200-NEXT:  entry:
-// CHECK-GFX1200-NEXT:    [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.global.load.tr.v2i32(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1200-NEXT:    [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.global.load.tr.b64.v2i32(ptr addrspace(1) [[INPTR:%.*]])
 // CHECK-GFX1200-NEXT:    ret <2 x i32> [[TMP0]]
 //
 v2i test_amdgcn_global_load_tr_b64_v2i32(global v2i* inptr)
@@ -19,30 +17,10 @@ v2i test_amdgcn_global_load_tr_b64_v2i32(global v2i* inptr)
 
 // CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v8i16(
 // CHECK-GFX1200-NEXT:  entry:
-// CHECK-GFX1200-NEXT:    [[TMP0:%.*]] = tail call <8 x i16> @llvm.amdgcn.global.load.tr.v8i16(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1200-NEXT:    [[TMP0:%.*]] = tail call <8 x i16> @llvm.amdgcn.global.load.tr.b128.v8i16(ptr addrspace(1) [[INPTR:%.*]])
 // CHECK-GFX1200-NEXT:    ret <8 x i16> [[TMP0]]
 //
 v8s test_amdgcn_global_load_tr_b128_v8i16(global v8s* inptr)
 {
   return __builtin_amdgcn_global_load_tr_b128_v8i16(inptr);
 }
-
-// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v8f16(
-// CHECK-GFX1200-NEXT:  entry:
-// CHECK-GFX1200-NEXT:    [[TMP0:%.*]] = tail call <8 x half> @llvm.amdgcn.global.load.tr.v8f16(ptr addrspace(1) [[INPTR:%.*]])
-// CHECK-GFX1200-NEXT:    ret <8 x half> [[TMP0]]
-//
-v8h test_amdgcn_global_load_tr_b128_v8f16(global v8h* inptr)
-{
-  return __builtin_amdgcn_global_load_tr_b128_v8f16(inptr);
-}
-
-// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v8bf16(
-// CHECK-GFX1200-NEXT:  entry:
-// CHECK-GFX1200-NEXT:    [[TMP0:%.*]] = tail call <8 x bfloat> @llvm.amdgcn.global.load.tr.v8bf16(ptr addrspace(1) [[INPTR:%.*]])
-// CHECK-GFX1200-NEXT:    ret <8 x bfloat> [[TMP0]]
-//
-v8bf16 test_amdgcn_global_load_tr_b128_v8bf16(global v8bf16* inptr)
-{
-  return __builtin_amdgcn_global_load_tr_b128_v8bf16(inptr);
-}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl
index 6c025bb5a55a..7c70ccf73ad3 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl
@@ -2,13 +2,11 @@
 // REQUIRES: amdgpu-registered-target
 // RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1200 -target-feature +wavefrontsize64 -S -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-GFX1200
 
-typedef half   v4h   __attribute__((ext_vector_type(4)));
 typedef short  v4s   __attribute__((ext_vector_type(4)));
-typedef __bf16 v4bf16 __attribute__((ext_vector_type(4)));
 
 // CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b64_i32(
 // CHECK-GFX1200-NEXT:  entry:
-// CHECK-GFX1200-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.global.load.tr.i32(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1200-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.global.load.tr.b64.i32(ptr addrspace(1) [[INPTR:%.*]])
 // CHECK-GFX1200-NEXT:    ret i32 [[TMP0]]
 //
 int test_amdgcn_global_load_tr_b64_i32(global int* inptr)
@@ -18,30 +16,10 @@ int test_amdgcn_global_load_tr_b64_i32(global int* inptr)
 
 // CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v4i16(
 // CHECK-GFX1200-NEXT:  entry:
-// CHECK-GFX1200-NEXT:    [[TMP0:%.*]] = tail call <4 x i16> @llvm.amdgcn.global.load.tr.v4i16(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1200-NEXT:    [[TMP0:%.*]] = tail call <4 x i16> @llvm.amdgcn.global.load.tr.b128.v4i16(ptr addrspace(1) [[INPTR:%.*]])
 // CHECK-GFX1200-NEXT:    ret <4 x i16> [[TMP0]]
 //
 v4s test_amdgcn_global_load_tr_b128_v4i16(global v4s* inptr)
 {
   return __builtin_amdgcn_global_load_tr_b128_v4i16(inptr);
 }
-
-// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v4f16(
-// CHECK-GFX1200-NEXT:  entry:
-// CHECK-GFX1200-NEXT:    [[TMP0:%.*]] = tail call <4 x half> @llvm.amdgcn.global.load.tr.v4f16(ptr addrspace(1) [[INPTR:%.*]])
-// CHECK-GFX1200-NEXT:    ret <4 x half> [[TMP0]]
-//
-v4h test_amdgcn_global_load_tr_b128_v4f16(global v4h* inptr)
-{
-  return __builtin_amdgcn_global_load_tr_b128_v4f16(inptr);
-}
-
-// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v4bf16(
-// CHECK-GFX1200-NEXT:  entry:
-// CHECK-GFX1200-NEXT:    [[TMP0:%.*]] = tail call <4 x bfloat> @llvm.amdgcn.global.load.tr.v4bf16(ptr addrspace(1) [[INPTR:%.*]])
-// CHECK-GFX1200-NEXT:    ret <4 x bfloat> [[TMP0]]
-//
-v4bf16 test_amdgcn_global_load_tr_b128_v4bf16(global v4bf16* inptr)
-{
-  return __builtin_amdgcn_global_load_tr_b128_v4bf16(inptr);
-}
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index fff03dee20a1..bda3b066b776 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -2769,17 +2769,14 @@ class AMDGPULoadIntrinsic<LLVMType ptr_ty>:
   >;
 
 // Wave32
-// <2 x i32>    @llvm.amdgcn.global.load.tr.v2i32(ptr addrspace(1))  -> global_load_tr_b64
-// <8 x i16>    @llvm.amdgcn.global.load.tr.v8i16(ptr addrspace(1))  -> global_load_tr_b128
-// <8 x half>   @llvm.amdgcn.global.load.tr.v8f16(ptr addrspace(1))  -> global_load_tr_b128
-// <8 x bfloat> @llvm.amdgcn.global.load.tr.v8bf16(ptr addrspace(1)) -> global_load_tr_b128
+// <2 x i32>    @llvm.amdgcn.global.load.tr.b64.v2i32(ptr addrspace(1))  -> global_load_tr_b64
+// <8 x i16>    @llvm.amdgcn.global.load.tr.b128.v8i16(ptr addrspace(1))  -> global_load_tr_b128
 // Wave64
-// i32          @llvm.amdgcn.global.load.tr.i32(ptr addrspace(1))    -> global_load_tr_b64
-// <4 x i16>    @llvm.amdgcn.global.load.tr.v4i16(ptr addrspace(1))  -> global_load_tr_b128
-// <4 x half>   @llvm.amdgcn.global.load.tr.v4f16(ptr addrspace(1))  -> global_load_tr_b128
-// <4 x bfloat> @llvm.amdgcn.global.load.tr.v4bf16(ptr addrspace(1)) -> global_load_tr_b128
+// i32          @llvm.amdgcn.global.load.tr.b64.i32(ptr addrspace(1))    -> global_load_tr_b64
+// <4 x i16>    @llvm.amdgcn.global.load.tr.b128.v4i16(ptr addrspace(1))  -> global_load_tr_b128
 
-def int_amdgcn_global_load_tr : AMDGPULoadIntrinsic<global_ptr_ty>;
+def int_amdgcn_global_load_tr_b64  : AMDGPULoadIntrinsic<global_ptr_ty>;
+def int_amdgcn_global_load_tr_b128 : AMDGPULoadIntrinsic<global_ptr_ty>;
 
 // i32 @llvm.amdgcn.wave.id()
 def int_amdgcn_wave_id :
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 0037825ce089..a42e95f140ce 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -4889,7 +4889,8 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     case Intrinsic::amdgcn_flat_atomic_fadd_v2bf16:
     case Intrinsic::amdgcn_atomic_cond_sub_u32:
     case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
-    case Intrinsic::amdgcn_global_load_tr:
+    case Intrinsic::amdgcn_global_load_tr_b64:
+    case Intrinsic::amdgcn_global_load_tr_b128:
       return getDefaultMappingAllVGPR(MI);
     case Intrinsic::amdgcn_ds_ordered_add:
     case Intrinsic::amdgcn_ds_ordered_swap:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
index 8eb46a980148..410dc83d45c5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
@@ -365,7 +365,8 @@ def : SourceOfDivergence<intr>;
 foreach intr = AMDGPUWMMAIntrinsicsGFX12 in
 def : SourceOfDivergence<intr>;
 
-def : SourceOfDivergence<int_amdgcn_global_load_tr>;
+def : SourceOfDivergence<int_amdgcn_global_load_tr_b64>;
+def : SourceOfDivergence<int_amdgcn_global_load_tr_b128>;
 
 // The dummy boolean output is divergent from the IR's perspective,
 // but the mask results are uniform. These produce a divergent and
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index db1f8c187265..d017ec4a7415 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -1593,16 +1593,12 @@ let OtherPredicates = [isGFX12Plus] in {
   defm : GlobalFLATAtomicPatsRtn <"GLOBAL_ATOMIC_ORDERED_ADD_B64", "int_amdgcn_global_atomic_ordered_add_b64", i64, i64, /* isIntr */ 1>;
 
   let WaveSizePredicate = isWave32 in {
-    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B64_w32, int_amdgcn_global_load_tr, v2i32>;
-    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w32, int_amdgcn_global_load_tr, v8i16>;
-    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w32, int_amdgcn_global_load_tr, v8f16>;
-    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w32, int_amdgcn_global_load_tr, v8bf16>;
+    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B64_w32, int_amdgcn_global_load_tr_b64, v2i32>;
+    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w32, int_amdgcn_global_load_tr_b128, v8i16>;
   }
   let WaveSizePredicate = isWave64 in {
-    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B64_w64, int_amdgcn_global_load_tr, i32>;
-    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w64, int_amdgcn_global_load_tr, v4i16>;
-    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w64, int_amdgcn_global_load_tr, v4f16>;
-    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w64, int_amdgcn_global_load_tr, v4bf16>;
+    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B64_w64, int_amdgcn_global_load_tr_b64, i32>;
+    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w64, int_amdgcn_global_load_tr_b128, v4i16>;
   }
 }
 
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 81a231f0cade..375ddb24674b 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1355,7 +1355,8 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
                   MachineMemOperand::MOVolatile;
     return true;
   }
-  case Intrinsic::amdgcn_global_load_tr: {
+  case Intrinsic::amdgcn_global_load_tr_b64:
+  case Intrinsic::amdgcn_global_load_tr_b128: {
     Info.opc = ISD::INTRINSIC_W_CHAIN;
     Info.memVT = MVT::getVT(CI.getType());
     Info.ptrVal = CI.getOperand(0);
@@ -1462,7 +1463,8 @@ bool SITargetLowering::getAddrModeArguments(IntrinsicInst *II,
   case Intrinsic::amdgcn_global_atomic_fmin:
   case Intrinsic::amdgcn_global_atomic_fmin_num:
   case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
-  case Intrinsic::amdgcn_global_load_tr:
+  case Intrinsic::amdgcn_global_load_tr_b64:
+  case Intrinsic::amdgcn_global_load_tr_b128:
     Ptr = II->getArgOperand(0);
     break;
   case Intrinsic::amdgcn_global_load_lds:
diff --git a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
index 333ec6dcaf48..26c85e83b53a 100644
--- a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
+++ b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
@@ -197,78 +197,38 @@ bb:
   ret void
 }
 
-; CHECK: DIVERGENT: %tmp0 = call <2 x i32> @llvm.amdgcn.global.load.tr.v2i32(ptr addrspace(1) %gep)
+; CHECK: DIVERGENT: %tmp0 = call <2 x i32> @llvm.amdgcn.global.load.tr.b64.v2i32(ptr addrspace(1) %addr)
 define amdgpu_kernel void @global_load_tr_b64_v2i32(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
 bb:
-  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
-  %tmp0 = call <2 x i32> @llvm.amdgcn.global.load.tr.v2i32(ptr addrspace(1) %gep)
+  %tmp0 = call <2 x i32> @llvm.amdgcn.global.load.tr.b64.v2i32(ptr addrspace(1) %addr)
   store <2 x i32> %tmp0, ptr addrspace(1) %out, align 8
   ret void
 }
 
-; CHECK: DIVERGENT: %tmp0 = call <8 x i16> @llvm.amdgcn.global.load.tr.v8i16(ptr addrspace(1) %gep)
+; CHECK: DIVERGENT: %tmp0 = call <8 x i16> @llvm.amdgcn.global.load.tr.b128.v8i16(ptr addrspace(1) %addr)
 define amdgpu_kernel void @global_load_tr_b128_v8i16(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
 bb:
-  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
-  %tmp0 = call <8 x i16> @llvm.amdgcn.global.load.tr.v8i16(ptr addrspace(1) %gep)
+  %tmp0 = call <8 x i16> @llvm.amdgcn.global.load.tr.b128.v8i16(ptr addrspace(1) %addr)
   store <8 x i16> %tmp0, ptr addrspace(1) %out, align 16
   ret void
 }
 
-; CHECK: DIVERGENT: %tmp0 = call <8 x half> @llvm.amdgcn.global.load.tr.v8f16(ptr addrspace(1) %gep)
-define amdgpu_kernel void @global_load_tr_b128_v8f16(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
-bb:
-  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
-  %tmp0 = call <8 x half> @llvm.amdgcn.global.load.tr.v8f16(ptr addrspace(1) %gep)
-  store <8 x half> %tmp0, ptr addrspace(1) %out, align 16
-  ret void
-}
-
-; CHECK: DIVERGENT: %tmp0 = call <8 x bfloat> @llvm.amdgcn.global.load.tr.v8bf16(ptr addrspace(1) %gep)
-define amdgpu_kernel void @global_load_tr_b128_v8bf16(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
-bb:
-  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
-  %tmp0 = call <8 x bfloat> @llvm.amdgcn.global.load.tr.v8bf16(ptr addrspace(1) %gep)
-  store <8 x bfloat> %tmp0, ptr addrspace(1) %out, align 16
-  ret void
-}
-
-; CHECK: DIVERGENT: %tmp0 = call i32 @llvm.amdgcn.global.load.tr.i32(ptr addrspace(1) %gep)
+; CHECK: DIVERGENT: %tmp0 = call i32 @llvm.amdgcn.global.load.tr.b64.i32(ptr addrspace(1) %addr)
 define amdgpu_kernel void @global_load_tr_b64_i32(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
 bb:
-  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
-  %tmp0 = call i32 @llvm.amdgcn.global.load.tr.i32(ptr addrspace(1) %gep)
+  %tmp0 = call i32 @llvm.amdgcn.global.load.tr.b64.i32(ptr addrspace(1) %addr)
   store i32 %tmp0, ptr addrspace(1) %out, align 4
   ret void
 }
 
-; CHECK: DIVERGENT: %tmp0 = call <4 x i16> @llvm.amdgcn.global.load.tr.v4i16(ptr addrspace(1) %gep)
-define amdgpu_kernel void @global_load_tr_b128_v4i16_(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
+; CHECK: DIVERGENT: %tmp0 = call <4 x i16> @llvm.amdgcn.global.load.tr.b128.v4i16(ptr addrspace(1) %addr)
+define amdgpu_kernel void @global_load_tr_b128_v4i16(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
 bb:
-  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
-  %tmp0 = call <4 x i16> @llvm.amdgcn.global.load.tr.v4i16(ptr addrspace(1) %gep)
+  %tmp0 = call <4 x i16> @llvm.amdgcn.global.load.tr.b128.v4i16(ptr addrspace(1) %addr)
   store <4 x i16> %tmp0, ptr addrspace(1) %out, align 8
   ret void
 }
 
-; CHECK: DIVERGENT: %tmp0 = call <4 x half> @llvm.amdgcn.global.load.tr.v4f16(ptr addrspace(1) %gep)
-define amdgpu_kernel void @global_load_tr_b128_v4f16(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
-bb:
-  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
-  %tmp0 = call <4 x half> @llvm.amdgcn.global.load.tr.v4f16(ptr addrspace(1) %gep)
-  store <4 x half> %tmp0, ptr addrspace(1) %out, align 8
-  ret void
-}
-
-; CHECK: DIVERGENT: %tmp0 = call <4 x bfloat> @llvm.amdgcn.global.load.tr.v4bf16(ptr addrspace(1) %gep)
-define amdgpu_kernel void @global_load_tr_b128_v4bf16(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
-bb:
-  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
-  %tmp0 = call <4 x bfloat> @llvm.amdgcn.global.load.tr.v4bf16(ptr addrspace(1) %gep)
-  store <4 x bfloat> %tmp0, ptr addrspace(1) %out, align 8
-  ret void
-}
-
 declare i32 @llvm.amdgcn.ds.swizzle(i32, i32) #1
 declare i32 @llvm.amdgcn.permlane16(i32, i32, i32, i32, i1, i1) #1
 declare i32 @llvm.amdgcn.permlanex16(i32, i32, i32, i32, i1, i1) #1
@@ -296,14 +256,10 @@ declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x32.fp8.bf8(<2 x i32>, <4 x i32
 declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf8.fp8(<2 x i32>, <4 x i32>, <8 x float>, i16)
 declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x32.bf8.bf8(<2 x i32>, <4 x i32>, <8 x float>, i16)
 
-declare <2 x i32> @llvm.amdgcn.global.load.tr.v2i32(ptr addrspace(1))
-declare <8 x i16> @llvm.amdgcn.global.load.tr.v8i16(ptr addrspace(1))
-declare <8 x half> @llvm.amdgcn.global.load.tr.v8f16(ptr addrspace(1))
-declare <8 x bfloat> @llvm.amdgcn.global.load.tr.v8bf16(ptr addrspace(1))
-declare i32 @llvm.amdgcn.global.load.tr.i32(ptr addrspace(1))
-declare <4 x i16> @llvm.amdgcn.global.load.tr.v4i16(ptr addrspace(1))
-declare <4 x half> @llvm.amdgcn.global.load.tr.v4f16(ptr addrspace(1))
-declare <4 x bfloat> @llvm.amdgcn.global.load.tr.v4bf16(ptr addrspace(1))
+declare <2 x i32> @llvm.amdgcn.global.load.tr.b64.v2i32(ptr addrspace(1))
+declare <8 x i16> @llvm.amdgcn.global.load.tr.b128.v8i16(ptr addrspace(1))
+declare i32 @llvm.amdgcn.global.load.tr.b64.i32(ptr addrspace(1))
+declare <4 x i16> @llvm.amdgcn.global.load.tr.b128.v4i16(ptr addrspace(1))
 
 attributes #0 = { nounwind convergent }
 attributes #1 = { nounwind readnone convergent }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w32.ll
index b4415c12926a..f6197e077021 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w32.ll
@@ -1,132 +1,44 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=+wavefrontsize32,-wavefrontsize64 < %s | FileCheck -check-prefixes=GFX12-SDAG-W32 %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=+wavefrontsize32,-wavefrontsize64 < %s | FileCheck -check-prefixes=GFX12-GISEL-W32 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=+wavefrontsize32,-wavefrontsize64 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=+wavefrontsize32,-wavefrontsize64 < %s | FileCheck -check-prefix=GFX12 %s
 
-declare <2 x i32> @llvm.amdgcn.global.load.tr.v2i32.p1(ptr addrspace(1))
-declare <8 x i16> @llvm.amdgcn.global.load.tr.v8i16.p1(ptr addrspace(1))
-declare <8 x half> @llvm.amdgcn.global.load.tr.v8f16.p1(ptr addrspace(1))
-declare <8 x bfloat> @llvm.amdgcn.global.load.tr.v8bf16.p1(ptr addrspace(1))
+declare <2 x i32> @llvm.amdgcn.global.load.tr.b64.v2i32.p1(ptr addrspace(1))
+declare <8 x i16> @llvm.amdgcn.global.load.tr.b128.v8i16.p1(ptr addrspace(1))
 
 define amdgpu_kernel void @global_load_tr_b64(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
-; GFX12-SDAG-W32-LABEL: global_load_tr_b64:
-; GFX12-SDAG-W32:       ; %bb.0: ; %entry
-; GFX12-SDAG-W32-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX12-SDAG-W32-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-SDAG-W32-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-W32-NEXT:    global_load_tr_b64 v[0:1], v2, s[0:1] offset:32
-; GFX12-SDAG-W32-NEXT:    s_wait_loadcnt 0x0
-; GFX12-SDAG-W32-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
-; GFX12-SDAG-W32-NEXT:    s_nop 0
-; GFX12-SDAG-W32-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX12-SDAG-W32-NEXT:    s_endpgm
-;
-; GFX12-GISEL-W32-LABEL: global_load_tr_b64:
-; GFX12-GISEL-W32:       ; %bb.0: ; %entry
-; GFX12-GISEL-W32-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX12-GISEL-W32-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-GISEL-W32-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-W32-NEXT:    global_load_tr_b64 v[0:1], v2, s[0:1] offset:32
-; GFX12-GISEL-W32-NEXT:    s_wait_loadcnt 0x0
-; GFX12-GISEL-W32-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
-; GFX12-GISEL-W32-NEXT:    s_nop 0
-; GFX12-GISEL-W32-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX12-GISEL-W32-NEXT:    s_endpgm
+; GFX12-LABEL: global_load_tr_b64:
+; GFX12:       ; %bb.0: ; %entry
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    global_load_tr_b64 v[0:1], v2, s[0:1] offset:32
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
-  %val = call <2 x i32> @llvm.amdgcn.global.load.tr.v2i32.p1(ptr addrspace(1) %gep)
+  %val = call <2 x i32> @llvm.amdgcn.global.load.tr.b64.v2i32.p1(ptr addrspace(1) %gep)
   store <2 x i32> %val, ptr addrspace(1) %use
   ret void
 }
 
-define amdgpu_kernel void @global_load_tr_b128_i16(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
-; GFX12-SDAG-W32-LABEL: global_load_tr_b128_i16:
-; GFX12-SDAG-W32:       ; %bb.0: ; %entry
-; GFX12-SDAG-W32-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX12-SDAG-W32-NEXT:    v_mov_b32_e32 v4, 0
-; GFX12-SDAG-W32-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-W32-NEXT:    global_load_tr_b128 v[0:3], v4, s[0:1] offset:32
-; GFX12-SDAG-W32-NEXT:    s_wait_loadcnt 0x0
-; GFX12-SDAG-W32-NEXT:    global_store_b128 v4, v[0:3], s[2:3]
-; GFX12-SDAG-W32-NEXT:    s_nop 0
-; GFX12-SDAG-W32-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX12-SDAG-W32-NEXT:    s_endpgm
-;
-; GFX12-GISEL-W32-LABEL: global_load_tr_b128_i16:
-; GFX12-GISEL-W32:       ; %bb.0: ; %entry
-; GFX12-GISEL-W32-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX12-GISEL-W32-NEXT:    v_mov_b32_e32 v4, 0
-; GFX12-GISEL-W32-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-W32-NEXT:    global_load_tr_b128 v[0:3], v4, s[0:1] offset:32
-; GFX12-GISEL-W32-NEXT:    s_wait_loadcnt 0x0
-; GFX12-GISEL-W32-NEXT:    global_store_b128 v4, v[0:3], s[2:3]
-; GFX12-GISEL-W32-NEXT:    s_nop 0
-; GFX12-GISEL-W32-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX12-GISEL-W32-NEXT:    s_endpgm
+define amdgpu_kernel void @global_load_tr_b128(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX12-LABEL: global_load_tr_b128:
+; GFX12:       ; %bb.0: ; %entry
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_mov_b32_e32 v4, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    global_load_tr_b128 v[0:3], v4, s[0:1] offset:32
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    global_store_b128 v4, v[0:3], s[2:3]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
-  %val = call <8 x i16> @llvm.amdgcn.global.load.tr.v8i16.p1(ptr addrspace(1) %gep)
+  %val = call <8 x i16> @llvm.amdgcn.global.load.tr.b128.v8i16.p1(ptr addrspace(1) %gep)
   store <8 x i16> %val, ptr addrspace(1) %use
   ret void
 }
-
-define amdgpu_kernel void @global_load_tr_b128_half(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
-; GFX12-SDAG-W32-LABEL: global_load_tr_b128_half:
-; GFX12-SDAG-W32:       ; %bb.0: ; %entry
-; GFX12-SDAG-W32-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX12-SDAG-W32-NEXT:    v_mov_b32_e32 v4, 0
-; GFX12-SDAG-W32-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-W32-NEXT:    global_load_tr_b128 v[0:3], v4, s[0:1] offset:32
-; GFX12-SDAG-W32-NEXT:    s_wait_loadcnt 0x0
-; GFX12-SDAG-W32-NEXT:    global_store_b128 v4, v[0:3], s[2:3]
-; GFX12-SDAG-W32-NEXT:    s_nop 0
-; GFX12-SDAG-W32-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX12-SDAG-W32-NEXT:    s_endpgm
-;
-; GFX12-GISEL-W32-LABEL: global_load_tr_b128_half:
-; GFX12-GISEL-W32:       ; %bb.0: ; %entry
-; GFX12-GISEL-W32-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX12-GISEL-W32-NEXT:    v_mov_b32_e32 v4, 0
-; GFX12-GISEL-W32-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-W32-NEXT:    global_load_tr_b128 v[0:3], v4, s[0:1] offset:32
-; GFX12-GISEL-W32-NEXT:    s_wait_loadcnt 0x0
-; GFX12-GISEL-W32-NEXT:    global_store_b128 v4, v[0:3], s[2:3]
-; GFX12-GISEL-W32-NEXT:    s_nop 0
-; GFX12-GISEL-W32-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX12-GISEL-W32-NEXT:    s_endpgm
-entry:
-  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
-  %val = call <8 x half> @llvm.amdgcn.global.load.tr.v8f16.p1(ptr addrspace(1) %gep)
-  store <8 x half> %val, ptr addrspace(1) %use
-  ret void
-}
-
-define amdgpu_kernel void @global_load_tr_b128_bfloat(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
-; GFX12-SDAG-W32-LABEL: global_load_tr_b128_bfloat:
-; GFX12-SDAG-W32:       ; %bb.0: ; %entry
-; GFX12-SDAG-W32-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX12-SDAG-W32-NEXT:    v_mov_b32_e32 v4, 0
-; GFX12-SDAG-W32-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-W32-NEXT:    global_load_tr_b128 v[0:3], v4, s[0:1] offset:32
-; GFX12-SDAG-W32-NEXT:    s_wait_loadcnt 0x0
-; GFX12-SDAG-W32-NEXT:    global_store_b128 v4, v[0:3], s[2:3]
-; GFX12-SDAG-W32-NEXT:    s_nop 0
-; GFX12-SDAG-W32-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX12-SDAG-W32-NEXT:    s_endpgm
-;
-; GFX12-GISEL-W32-LABEL: global_load_tr_b128_bfloat:
-; GFX12-GISEL-W32:       ; %bb.0: ; %entry
-; GFX12-GISEL-W32-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX12-GISEL-W32-NEXT:    v_mov_b32_e32 v4, 0
-; GFX12-GISEL-W32-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-W32-NEXT:    global_load_tr_b128 v[0:3], v4, s[0:1] offset:32
-; GFX12-GISEL-W32-NEXT:    s_wait_loadcnt 0x0
-; GFX12-GISEL-W32-NEXT:    global_store_b128 v4, v[0:3], s[2:3]
-; GFX12-GISEL-W32-NEXT:    s_nop 0
-; GFX12-GISEL-W32-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX12-GISEL-W32-NEXT:    s_endpgm
-entry:
-  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
-  %val = call <8 x bfloat> @llvm.amdgcn.global.load.tr.v8bf16.p1(ptr addrspace(1) %gep)
-  store <8 x bfloat> %val, ptr addrspace(1) %use
-  ret void
-}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w64.ll
index 7ad1416789de..a2dc3662fcc4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.tr-w64.ll
@@ -1,132 +1,44 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=-wavefrontsize32,+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX12-SDAG-W64 %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=-wavefrontsize32,+wavefrontsize64 < %s | FileCheck -check-prefixes=GFX12-GISEL-W64 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=-wavefrontsize32,+wavefrontsize64 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -mattr=-wavefrontsize32,+wavefrontsize64 < %s | FileCheck -check-prefix=GFX12 %s
 
-declare i32 @llvm.amdgcn.global.load.tr.i32.p1(ptr addrspace(1))
-declare <4 x i16> @llvm.amdgcn.global.load.tr.v4i16.p1(ptr addrspace(1))
-declare <4 x half> @llvm.amdgcn.global.load.tr.v4f16.p1(ptr addrspace(1))
-declare <4 x bfloat> @llvm.amdgcn.global.load.tr.v4bf16.p1(ptr addrspace(1))
+declare i32 @llvm.amdgcn.global.load.tr.b64.i32.p1(ptr addrspace(1))
+declare <4 x i16> @llvm.amdgcn.global.load.tr.b128.v4i16.p1(ptr addrspace(1))
 
 define amdgpu_kernel void @global_load_tr_b64(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
-; GFX12-SDAG-W64-LABEL: global_load_tr_b64:
-; GFX12-SDAG-W64:       ; %bb.0: ; %entry
-; GFX12-SDAG-W64-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX12-SDAG-W64-NEXT:    v_mov_b32_e32 v0, 0
-; GFX12-SDAG-W64-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-W64-NEXT:    global_load_tr_b64 v1, v0, s[0:1] offset:32
-; GFX12-SDAG-W64-NEXT:    s_wait_loadcnt 0x0
-; GFX12-SDAG-W64-NEXT:    global_store_b32 v0, v1, s[2:3]
-; GFX12-SDAG-W64-NEXT:    s_nop 0
-; GFX12-SDAG-W64-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX12-SDAG-W64-NEXT:    s_endpgm
-;
-; GFX12-GISEL-W64-LABEL: global_load_tr_b64:
-; GFX12-GISEL-W64:       ; %bb.0: ; %entry
-; GFX12-GISEL-W64-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX12-GISEL-W64-NEXT:    v_mov_b32_e32 v0, 0
-; GFX12-GISEL-W64-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-W64-NEXT:    global_load_tr_b64 v1, v0, s[0:1] offset:32
-; GFX12-GISEL-W64-NEXT:    s_wait_loadcnt 0x0
-; GFX12-GISEL-W64-NEXT:    global_store_b32 v0, v1, s[2:3]
-; GFX12-GISEL-W64-NEXT:    s_nop 0
-; GFX12-GISEL-W64-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX12-GISEL-W64-NEXT:    s_endpgm
+; GFX12-LABEL: global_load_tr_b64:
+; GFX12:       ; %bb.0: ; %entry
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    global_load_tr_b64 v1, v0, s[0:1] offset:32
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
-  %val = call i32 @llvm.amdgcn.global.load.tr.i32.p1(ptr addrspace(1) %gep)
+  %val = call i32 @llvm.amdgcn.global.load.tr.b64.i32.p1(ptr addrspace(1) %gep)
   store i32 %val, ptr addrspace(1) %use
   ret void
 }
 
-define amdgpu_kernel void @global_load_tr_b128_i16(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
-; GFX12-SDAG-W64-LABEL: global_load_tr_b128_i16:
-; GFX12-SDAG-W64:       ; %bb.0: ; %entry
-; GFX12-SDAG-W64-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX12-SDAG-W64-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-SDAG-W64-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-W64-NEXT:    global_load_tr_b128 v[0:1], v2, s[0:1] offset:32
-; GFX12-SDAG-W64-NEXT:    s_wait_loadcnt 0x0
-; GFX12-SDAG-W64-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
-; GFX12-SDAG-W64-NEXT:    s_nop 0
-; GFX12-SDAG-W64-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX12-SDAG-W64-NEXT:    s_endpgm
-;
-; GFX12-GISEL-W64-LABEL: global_load_tr_b128_i16:
-; GFX12-GISEL-W64:       ; %bb.0: ; %entry
-; GFX12-GISEL-W64-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX12-GISEL-W64-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-GISEL-W64-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-W64-NEXT:    global_load_tr_b128 v[0:1], v2, s[0:1] offset:32
-; GFX12-GISEL-W64-NEXT:    s_wait_loadcnt 0x0
-; GFX12-GISEL-W64-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
-; GFX12-GISEL-W64-NEXT:    s_nop 0
-; GFX12-GISEL-W64-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX12-GISEL-W64-NEXT:    s_endpgm
+define amdgpu_kernel void @global_load_tr_b128(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX12-LABEL: global_load_tr_b128:
+; GFX12:       ; %bb.0: ; %entry
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_mov_b32_e32 v2, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    global_load_tr_b128 v[0:1], v2, s[0:1] offset:32
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
-  %val = call <4 x i16> @llvm.amdgcn.global.load.tr.v4i16.p1(ptr addrspace(1) %gep)
+  %val = call <4 x i16> @llvm.amdgcn.global.load.tr.b128.v4i16.p1(ptr addrspace(1) %gep)
   store <4 x i16> %val, ptr addrspace(1) %use
   ret void
 }
-
-define amdgpu_kernel void @global_load_tr_b128_half(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
-; GFX12-SDAG-W64-LABEL: global_load_tr_b128_half:
-; GFX12-SDAG-W64:       ; %bb.0: ; %entry
-; GFX12-SDAG-W64-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX12-SDAG-W64-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-SDAG-W64-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-W64-NEXT:    global_load_tr_b128 v[0:1], v2, s[0:1] offset:32
-; GFX12-SDAG-W64-NEXT:    s_wait_loadcnt 0x0
-; GFX12-SDAG-W64-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
-; GFX12-SDAG-W64-NEXT:    s_nop 0
-; GFX12-SDAG-W64-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX12-SDAG-W64-NEXT:    s_endpgm
-;
-; GFX12-GISEL-W64-LABEL: global_load_tr_b128_half:
-; GFX12-GISEL-W64:       ; %bb.0: ; %entry
-; GFX12-GISEL-W64-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX12-GISEL-W64-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-GISEL-W64-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-W64-NEXT:    global_load_tr_b128 v[0:1], v2, s[0:1] offset:32
-; GFX12-GISEL-W64-NEXT:    s_wait_loadcnt 0x0
-; GFX12-GISEL-W64-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
-; GFX12-GISEL-W64-NEXT:    s_nop 0
-; GFX12-GISEL-W64-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX12-GISEL-W64-NEXT:    s_endpgm
-entry:
-  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
-  %val = call <4 x half> @llvm.amdgcn.global.load.tr.v4f16.p1(ptr addrspace(1) %gep)
-  store <4 x half> %val, ptr addrspace(1) %use
-  ret void
-}
-
-define amdgpu_kernel void @global_load_tr_b128_bfloat(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
-; GFX12-SDAG-W64-LABEL: global_load_tr_b128_bfloat:
-; GFX12-SDAG-W64:       ; %bb.0: ; %entry
-; GFX12-SDAG-W64-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX12-SDAG-W64-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-SDAG-W64-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-W64-NEXT:    global_load_tr_b128 v[0:1], v2, s[0:1] offset:32
-; GFX12-SDAG-W64-NEXT:    s_wait_loadcnt 0x0
-; GFX12-SDAG-W64-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
-; GFX12-SDAG-W64-NEXT:    s_nop 0
-; GFX12-SDAG-W64-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX12-SDAG-W64-NEXT:    s_endpgm
-;
-; GFX12-GISEL-W64-LABEL: global_load_tr_b128_bfloat:
-; GFX12-GISEL-W64:       ; %bb.0: ; %entry
-; GFX12-GISEL-W64-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX12-GISEL-W64-NEXT:    v_mov_b32_e32 v2, 0
-; GFX12-GISEL-W64-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-W64-NEXT:    global_load_tr_b128 v[0:1], v2, s[0:1] offset:32
-; GFX12-GISEL-W64-NEXT:    s_wait_loadcnt 0x0
-; GFX12-GISEL-W64-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
-; GFX12-GISEL-W64-NEXT:    s_nop 0
-; GFX12-GISEL-W64-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX12-GISEL-W64-NEXT:    s_endpgm
-entry:
-  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
-  %val = call <4 x bfloat> @llvm.amdgcn.global.load.tr.v4bf16.p1(ptr addrspace(1) %gep)
-  store <4 x bfloat> %val, ptr addrspace(1) %use
-  ret void
-}
-- 
GitLab


From 366592b4eb54d807ebecee122b7dd06031c5f4f9 Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com>
Date: Mon, 25 Mar 2024 20:27:52 -0400
Subject: [PATCH 226/404] [HLSL] prevent generation of double intrinsics via
 the builtins (#86555)

fixes #86551
closes #86552

Thanks to #86440 and #86407 it makes more sense for us to do type checks
early via Sema to prevent the generation of invalid intrinsics.
---
 clang/lib/Headers/hlsl/hlsl_intrinsics.h          |  9 ---------
 clang/lib/Sema/SemaChecking.cpp                   | 15 +++++++++++++++
 .../SemaHLSL/BuiltIns/half-float-only-errors.hlsl | 13 +++++++++++++
 clang/test/SemaHLSL/BuiltIns/pow-errors.hlsl      |  6 ++++++
 4 files changed, 34 insertions(+), 9 deletions(-)
 create mode 100644 clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl
 create mode 100644 clang/test/SemaHLSL/BuiltIns/pow-errors.hlsl

diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
index fcb64fde1b91..18472f728eef 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
@@ -392,15 +392,6 @@ float3 cos(float3);
 _HLSL_BUILTIN_ALIAS(__builtin_elementwise_cos)
 float4 cos(float4);
 
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_cos)
-double cos(double);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_cos)
-double2 cos(double2);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_cos)
-double3 cos(double3);
-_HLSL_BUILTIN_ALIAS(__builtin_elementwise_cos)
-double4 cos(double4);
-
 //===----------------------------------------------------------------------===//
 // dot product builtins
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 246e3577809a..c725ec0ef055 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -5624,6 +5624,21 @@ bool Sema::CheckHLSLBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
             TheCall, /*CheckForFloatArgs*/
             TheCall->getArg(0)->getType()->hasFloatingRepresentation()))
       return true;
+    break;
+  }
+  // Note these are llvm builtins that we want to catch invalid intrinsic
+  // generation. Normal handling of these builitns will occur elsewhere.
+  case Builtin::BI__builtin_elementwise_cos:
+  case Builtin::BI__builtin_elementwise_sin:
+  case Builtin::BI__builtin_elementwise_log:
+  case Builtin::BI__builtin_elementwise_log2:
+  case Builtin::BI__builtin_elementwise_log10:
+  case Builtin::BI__builtin_elementwise_pow:
+  case Builtin::BI__builtin_elementwise_sqrt:
+  case Builtin::BI__builtin_elementwise_trunc: {
+    if (CheckFloatOrHalfRepresentations(this, TheCall))
+      return true;
+    break;
   }
   }
   return false;
diff --git a/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl
new file mode 100644
index 000000000000..92dd06d5a330
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/half-float-only-errors.hlsl
@@ -0,0 +1,13 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_cos 
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_sin 
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_log 
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_log2 
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_log10
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_sqrt
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm -disable-llvm-passes -verify -DTEST_FUNC=__builtin_elementwise_trunc
+
+
+double2 test_double_builtin(double2 p0) {
+    return TEST_FUNC(p0);
+  // expected-error@-1 {{passing 'double2' (aka 'vector<double, 2>') to parameter of incompatible type '__attribute__((__vector_size__(2 * sizeof(float)))) float' (vector of 2 'float' values)}}
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/pow-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/pow-errors.hlsl
new file mode 100644
index 000000000000..949028aacf24
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/pow-errors.hlsl
@@ -0,0 +1,6 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -fnative-half-type -emit-llvm -disable-llvm-passes -verify
+
+double2 test_double_builtin(double2 p0, double2 p1) {
+    return __builtin_elementwise_pow(p0,p1);
+  // expected-error@-1 {{passing 'double2' (aka 'vector<double, 2>') to parameter of incompatible type '__attribute__((__vector_size__(2 * sizeof(float)))) float' (vector of 2 'float' values)}}
+}
-- 
GitLab


From c48aa781a31c76b27cadcd1b78022393c450824b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20K=C3=B6ppe?= <tkoeppe@google.com>
Date: Tue, 26 Mar 2024 00:30:49 +0000
Subject: [PATCH 227/404] Add missing declarations of explicit template
 instantiations. (#86591)

Found with `-Wundefined-func-template`.
---
 mlir/include/mlir/Analysis/Presburger/Matrix.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/mlir/include/mlir/Analysis/Presburger/Matrix.h b/mlir/include/mlir/Analysis/Presburger/Matrix.h
index 4484ebc747e6..c20a7bcecd52 100644
--- a/mlir/include/mlir/Analysis/Presburger/Matrix.h
+++ b/mlir/include/mlir/Analysis/Presburger/Matrix.h
@@ -244,6 +244,9 @@ protected:
   SmallVector<T, 16> data;
 };
 
+extern template class Matrix<MPInt>;
+extern template class Matrix<Fraction>;
+
 // An inherited class for integer matrices, with no new data attributes.
 // This is only used for the matrix-related methods which apply only
 // to integers (hermite normal form computation and row normalisation).
-- 
GitLab


From 83eb8aee4bf9d518b3a2b485640207e7717805b4 Mon Sep 17 00:00:00 2001
From: Takuto Ikuta <tikuta@google.com>
Date: Mon, 25 Mar 2024 18:04:40 -0700
Subject: [PATCH 228/404] Use timeTraceAsyncProfilerBegin for Source span
 (#83961)

This fixes incorrect trace for https://github.com/llvm/llvm-project/issues/56554.

This shows trace like

https://ui.perfetto.dev/#!/?s=aa809778dc50f2b155b062317fa18bbe2bb2fb9175e6282add8121c7c178214e
for the case shown in https://github.com/llvm/llvm-project/issues/83236.

https://github.com/llvm/llvm-project/pull/83778 is preparing PR.

Fix #56554
---
 clang/lib/Sema/Sema.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index cd0c42d5ffba..b4a47898cd17 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -135,6 +135,7 @@ namespace sema {
 class SemaPPCallbacks : public PPCallbacks {
   Sema *S = nullptr;
   llvm::SmallVector<SourceLocation, 8> IncludeStack;
+  llvm::SmallVector<llvm::TimeTraceProfilerEntry *, 8> ProfilerStack;
 
 public:
   void set(Sema &S) { this->S = &S; }
@@ -153,8 +154,8 @@ public:
       if (IncludeLoc.isValid()) {
         if (llvm::timeTraceProfilerEnabled()) {
           OptionalFileEntryRef FE = SM.getFileEntryRefForID(SM.getFileID(Loc));
-          llvm::timeTraceProfilerBegin("Source", FE ? FE->getName()
-                                                    : StringRef("<unknown>"));
+          ProfilerStack.push_back(llvm::timeTraceAsyncProfilerBegin(
+              "Source", FE ? FE->getName() : StringRef("<unknown>")));
         }
 
         IncludeStack.push_back(IncludeLoc);
@@ -167,7 +168,7 @@ public:
     case ExitFile:
       if (!IncludeStack.empty()) {
         if (llvm::timeTraceProfilerEnabled())
-          llvm::timeTraceProfilerEnd();
+          llvm::timeTraceProfilerEnd(ProfilerStack.pop_back_val());
 
         S->DiagnoseNonDefaultPragmaAlignPack(
             Sema::PragmaAlignPackDiagnoseKind::ChangedStateAtExit,
-- 
GitLab


From 8bd1f9116aab879183f34707e6d21c7051d083b6 Mon Sep 17 00:00:00 2001
From: Akira Hatanaka <ahatanak@gmail.com>
Date: Mon, 25 Mar 2024 18:05:42 -0700
Subject: [PATCH 229/404] [CodeGen][arm64e] Add methods and data members to
 Address, which are needed to authenticate signed pointers (#67454)

To authenticate pointers, CodeGen needs access to the key and
discriminators that were used to sign the pointer. That information is
sometimes known from the context, but not always, which is why `Address`
needs to hold that information.

This patch adds methods and data members to `Address`, which will be
needed in subsequent patches to authenticate signed pointers, and uses
the newly added methods throughout CodeGen. Although this patch isn't
strictly NFC as it causes CodeGen to use different code paths in some
cases (e.g., `mergeAddressesInConditionalExpr`), it doesn't cause any
changes in functionality as it doesn't add any information needed for
authentication.

In addition to the changes mentioned above, this patch introduces class
`RawAddress`, which contains a pointer that we know is unsigned, and
adds several new functions for creating `Address` and `LValue` objects.
---
 clang/lib/CodeGen/ABIInfoImpl.cpp        |  10 +-
 clang/lib/CodeGen/Address.h              | 195 ++++++++++++++---
 clang/lib/CodeGen/CGAtomic.cpp           |  53 ++---
 clang/lib/CodeGen/CGBlocks.cpp           |  34 +--
 clang/lib/CodeGen/CGBlocks.h             |   3 +-
 clang/lib/CodeGen/CGBuilder.h            | 234 ++++++++++++++-------
 clang/lib/CodeGen/CGBuiltin.cpp          | 173 +++++++--------
 clang/lib/CodeGen/CGCUDANV.cpp           |  19 +-
 clang/lib/CodeGen/CGCXXABI.cpp           |  21 +-
 clang/lib/CodeGen/CGCXXABI.h             |  14 +-
 clang/lib/CodeGen/CGCall.cpp             | 171 ++++++++-------
 clang/lib/CodeGen/CGCall.h               |   1 +
 clang/lib/CodeGen/CGClass.cpp            |  76 ++++---
 clang/lib/CodeGen/CGCleanup.cpp          | 110 ++++------
 clang/lib/CodeGen/CGCleanup.h            |   2 +-
 clang/lib/CodeGen/CGCoroutine.cpp        |   4 +-
 clang/lib/CodeGen/CGDecl.cpp             |  28 +--
 clang/lib/CodeGen/CGException.cpp        |  19 +-
 clang/lib/CodeGen/CGExpr.cpp             | 227 ++++++++++----------
 clang/lib/CodeGen/CGExprAgg.cpp          |  29 +--
 clang/lib/CodeGen/CGExprCXX.cpp          | 111 +++++-----
 clang/lib/CodeGen/CGExprConstant.cpp     |   4 +-
 clang/lib/CodeGen/CGExprScalar.cpp       |  23 +-
 clang/lib/CodeGen/CGNonTrivialStruct.cpp |   8 +-
 clang/lib/CodeGen/CGObjC.cpp             |  43 ++--
 clang/lib/CodeGen/CGObjCGNU.cpp          |  42 ++--
 clang/lib/CodeGen/CGObjCMac.cpp          |  95 ++++-----
 clang/lib/CodeGen/CGObjCRuntime.cpp      |   6 +-
 clang/lib/CodeGen/CGOpenMPRuntime.cpp    | 194 +++++++++--------
 clang/lib/CodeGen/CGOpenMPRuntime.h      |   5 +-
 clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp |  76 +++----
 clang/lib/CodeGen/CGStmt.cpp             |   8 +-
 clang/lib/CodeGen/CGStmtOpenMP.cpp       |  87 ++++----
 clang/lib/CodeGen/CGVTables.cpp          |   9 +-
 clang/lib/CodeGen/CGValue.h              | 249 +++++++++++-----------
 clang/lib/CodeGen/CodeGenFunction.cpp    |  71 ++++---
 clang/lib/CodeGen/CodeGenFunction.h      | 257 ++++++++++++++++-------
 clang/lib/CodeGen/CodeGenModule.cpp      |   2 +-
 clang/lib/CodeGen/CodeGenPGO.cpp         |  10 +-
 clang/lib/CodeGen/CodeGenPGO.h           |   6 +-
 clang/lib/CodeGen/ItaniumCXXABI.cpp      |  52 ++---
 clang/lib/CodeGen/MicrosoftCXXABI.cpp    |  58 ++---
 clang/lib/CodeGen/TargetInfo.h           |   5 +
 clang/lib/CodeGen/Targets/NVPTX.cpp      |   2 +-
 clang/lib/CodeGen/Targets/PPC.cpp        |  11 +-
 clang/lib/CodeGen/Targets/Sparc.cpp      |   2 +-
 clang/lib/CodeGen/Targets/SystemZ.cpp    |   9 +-
 clang/lib/CodeGen/Targets/XCore.cpp      |   2 +-
 clang/utils/TableGen/MveEmitter.cpp      |   2 +-
 llvm/include/llvm/IR/IRBuilder.h         |   1 +
 50 files changed, 1639 insertions(+), 1234 deletions(-)

diff --git a/clang/lib/CodeGen/ABIInfoImpl.cpp b/clang/lib/CodeGen/ABIInfoImpl.cpp
index 2b20d5a13346..1dc76065a355 100644
--- a/clang/lib/CodeGen/ABIInfoImpl.cpp
+++ b/clang/lib/CodeGen/ABIInfoImpl.cpp
@@ -187,7 +187,7 @@ CodeGen::emitVoidPtrDirectVAArg(CodeGenFunction &CGF, Address VAListAddr,
   CharUnits FullDirectSize = DirectSize.alignTo(SlotSize);
   Address NextPtr =
       CGF.Builder.CreateConstInBoundsByteGEP(Addr, FullDirectSize, "argp.next");
-  CGF.Builder.CreateStore(NextPtr.getPointer(), VAListAddr);
+  CGF.Builder.CreateStore(NextPtr.emitRawPointer(CGF), VAListAddr);
 
   // If the argument is smaller than a slot, and this is a big-endian
   // target, the argument will be right-adjusted in its slot.
@@ -239,8 +239,8 @@ Address CodeGen::emitMergePHI(CodeGenFunction &CGF, Address Addr1,
                               const llvm::Twine &Name) {
   assert(Addr1.getType() == Addr2.getType());
   llvm::PHINode *PHI = CGF.Builder.CreatePHI(Addr1.getType(), 2, Name);
-  PHI->addIncoming(Addr1.getPointer(), Block1);
-  PHI->addIncoming(Addr2.getPointer(), Block2);
+  PHI->addIncoming(Addr1.emitRawPointer(CGF), Block1);
+  PHI->addIncoming(Addr2.emitRawPointer(CGF), Block2);
   CharUnits Align = std::min(Addr1.getAlignment(), Addr2.getAlignment());
   return Address(PHI, Addr1.getElementType(), Align);
 }
@@ -400,7 +400,7 @@ Address CodeGen::EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr,
     llvm::Type *ElementTy = CGF.ConvertTypeForMem(Ty);
     llvm::Type *BaseTy = llvm::PointerType::getUnqual(ElementTy);
     llvm::Value *Addr =
-        CGF.Builder.CreateVAArg(VAListAddr.getPointer(), BaseTy);
+        CGF.Builder.CreateVAArg(VAListAddr.emitRawPointer(CGF), BaseTy);
     return Address(Addr, ElementTy, TyAlignForABI);
   } else {
     assert((AI.isDirect() || AI.isExtend()) &&
@@ -416,7 +416,7 @@ Address CodeGen::EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr,
            "Unexpected CoerceToType seen in arginfo in generic VAArg emitter!");
 
     Address Temp = CGF.CreateMemTemp(Ty, "varet");
-    Val = CGF.Builder.CreateVAArg(VAListAddr.getPointer(),
+    Val = CGF.Builder.CreateVAArg(VAListAddr.emitRawPointer(CGF),
                                   CGF.ConvertTypeForMem(Ty));
     CGF.Builder.CreateStore(Val, Temp);
     return Temp;
diff --git a/clang/lib/CodeGen/Address.h b/clang/lib/CodeGen/Address.h
index cf48df8f5e73..35ec370a139c 100644
--- a/clang/lib/CodeGen/Address.h
+++ b/clang/lib/CodeGen/Address.h
@@ -15,6 +15,7 @@
 #define LLVM_CLANG_LIB_CODEGEN_ADDRESS_H
 
 #include "clang/AST/CharUnits.h"
+#include "clang/AST/Type.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/Support/MathExtras.h"
@@ -22,28 +23,41 @@
 namespace clang {
 namespace CodeGen {
 
+class Address;
+class CGBuilderTy;
+class CodeGenFunction;
+class CodeGenModule;
+
 // Indicates whether a pointer is known not to be null.
 enum KnownNonNull_t { NotKnownNonNull, KnownNonNull };
 
-/// An aligned address.
-class Address {
+/// An abstract representation of an aligned address. This is designed to be an
+/// IR-level abstraction, carrying just the information necessary to perform IR
+/// operations on an address like loads and stores.  In particular, it doesn't
+/// carry C type information or allow the representation of things like
+/// bit-fields; clients working at that level should generally be using
+/// `LValue`.
+/// The pointer contained in this class is known to be unsigned.
+class RawAddress {
   llvm::PointerIntPair<llvm::Value *, 1, bool> PointerAndKnownNonNull;
   llvm::Type *ElementType;
   CharUnits Alignment;
 
 protected:
-  Address(std::nullptr_t) : ElementType(nullptr) {}
+  RawAddress(std::nullptr_t) : ElementType(nullptr) {}
 
 public:
-  Address(llvm::Value *Pointer, llvm::Type *ElementType, CharUnits Alignment,
-          KnownNonNull_t IsKnownNonNull = NotKnownNonNull)
+  RawAddress(llvm::Value *Pointer, llvm::Type *ElementType, CharUnits Alignment,
+             KnownNonNull_t IsKnownNonNull = NotKnownNonNull)
       : PointerAndKnownNonNull(Pointer, IsKnownNonNull),
         ElementType(ElementType), Alignment(Alignment) {
     assert(Pointer != nullptr && "Pointer cannot be null");
     assert(ElementType != nullptr && "Element type cannot be null");
   }
 
-  static Address invalid() { return Address(nullptr); }
+  inline RawAddress(Address Addr);
+
+  static RawAddress invalid() { return RawAddress(nullptr); }
   bool isValid() const {
     return PointerAndKnownNonNull.getPointer() != nullptr;
   }
@@ -80,6 +94,133 @@ public:
     return Alignment;
   }
 
+  /// Return address with different element type, but same pointer and
+  /// alignment.
+  RawAddress withElementType(llvm::Type *ElemTy) const {
+    return RawAddress(getPointer(), ElemTy, getAlignment(), isKnownNonNull());
+  }
+
+  KnownNonNull_t isKnownNonNull() const {
+    assert(isValid());
+    return (KnownNonNull_t)PointerAndKnownNonNull.getInt();
+  }
+};
+
+/// Like RawAddress, an abstract representation of an aligned address, but the
+/// pointer contained in this class is possibly signed.
+class Address {
+  friend class CGBuilderTy;
+
+  // The boolean flag indicates whether the pointer is known to be non-null.
+  llvm::PointerIntPair<llvm::Value *, 1, bool> Pointer;
+
+  /// The expected IR type of the pointer. Carrying accurate element type
+  /// information in Address makes it more convenient to work with Address
+  /// values and allows frontend assertions to catch simple mistakes.
+  llvm::Type *ElementType = nullptr;
+
+  CharUnits Alignment;
+
+  /// Offset from the base pointer.
+  llvm::Value *Offset = nullptr;
+
+  llvm::Value *emitRawPointerSlow(CodeGenFunction &CGF) const;
+
+protected:
+  Address(std::nullptr_t) : ElementType(nullptr) {}
+
+public:
+  Address(llvm::Value *pointer, llvm::Type *elementType, CharUnits alignment,
+          KnownNonNull_t IsKnownNonNull = NotKnownNonNull)
+      : Pointer(pointer, IsKnownNonNull), ElementType(elementType),
+        Alignment(alignment) {
+    assert(pointer != nullptr && "Pointer cannot be null");
+    assert(elementType != nullptr && "Element type cannot be null");
+    assert(!alignment.isZero() && "Alignment cannot be zero");
+  }
+
+  Address(llvm::Value *BasePtr, llvm::Type *ElementType, CharUnits Alignment,
+          llvm::Value *Offset, KnownNonNull_t IsKnownNonNull = NotKnownNonNull)
+      : Pointer(BasePtr, IsKnownNonNull), ElementType(ElementType),
+        Alignment(Alignment), Offset(Offset) {}
+
+  Address(RawAddress RawAddr)
+      : Pointer(RawAddr.isValid() ? RawAddr.getPointer() : nullptr),
+        ElementType(RawAddr.isValid() ? RawAddr.getElementType() : nullptr),
+        Alignment(RawAddr.isValid() ? RawAddr.getAlignment()
+                                    : CharUnits::Zero()) {}
+
+  static Address invalid() { return Address(nullptr); }
+  bool isValid() const { return Pointer.getPointer() != nullptr; }
+
+  /// This function is used in situations where the caller is doing some sort of
+  /// opaque "laundering" of the pointer.
+  void replaceBasePointer(llvm::Value *P) {
+    assert(isValid() && "pointer isn't valid");
+    assert(P->getType() == Pointer.getPointer()->getType() &&
+           "Pointer's type changed");
+    Pointer.setPointer(P);
+    assert(isValid() && "pointer is invalid after replacement");
+  }
+
+  CharUnits getAlignment() const { return Alignment; }
+
+  void setAlignment(CharUnits Value) { Alignment = Value; }
+
+  llvm::Value *getBasePointer() const {
+    assert(isValid() && "pointer isn't valid");
+    return Pointer.getPointer();
+  }
+
+  /// Return the type of the pointer value.
+  llvm::PointerType *getType() const {
+    return llvm::PointerType::get(
+        ElementType,
+        llvm::cast<llvm::PointerType>(Pointer.getPointer()->getType())
+            ->getAddressSpace());
+  }
+
+  /// Return the type of the values stored in this address.
+  llvm::Type *getElementType() const {
+    assert(isValid());
+    return ElementType;
+  }
+
+  /// Return the address space that this address resides in.
+  unsigned getAddressSpace() const { return getType()->getAddressSpace(); }
+
+  /// Return the IR name of the pointer value.
+  llvm::StringRef getName() const { return Pointer.getPointer()->getName(); }
+
+  // This function is called only in CGBuilderBaseTy::CreateElementBitCast.
+  void setElementType(llvm::Type *Ty) {
+    assert(hasOffset() &&
+           "this funcion shouldn't be called when there is no offset");
+    ElementType = Ty;
+  }
+
+  /// Whether the pointer is known not to be null.
+  KnownNonNull_t isKnownNonNull() const {
+    assert(isValid());
+    return (KnownNonNull_t)Pointer.getInt();
+  }
+
+  Address setKnownNonNull() {
+    assert(isValid());
+    Pointer.setInt(KnownNonNull);
+    return *this;
+  }
+
+  bool hasOffset() const { return Offset; }
+
+  llvm::Value *getOffset() const { return Offset; }
+
+  /// Return the pointer contained in this class after authenticating it and
+  /// adding offset to it if necessary.
+  llvm::Value *emitRawPointer(CodeGenFunction &CGF) const {
+    return getBasePointer();
+  }
+
   /// Return address with different pointer, but same element type and
   /// alignment.
   Address withPointer(llvm::Value *NewPointer,
@@ -91,61 +232,59 @@ public:
   /// Return address with different alignment, but same pointer and element
   /// type.
   Address withAlignment(CharUnits NewAlignment) const {
-    return Address(getPointer(), getElementType(), NewAlignment,
+    return Address(Pointer.getPointer(), getElementType(), NewAlignment,
                    isKnownNonNull());
   }
 
   /// Return address with different element type, but same pointer and
   /// alignment.
   Address withElementType(llvm::Type *ElemTy) const {
-    return Address(getPointer(), ElemTy, getAlignment(), isKnownNonNull());
-  }
-
-  /// Whether the pointer is known not to be null.
-  KnownNonNull_t isKnownNonNull() const {
-    assert(isValid());
-    return (KnownNonNull_t)PointerAndKnownNonNull.getInt();
-  }
-
-  /// Set the non-null bit.
-  Address setKnownNonNull() {
-    assert(isValid());
-    PointerAndKnownNonNull.setInt(true);
-    return *this;
+    if (!hasOffset())
+      return Address(getBasePointer(), ElemTy, getAlignment(), nullptr,
+                     isKnownNonNull());
+    Address A(*this);
+    A.ElementType = ElemTy;
+    return A;
   }
 };
 
+inline RawAddress::RawAddress(Address Addr)
+    : PointerAndKnownNonNull(Addr.isValid() ? Addr.getBasePointer() : nullptr,
+                             Addr.isValid() ? Addr.isKnownNonNull()
+                                            : NotKnownNonNull),
+      ElementType(Addr.isValid() ? Addr.getElementType() : nullptr),
+      Alignment(Addr.isValid() ? Addr.getAlignment() : CharUnits::Zero()) {}
+
 /// A specialization of Address that requires the address to be an
 /// LLVM Constant.
-class ConstantAddress : public Address {
-  ConstantAddress(std::nullptr_t) : Address(nullptr) {}
+class ConstantAddress : public RawAddress {
+  ConstantAddress(std::nullptr_t) : RawAddress(nullptr) {}
 
 public:
   ConstantAddress(llvm::Constant *pointer, llvm::Type *elementType,
                   CharUnits alignment)
-      : Address(pointer, elementType, alignment) {}
+      : RawAddress(pointer, elementType, alignment) {}
 
   static ConstantAddress invalid() {
     return ConstantAddress(nullptr);
   }
 
   llvm::Constant *getPointer() const {
-    return llvm::cast<llvm::Constant>(Address::getPointer());
+    return llvm::cast<llvm::Constant>(RawAddress::getPointer());
   }
 
   ConstantAddress withElementType(llvm::Type *ElemTy) const {
     return ConstantAddress(getPointer(), ElemTy, getAlignment());
   }
 
-  static bool isaImpl(Address addr) {
+  static bool isaImpl(RawAddress addr) {
     return llvm::isa<llvm::Constant>(addr.getPointer());
   }
-  static ConstantAddress castImpl(Address addr) {
+  static ConstantAddress castImpl(RawAddress addr) {
     return ConstantAddress(llvm::cast<llvm::Constant>(addr.getPointer()),
                            addr.getElementType(), addr.getAlignment());
   }
 };
-
 }
 
 // Present a minimal LLVM-like casting interface.
diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp
index fb03d013e8af..56198385de9d 100644
--- a/clang/lib/CodeGen/CGAtomic.cpp
+++ b/clang/lib/CodeGen/CGAtomic.cpp
@@ -80,7 +80,7 @@ namespace {
         AtomicSizeInBits = C.toBits(
             C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
                 .alignTo(lvalue.getAlignment()));
-        llvm::Value *BitFieldPtr = lvalue.getBitFieldPointer();
+        llvm::Value *BitFieldPtr = lvalue.getRawBitFieldPointer(CGF);
         auto OffsetInChars =
             (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
             lvalue.getAlignment();
@@ -139,13 +139,13 @@ namespace {
     const LValue &getAtomicLValue() const { return LVal; }
     llvm::Value *getAtomicPointer() const {
       if (LVal.isSimple())
-        return LVal.getPointer(CGF);
+        return LVal.emitRawPointer(CGF);
       else if (LVal.isBitField())
-        return LVal.getBitFieldPointer();
+        return LVal.getRawBitFieldPointer(CGF);
       else if (LVal.isVectorElt())
-        return LVal.getVectorPointer();
+        return LVal.getRawVectorPointer(CGF);
       assert(LVal.isExtVectorElt());
-      return LVal.getExtVectorPointer();
+      return LVal.getRawExtVectorPointer(CGF);
     }
     Address getAtomicAddress() const {
       llvm::Type *ElTy;
@@ -368,7 +368,7 @@ bool AtomicInfo::emitMemSetZeroIfNecessary() const {
     return false;
 
   CGF.Builder.CreateMemSet(
-      addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0),
+      addr.emitRawPointer(CGF), llvm::ConstantInt::get(CGF.Int8Ty, 0),
       CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
       LVal.getAlignment().getAsAlign());
   return true;
@@ -1055,7 +1055,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
       return getTargetHooks().performAddrSpaceCast(
           *this, V, AS, LangAS::opencl_generic, DestType, false);
     };
-    Args.add(RValue::get(CastToGenericAddrSpace(Ptr.getPointer(),
+
+    Args.add(RValue::get(CastToGenericAddrSpace(Ptr.emitRawPointer(*this),
                                                 E->getPtr()->getType())),
              getContext().VoidPtrTy);
 
@@ -1086,10 +1087,10 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
       LibCallName = "__atomic_compare_exchange";
       RetTy = getContext().BoolTy;
       HaveRetTy = true;
-      Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(),
+      Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this),
                                                   E->getVal1()->getType())),
                getContext().VoidPtrTy);
-      Args.add(RValue::get(CastToGenericAddrSpace(Val2.getPointer(),
+      Args.add(RValue::get(CastToGenericAddrSpace(Val2.emitRawPointer(*this),
                                                   E->getVal2()->getType())),
                getContext().VoidPtrTy);
       Args.add(RValue::get(Order), getContext().IntTy);
@@ -1105,7 +1106,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
     case AtomicExpr::AO__scoped_atomic_exchange:
     case AtomicExpr::AO__scoped_atomic_exchange_n:
       LibCallName = "__atomic_exchange";
-      Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(),
+      Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this),
                                                   E->getVal1()->getType())),
                getContext().VoidPtrTy);
       break;
@@ -1120,7 +1121,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
       LibCallName = "__atomic_store";
       RetTy = getContext().VoidTy;
       HaveRetTy = true;
-      Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(),
+      Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this),
                                                   E->getVal1()->getType())),
                getContext().VoidPtrTy);
       break;
@@ -1199,7 +1200,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
     if (!HaveRetTy) {
       // Value is returned through parameter before the order.
       RetTy = getContext().VoidTy;
-      Args.add(RValue::get(CastToGenericAddrSpace(Dest.getPointer(), RetTy)),
+      Args.add(RValue::get(
+                   CastToGenericAddrSpace(Dest.emitRawPointer(*this), RetTy)),
                getContext().VoidPtrTy);
     }
     // Order is always the last parameter.
@@ -1513,7 +1515,7 @@ RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
     } else
       TempAddr = CreateTempAlloca();
 
-    EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
+    EmitAtomicLoadLibcall(TempAddr.emitRawPointer(CGF), AO, IsVolatile);
 
     // Okay, turn that back into the original value or whole atomic (for
     // non-simple lvalues) type.
@@ -1673,9 +1675,9 @@ std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
   if (shouldUseLibcall()) {
     // Produce a source address.
     Address ExpectedAddr = materializeRValue(Expected);
-    Address DesiredAddr = materializeRValue(Desired);
-    auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
-                                                 DesiredAddr.getPointer(),
+    llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
+    llvm::Value *DesiredPtr = materializeRValue(Desired).emitRawPointer(CGF);
+    auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr,
                                                  Success, Failure);
     return std::make_pair(
         convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
@@ -1757,7 +1759,7 @@ void AtomicInfo::EmitAtomicUpdateLibcall(
 
   Address ExpectedAddr = CreateTempAlloca();
 
-  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
+  EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile);
   auto *ContBB = CGF.createBasicBlock("atomic_cont");
   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
   CGF.EmitBlock(ContBB);
@@ -1771,10 +1773,10 @@ void AtomicInfo::EmitAtomicUpdateLibcall(
                                            AggValueSlot::ignored(),
                                            SourceLocation(), /*AsValue=*/false);
   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
+  llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
+  llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF);
   auto *Res =
-      EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
-                                       DesiredAddr.getPointer(),
-                                       AO, Failure);
+      EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure);
   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
 }
@@ -1843,7 +1845,7 @@ void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
 
   Address ExpectedAddr = CreateTempAlloca();
 
-  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
+  EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile);
   auto *ContBB = CGF.createBasicBlock("atomic_cont");
   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
   CGF.EmitBlock(ContBB);
@@ -1854,10 +1856,10 @@ void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
     CGF.Builder.CreateStore(OldVal, DesiredAddr);
   }
   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
+  llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
+  llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF);
   auto *Res =
-      EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
-                                       DesiredAddr.getPointer(),
-                                       AO, Failure);
+      EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure);
   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
 }
@@ -1957,7 +1959,8 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
       args.add(RValue::get(atomics.getAtomicSizeValue()),
                getContext().getSizeType());
       args.add(RValue::get(atomics.getAtomicPointer()), getContext().VoidPtrTy);
-      args.add(RValue::get(srcAddr.getPointer()), getContext().VoidPtrTy);
+      args.add(RValue::get(srcAddr.emitRawPointer(*this)),
+               getContext().VoidPtrTy);
       args.add(
           RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
           getContext().IntTy);
diff --git a/clang/lib/CodeGen/CGBlocks.cpp b/clang/lib/CodeGen/CGBlocks.cpp
index ad0b50d79961..a01f2c7c9798 100644
--- a/clang/lib/CodeGen/CGBlocks.cpp
+++ b/clang/lib/CodeGen/CGBlocks.cpp
@@ -36,7 +36,8 @@ CGBlockInfo::CGBlockInfo(const BlockDecl *block, StringRef name)
     : Name(name), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false),
       NoEscape(false), HasCXXObject(false), UsesStret(false),
       HasCapturedVariableLayout(false), CapturesNonExternalType(false),
-      LocalAddress(Address::invalid()), StructureType(nullptr), Block(block) {
+      LocalAddress(RawAddress::invalid()), StructureType(nullptr),
+      Block(block) {
 
   // Skip asm prefix, if any.  'name' is usually taken directly from
   // the mangled name of the enclosing function.
@@ -794,7 +795,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
 
   // Otherwise, we have to emit this as a local block.
 
-  Address blockAddr = blockInfo.LocalAddress;
+  RawAddress blockAddr = blockInfo.LocalAddress;
   assert(blockAddr.isValid() && "block has no address!");
 
   llvm::Constant *isa;
@@ -939,7 +940,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
       if (CI.isNested())
         byrefPointer = Builder.CreateLoad(src, "byref.capture");
       else
-        byrefPointer = src.getPointer();
+        byrefPointer = src.emitRawPointer(*this);
 
       // Write that void* into the capture field.
       Builder.CreateStore(byrefPointer, blockField);
@@ -961,10 +962,10 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
       }
 
     // If it's a reference variable, copy the reference into the block field.
-    } else if (type->isReferenceType()) {
-      Builder.CreateStore(src.getPointer(), blockField);
+    } else if (auto refType = type->getAs<ReferenceType>()) {
+      Builder.CreateStore(src.emitRawPointer(*this), blockField);
 
-    // If type is const-qualified, copy the value into the block field.
+      // If type is const-qualified, copy the value into the block field.
     } else if (type.isConstQualified() &&
                type.getObjCLifetime() == Qualifiers::OCL_Strong &&
                CGM.getCodeGenOpts().OptimizationLevel != 0) {
@@ -1377,7 +1378,7 @@ void CodeGenFunction::setBlockContextParameter(const ImplicitParamDecl *D,
 
   // Allocate a stack slot like for any local variable to guarantee optimal
   // debug info at -O0. The mem2reg pass will eliminate it when optimizing.
-  Address alloc = CreateMemTemp(D->getType(), D->getName() + ".addr");
+  RawAddress alloc = CreateMemTemp(D->getType(), D->getName() + ".addr");
   Builder.CreateStore(arg, alloc);
   if (CGDebugInfo *DI = getDebugInfo()) {
     if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
@@ -1497,7 +1498,7 @@ llvm::Function *CodeGenFunction::GenerateBlockFunction(
     // frame setup instruction by llvm::DwarfDebug::beginFunction().
     auto NL = ApplyDebugLocation::CreateEmpty(*this);
     Builder.CreateStore(BlockPointer, Alloca);
-    BlockPointerDbgLoc = Alloca.getPointer();
+    BlockPointerDbgLoc = Alloca.emitRawPointer(*this);
   }
 
   // If we have a C++ 'this' reference, go ahead and force it into
@@ -1557,8 +1558,8 @@ llvm::Function *CodeGenFunction::GenerateBlockFunction(
         const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);
         if (capture.isConstant()) {
           auto addr = LocalDeclMap.find(variable)->second;
-          (void)DI->EmitDeclareOfAutoVariable(variable, addr.getPointer(),
-                                              Builder);
+          (void)DI->EmitDeclareOfAutoVariable(
+              variable, addr.emitRawPointer(*this), Builder);
           continue;
         }
 
@@ -1662,7 +1663,7 @@ struct CallBlockRelease final : EHScopeStack::Cleanup {
     if (LoadBlockVarAddr) {
       BlockVarAddr = CGF.Builder.CreateLoad(Addr);
     } else {
-      BlockVarAddr = Addr.getPointer();
+      BlockVarAddr = Addr.emitRawPointer(CGF);
     }
 
     CGF.BuildBlockRelease(BlockVarAddr, FieldFlags, CanThrow);
@@ -1962,13 +1963,15 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
         // it. It's not quite worth the annoyance to avoid creating it in the
         // first place.
         if (!needsEHCleanup(captureType.isDestructedType()))
-          cast<llvm::Instruction>(dstField.getPointer())->eraseFromParent();
+          if (auto *I =
+                  cast_or_null<llvm::Instruction>(dstField.getBasePointer()))
+            I->eraseFromParent();
       }
       break;
     }
     case BlockCaptureEntityKind::BlockObject: {
       llvm::Value *srcValue = Builder.CreateLoad(srcField, "blockcopy.src");
-      llvm::Value *dstAddr = dstField.getPointer();
+      llvm::Value *dstAddr = dstField.emitRawPointer(*this);
       llvm::Value *args[] = {
         dstAddr, srcValue, llvm::ConstantInt::get(Int32Ty, flags.getBitMask())
       };
@@ -2139,7 +2142,7 @@ public:
     llvm::Value *flagsVal = llvm::ConstantInt::get(CGF.Int32Ty, flags);
     llvm::FunctionCallee fn = CGF.CGM.getBlockObjectAssign();
 
-    llvm::Value *args[] = { destField.getPointer(), srcValue, flagsVal };
+    llvm::Value *args[] = {destField.emitRawPointer(CGF), srcValue, flagsVal};
     CGF.EmitNounwindRuntimeCall(fn, args);
   }
 
@@ -2696,7 +2699,8 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) {
   storeHeaderField(V, getPointerSize(), "byref.isa");
 
   // Store the address of the variable into its own forwarding pointer.
-  storeHeaderField(addr.getPointer(), getPointerSize(), "byref.forwarding");
+  storeHeaderField(addr.emitRawPointer(*this), getPointerSize(),
+                   "byref.forwarding");
 
   // Blocks ABI:
   //   c) the flags field is set to either 0 if no helper functions are
diff --git a/clang/lib/CodeGen/CGBlocks.h b/clang/lib/CodeGen/CGBlocks.h
index 4ef1ae9f3365..8d10c4f69b20 100644
--- a/clang/lib/CodeGen/CGBlocks.h
+++ b/clang/lib/CodeGen/CGBlocks.h
@@ -271,7 +271,8 @@ public:
   /// The block's captures. Non-constant captures are sorted by their offsets.
   llvm::SmallVector<Capture, 4> SortedCaptures;
 
-  Address LocalAddress;
+  // Currently we assume that block-pointer types are never signed.
+  RawAddress LocalAddress;
   llvm::StructType *StructureType;
   const BlockDecl *Block;
   const BlockExpr *BlockExpression;
diff --git a/clang/lib/CodeGen/CGBuilder.h b/clang/lib/CodeGen/CGBuilder.h
index bf5ab171d720..6dd9da7c4cad 100644
--- a/clang/lib/CodeGen/CGBuilder.h
+++ b/clang/lib/CodeGen/CGBuilder.h
@@ -10,7 +10,9 @@
 #define LLVM_CLANG_LIB_CODEGEN_CGBUILDER_H
 
 #include "Address.h"
+#include "CGValue.h"
 #include "CodeGenTypeCache.h"
+#include "llvm/Analysis/Utils/Local.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Type.h"
@@ -18,12 +20,15 @@
 namespace clang {
 namespace CodeGen {
 
+class CGBuilderTy;
 class CodeGenFunction;
 
 /// This is an IRBuilder insertion helper that forwards to
 /// CodeGenFunction::InsertHelper, which adds necessary metadata to
 /// instructions.
 class CGBuilderInserter final : public llvm::IRBuilderDefaultInserter {
+  friend CGBuilderTy;
+
 public:
   CGBuilderInserter() = default;
   explicit CGBuilderInserter(CodeGenFunction *CGF) : CGF(CGF) {}
@@ -43,10 +48,42 @@ typedef llvm::IRBuilder<llvm::ConstantFolder, CGBuilderInserterTy>
     CGBuilderBaseTy;
 
 class CGBuilderTy : public CGBuilderBaseTy {
+  friend class Address;
+
   /// Storing a reference to the type cache here makes it a lot easier
   /// to build natural-feeling, target-specific IR.
   const CodeGenTypeCache &TypeCache;
 
+  CodeGenFunction *getCGF() const { return getInserter().CGF; }
+
+  llvm::Value *emitRawPointerFromAddress(Address Addr) const {
+    return Addr.getBasePointer();
+  }
+
+  template <bool IsInBounds>
+  Address createConstGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1,
+                             const llvm::Twine &Name) {
+    const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout();
+    llvm::GetElementPtrInst *GEP;
+    if (IsInBounds)
+      GEP = cast<llvm::GetElementPtrInst>(CreateConstInBoundsGEP2_32(
+          Addr.getElementType(), emitRawPointerFromAddress(Addr), Idx0, Idx1,
+          Name));
+    else
+      GEP = cast<llvm::GetElementPtrInst>(CreateConstGEP2_32(
+          Addr.getElementType(), emitRawPointerFromAddress(Addr), Idx0, Idx1,
+          Name));
+    llvm::APInt Offset(
+        DL.getIndexSizeInBits(Addr.getType()->getPointerAddressSpace()), 0,
+        /*isSigned=*/true);
+    if (!GEP->accumulateConstantOffset(DL, Offset))
+      llvm_unreachable("offset of GEP with constants is always computable");
+    return Address(GEP, GEP->getResultElementType(),
+                   Addr.getAlignment().alignmentAtOffset(
+                       CharUnits::fromQuantity(Offset.getSExtValue())),
+                   IsInBounds ? Addr.isKnownNonNull() : NotKnownNonNull);
+  }
+
 public:
   CGBuilderTy(const CodeGenTypeCache &TypeCache, llvm::LLVMContext &C)
       : CGBuilderBaseTy(C), TypeCache(TypeCache) {}
@@ -69,20 +106,22 @@ public:
   // Note that we intentionally hide the CreateLoad APIs that don't
   // take an alignment.
   llvm::LoadInst *CreateLoad(Address Addr, const llvm::Twine &Name = "") {
-    return CreateAlignedLoad(Addr.getElementType(), Addr.getPointer(),
+    return CreateAlignedLoad(Addr.getElementType(),
+                             emitRawPointerFromAddress(Addr),
                              Addr.getAlignment().getAsAlign(), Name);
   }
   llvm::LoadInst *CreateLoad(Address Addr, const char *Name) {
     // This overload is required to prevent string literals from
     // ending up in the IsVolatile overload.
-    return CreateAlignedLoad(Addr.getElementType(), Addr.getPointer(),
+    return CreateAlignedLoad(Addr.getElementType(),
+                             emitRawPointerFromAddress(Addr),
                              Addr.getAlignment().getAsAlign(), Name);
   }
   llvm::LoadInst *CreateLoad(Address Addr, bool IsVolatile,
                              const llvm::Twine &Name = "") {
-    return CreateAlignedLoad(Addr.getElementType(), Addr.getPointer(),
-                             Addr.getAlignment().getAsAlign(), IsVolatile,
-                             Name);
+    return CreateAlignedLoad(
+        Addr.getElementType(), emitRawPointerFromAddress(Addr),
+        Addr.getAlignment().getAsAlign(), IsVolatile, Name);
   }
 
   using CGBuilderBaseTy::CreateAlignedLoad;
@@ -96,7 +135,7 @@ public:
   // take an alignment.
   llvm::StoreInst *CreateStore(llvm::Value *Val, Address Addr,
                                bool IsVolatile = false) {
-    return CreateAlignedStore(Val, Addr.getPointer(),
+    return CreateAlignedStore(Val, emitRawPointerFromAddress(Addr),
                               Addr.getAlignment().getAsAlign(), IsVolatile);
   }
 
@@ -132,33 +171,41 @@ public:
                       llvm::AtomicOrdering FailureOrdering,
                       llvm::SyncScope::ID SSID = llvm::SyncScope::System) {
     return CGBuilderBaseTy::CreateAtomicCmpXchg(
-        Addr.getPointer(), Cmp, New, Addr.getAlignment().getAsAlign(),
-        SuccessOrdering, FailureOrdering, SSID);
+        Addr.emitRawPointer(*getCGF()), Cmp, New,
+        Addr.getAlignment().getAsAlign(), SuccessOrdering, FailureOrdering,
+        SSID);
   }
 
   llvm::AtomicRMWInst *
   CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val,
                   llvm::AtomicOrdering Ordering,
                   llvm::SyncScope::ID SSID = llvm::SyncScope::System) {
-    return CGBuilderBaseTy::CreateAtomicRMW(Op, Addr.getPointer(), Val,
-                                            Addr.getAlignment().getAsAlign(),
-                                            Ordering, SSID);
+    return CGBuilderBaseTy::CreateAtomicRMW(
+        Op, Addr.emitRawPointer(*getCGF()), Val,
+        Addr.getAlignment().getAsAlign(), Ordering, SSID);
   }
 
   using CGBuilderBaseTy::CreateAddrSpaceCast;
   Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty,
+                              llvm::Type *ElementTy,
                               const llvm::Twine &Name = "") {
-    return Addr.withPointer(CreateAddrSpaceCast(Addr.getPointer(), Ty, Name),
-                            Addr.isKnownNonNull());
+    if (!Addr.hasOffset())
+      return Address(CreateAddrSpaceCast(Addr.getBasePointer(), Ty, Name),
+                     ElementTy, Addr.getAlignment(), nullptr,
+                     Addr.isKnownNonNull());
+    // Eagerly force a raw address if these is an offset.
+    return RawAddress(
+        CreateAddrSpaceCast(Addr.emitRawPointer(*getCGF()), Ty, Name),
+        ElementTy, Addr.getAlignment(), Addr.isKnownNonNull());
   }
 
   using CGBuilderBaseTy::CreatePointerBitCastOrAddrSpaceCast;
   Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty,
                                               llvm::Type *ElementTy,
                                               const llvm::Twine &Name = "") {
-    llvm::Value *Ptr =
-        CreatePointerBitCastOrAddrSpaceCast(Addr.getPointer(), Ty, Name);
-    return Address(Ptr, ElementTy, Addr.getAlignment(), Addr.isKnownNonNull());
+    if (Addr.getType()->getAddressSpace() == Ty->getPointerAddressSpace())
+      return Addr.withElementType(ElementTy);
+    return CreateAddrSpaceCast(Addr, Ty, ElementTy, Name);
   }
 
   /// Given
@@ -176,10 +223,11 @@ public:
     const llvm::StructLayout *Layout = DL.getStructLayout(ElTy);
     auto Offset = CharUnits::fromQuantity(Layout->getElementOffset(Index));
 
-    return Address(
-        CreateStructGEP(Addr.getElementType(), Addr.getPointer(), Index, Name),
-        ElTy->getElementType(Index),
-        Addr.getAlignment().alignmentAtOffset(Offset), Addr.isKnownNonNull());
+    return Address(CreateStructGEP(Addr.getElementType(), Addr.getBasePointer(),
+                                   Index, Name),
+                   ElTy->getElementType(Index),
+                   Addr.getAlignment().alignmentAtOffset(Offset),
+                   Addr.isKnownNonNull());
   }
 
   /// Given
@@ -198,7 +246,7 @@ public:
         CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy->getElementType()));
 
     return Address(
-        CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(),
+        CreateInBoundsGEP(Addr.getElementType(), Addr.getBasePointer(),
                           {getSize(CharUnits::Zero()), getSize(Index)}, Name),
         ElTy->getElementType(),
         Addr.getAlignment().alignmentAtOffset(Index * EltSize),
@@ -216,10 +264,10 @@ public:
     const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout();
     CharUnits EltSize = CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy));
 
-    return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(),
-                                     getSize(Index), Name),
-                   ElTy, Addr.getAlignment().alignmentAtOffset(Index * EltSize),
-                   Addr.isKnownNonNull());
+    return Address(
+        CreateInBoundsGEP(ElTy, Addr.getBasePointer(), getSize(Index), Name),
+        ElTy, Addr.getAlignment().alignmentAtOffset(Index * EltSize),
+        Addr.isKnownNonNull());
   }
 
   /// Given
@@ -229,110 +277,133 @@ public:
   /// where i64 is actually the target word size.
   Address CreateConstGEP(Address Addr, uint64_t Index,
                          const llvm::Twine &Name = "") {
+    llvm::Type *ElTy = Addr.getElementType();
     const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout();
-    CharUnits EltSize =
-        CharUnits::fromQuantity(DL.getTypeAllocSize(Addr.getElementType()));
+    CharUnits EltSize = CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy));
 
-    return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(),
-                             getSize(Index), Name),
+    return Address(CreateGEP(ElTy, Addr.getBasePointer(), getSize(Index), Name),
                    Addr.getElementType(),
-                   Addr.getAlignment().alignmentAtOffset(Index * EltSize),
-                   NotKnownNonNull);
+                   Addr.getAlignment().alignmentAtOffset(Index * EltSize));
   }
 
   /// Create GEP with single dynamic index. The address alignment is reduced
   /// according to the element size.
   using CGBuilderBaseTy::CreateGEP;
-  Address CreateGEP(Address Addr, llvm::Value *Index,
+  Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index,
                     const llvm::Twine &Name = "") {
     const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout();
     CharUnits EltSize =
         CharUnits::fromQuantity(DL.getTypeAllocSize(Addr.getElementType()));
 
     return Address(
-        CreateGEP(Addr.getElementType(), Addr.getPointer(), Index, Name),
+        CreateGEP(Addr.getElementType(), Addr.emitRawPointer(CGF), Index, Name),
         Addr.getElementType(),
-        Addr.getAlignment().alignmentOfArrayElement(EltSize), NotKnownNonNull);
+        Addr.getAlignment().alignmentOfArrayElement(EltSize));
   }
 
   /// Given a pointer to i8, adjust it by a given constant offset.
   Address CreateConstInBoundsByteGEP(Address Addr, CharUnits Offset,
                                      const llvm::Twine &Name = "") {
     assert(Addr.getElementType() == TypeCache.Int8Ty);
-    return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(),
-                                     getSize(Offset), Name),
-                   Addr.getElementType(),
-                   Addr.getAlignment().alignmentAtOffset(Offset),
-                   Addr.isKnownNonNull());
+    return Address(
+        CreateInBoundsGEP(Addr.getElementType(), Addr.getBasePointer(),
+                          getSize(Offset), Name),
+        Addr.getElementType(), Addr.getAlignment().alignmentAtOffset(Offset),
+        Addr.isKnownNonNull());
   }
+
   Address CreateConstByteGEP(Address Addr, CharUnits Offset,
                              const llvm::Twine &Name = "") {
     assert(Addr.getElementType() == TypeCache.Int8Ty);
-    return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(),
+    return Address(CreateGEP(Addr.getElementType(), Addr.getBasePointer(),
                              getSize(Offset), Name),
                    Addr.getElementType(),
-                   Addr.getAlignment().alignmentAtOffset(Offset),
-                   NotKnownNonNull);
+                   Addr.getAlignment().alignmentAtOffset(Offset));
   }
 
   using CGBuilderBaseTy::CreateConstInBoundsGEP2_32;
   Address CreateConstInBoundsGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1,
                                      const llvm::Twine &Name = "") {
-    const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout();
+    return createConstGEP2_32<true>(Addr, Idx0, Idx1, Name);
+  }
 
-    auto *GEP = cast<llvm::GetElementPtrInst>(CreateConstInBoundsGEP2_32(
-        Addr.getElementType(), Addr.getPointer(), Idx0, Idx1, Name));
-    llvm::APInt Offset(
-        DL.getIndexSizeInBits(Addr.getType()->getPointerAddressSpace()), 0,
-        /*isSigned=*/true);
-    if (!GEP->accumulateConstantOffset(DL, Offset))
-      llvm_unreachable("offset of GEP with constants is always computable");
-    return Address(GEP, GEP->getResultElementType(),
-                   Addr.getAlignment().alignmentAtOffset(
-                       CharUnits::fromQuantity(Offset.getSExtValue())),
-                   Addr.isKnownNonNull());
+  using CGBuilderBaseTy::CreateConstGEP2_32;
+  Address CreateConstGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1,
+                             const llvm::Twine &Name = "") {
+    return createConstGEP2_32<false>(Addr, Idx0, Idx1, Name);
+  }
+
+  Address CreateGEP(Address Addr, ArrayRef<llvm::Value *> IdxList,
+                    llvm::Type *ElementType, CharUnits Align,
+                    const Twine &Name = "") {
+    llvm::Value *Ptr = emitRawPointerFromAddress(Addr);
+    return RawAddress(CreateGEP(Addr.getElementType(), Ptr, IdxList, Name),
+                      ElementType, Align);
+  }
+
+  using CGBuilderBaseTy::CreateInBoundsGEP;
+  Address CreateInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList,
+                            llvm::Type *ElementType, CharUnits Align,
+                            const Twine &Name = "") {
+    return RawAddress(CreateInBoundsGEP(Addr.getElementType(),
+                                        emitRawPointerFromAddress(Addr),
+                                        IdxList, Name),
+                      ElementType, Align, Addr.isKnownNonNull());
+  }
+
+  using CGBuilderBaseTy::CreateIsNull;
+  llvm::Value *CreateIsNull(Address Addr, const Twine &Name = "") {
+    if (!Addr.hasOffset())
+      return CreateIsNull(Addr.getBasePointer(), Name);
+    // The pointer isn't null if Addr has an offset since offsets can always
+    // be applied inbound.
+    return llvm::ConstantInt::getFalse(Context);
   }
 
   using CGBuilderBaseTy::CreateMemCpy;
   llvm::CallInst *CreateMemCpy(Address Dest, Address Src, llvm::Value *Size,
                                bool IsVolatile = false) {
-    return CreateMemCpy(Dest.getPointer(), Dest.getAlignment().getAsAlign(),
-                        Src.getPointer(), Src.getAlignment().getAsAlign(), Size,
-                        IsVolatile);
+    llvm::Value *DestPtr = emitRawPointerFromAddress(Dest);
+    llvm::Value *SrcPtr = emitRawPointerFromAddress(Src);
+    return CreateMemCpy(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr,
+                        Src.getAlignment().getAsAlign(), Size, IsVolatile);
   }
   llvm::CallInst *CreateMemCpy(Address Dest, Address Src, uint64_t Size,
                                bool IsVolatile = false) {
-    return CreateMemCpy(Dest.getPointer(), Dest.getAlignment().getAsAlign(),
-                        Src.getPointer(), Src.getAlignment().getAsAlign(), Size,
-                        IsVolatile);
+    llvm::Value *DestPtr = emitRawPointerFromAddress(Dest);
+    llvm::Value *SrcPtr = emitRawPointerFromAddress(Src);
+    return CreateMemCpy(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr,
+                        Src.getAlignment().getAsAlign(), Size, IsVolatile);
   }
 
   using CGBuilderBaseTy::CreateMemCpyInline;
   llvm::CallInst *CreateMemCpyInline(Address Dest, Address Src, uint64_t Size) {
-    return CreateMemCpyInline(
-        Dest.getPointer(), Dest.getAlignment().getAsAlign(), Src.getPointer(),
-        Src.getAlignment().getAsAlign(), getInt64(Size));
+    llvm::Value *DestPtr = emitRawPointerFromAddress(Dest);
+    llvm::Value *SrcPtr = emitRawPointerFromAddress(Src);
+    return CreateMemCpyInline(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr,
+                              Src.getAlignment().getAsAlign(), getInt64(Size));
   }
 
   using CGBuilderBaseTy::CreateMemMove;
   llvm::CallInst *CreateMemMove(Address Dest, Address Src, llvm::Value *Size,
                                 bool IsVolatile = false) {
-    return CreateMemMove(Dest.getPointer(), Dest.getAlignment().getAsAlign(),
-                         Src.getPointer(), Src.getAlignment().getAsAlign(),
-                         Size, IsVolatile);
+    llvm::Value *DestPtr = emitRawPointerFromAddress(Dest);
+    llvm::Value *SrcPtr = emitRawPointerFromAddress(Src);
+    return CreateMemMove(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr,
+                         Src.getAlignment().getAsAlign(), Size, IsVolatile);
   }
 
   using CGBuilderBaseTy::CreateMemSet;
   llvm::CallInst *CreateMemSet(Address Dest, llvm::Value *Value,
                                llvm::Value *Size, bool IsVolatile = false) {
-    return CreateMemSet(Dest.getPointer(), Value, Size,
+    return CreateMemSet(emitRawPointerFromAddress(Dest), Value, Size,
                         Dest.getAlignment().getAsAlign(), IsVolatile);
   }
 
   using CGBuilderBaseTy::CreateMemSetInline;
   llvm::CallInst *CreateMemSetInline(Address Dest, llvm::Value *Value,
                                      uint64_t Size) {
-    return CreateMemSetInline(Dest.getPointer(),
+    return CreateMemSetInline(emitRawPointerFromAddress(Dest),
                               Dest.getAlignment().getAsAlign(), Value,
                               getInt64(Size));
   }
@@ -346,16 +417,31 @@ public:
     const llvm::StructLayout *Layout = DL.getStructLayout(ElTy);
     auto Offset = CharUnits::fromQuantity(Layout->getElementOffset(Index));
 
-    return Address(CreatePreserveStructAccessIndex(ElTy, Addr.getPointer(),
-                                                   Index, FieldIndex, DbgInfo),
-                   ElTy->getElementType(Index),
-                   Addr.getAlignment().alignmentAtOffset(Offset));
+    return Address(
+        CreatePreserveStructAccessIndex(ElTy, emitRawPointerFromAddress(Addr),
+                                        Index, FieldIndex, DbgInfo),
+        ElTy->getElementType(Index),
+        Addr.getAlignment().alignmentAtOffset(Offset));
+  }
+
+  using CGBuilderBaseTy::CreatePreserveUnionAccessIndex;
+  Address CreatePreserveUnionAccessIndex(Address Addr, unsigned FieldIndex,
+                                         llvm::MDNode *DbgInfo) {
+    Addr.replaceBasePointer(CreatePreserveUnionAccessIndex(
+        Addr.getBasePointer(), FieldIndex, DbgInfo));
+    return Addr;
   }
 
   using CGBuilderBaseTy::CreateLaunderInvariantGroup;
   Address CreateLaunderInvariantGroup(Address Addr) {
-    return Addr.withPointer(CreateLaunderInvariantGroup(Addr.getPointer()),
-                            Addr.isKnownNonNull());
+    Addr.replaceBasePointer(CreateLaunderInvariantGroup(Addr.getBasePointer()));
+    return Addr;
+  }
+
+  using CGBuilderBaseTy::CreateStripInvariantGroup;
+  Address CreateStripInvariantGroup(Address Addr) {
+    Addr.replaceBasePointer(CreateStripInvariantGroup(Addr.getBasePointer()));
+    return Addr;
   }
 };
 
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 46a815155e7b..9ed7b0c57092 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2116,9 +2116,9 @@ llvm::Function *CodeGenFunction::generateBuiltinOSLogHelperFunction(
   auto AL = ApplyDebugLocation::CreateArtificial(*this);
 
   CharUnits Offset;
-  Address BufAddr =
-      Address(Builder.CreateLoad(GetAddrOfLocalVar(Args[0]), "buf"), Int8Ty,
-              BufferAlignment);
+  Address BufAddr = makeNaturalAddressForPointer(
+      Builder.CreateLoad(GetAddrOfLocalVar(Args[0]), "buf"), Ctx.VoidTy,
+      BufferAlignment);
   Builder.CreateStore(Builder.getInt8(Layout.getSummaryByte()),
                       Builder.CreateConstByteGEP(BufAddr, Offset++, "summary"));
   Builder.CreateStore(Builder.getInt8(Layout.getNumArgsByte()),
@@ -2161,7 +2161,7 @@ RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) {
 
   // Ignore argument 1, the format string. It is not currently used.
   CallArgList Args;
-  Args.add(RValue::get(BufAddr.getPointer()), Ctx.VoidPtrTy);
+  Args.add(RValue::get(BufAddr.emitRawPointer(*this)), Ctx.VoidPtrTy);
 
   for (const auto &Item : Layout.Items) {
     int Size = Item.getSizeByte();
@@ -2201,8 +2201,8 @@ RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) {
         if (!isa<Constant>(ArgVal)) {
           CleanupKind Cleanup = getARCCleanupKind();
           QualType Ty = TheExpr->getType();
-          Address Alloca = Address::invalid();
-          Address Addr = CreateMemTemp(Ty, "os.log.arg", &Alloca);
+          RawAddress Alloca = RawAddress::invalid();
+          RawAddress Addr = CreateMemTemp(Ty, "os.log.arg", &Alloca);
           ArgVal = EmitARCRetain(Ty, ArgVal);
           Builder.CreateStore(ArgVal, Addr);
           pushLifetimeExtendedDestroy(Cleanup, Alloca, Ty,
@@ -2235,7 +2235,7 @@ RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) {
   llvm::Function *F = CodeGenFunction(CGM).generateBuiltinOSLogHelperFunction(
       Layout, BufAddr.getAlignment());
   EmitCall(FI, CGCallee::forDirect(F), ReturnValueSlot(), Args);
-  return RValue::get(BufAddr.getPointer());
+  return RValue::get(BufAddr, *this);
 }
 
 static bool isSpecialUnsignedMultiplySignedResult(
@@ -2983,7 +2983,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   // Check NonnullAttribute/NullabilityArg and Alignment.
   auto EmitArgCheck = [&](TypeCheckKind Kind, Address A, const Expr *Arg,
                           unsigned ParmNum) {
-    Value *Val = A.getPointer();
+    Value *Val = A.emitRawPointer(*this);
     EmitNonNullArgCheck(RValue::get(Val), Arg->getType(), Arg->getExprLoc(), FD,
                         ParmNum);
 
@@ -3012,12 +3012,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   case Builtin::BI__builtin_va_end:
     EmitVAStartEnd(BuiltinID == Builtin::BI__va_start
                        ? EmitScalarExpr(E->getArg(0))
-                       : EmitVAListRef(E->getArg(0)).getPointer(),
+                       : EmitVAListRef(E->getArg(0)).emitRawPointer(*this),
                    BuiltinID != Builtin::BI__builtin_va_end);
     return RValue::get(nullptr);
   case Builtin::BI__builtin_va_copy: {
-    Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer();
-    Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer();
+    Value *DstPtr = EmitVAListRef(E->getArg(0)).emitRawPointer(*this);
+    Value *SrcPtr = EmitVAListRef(E->getArg(1)).emitRawPointer(*this);
     Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy), {DstPtr, SrcPtr});
     return RValue::get(nullptr);
   }
@@ -3847,13 +3847,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
 
     Address Src = EmitPointerWithAlignment(E->getArg(0));
-    EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(0)->getType(),
-                        E->getArg(0)->getExprLoc(), FD, 0);
+    EmitNonNullArgCheck(RValue::get(Src.emitRawPointer(*this)),
+                        E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD,
+                        0);
     Value *Result = MB.CreateColumnMajorLoad(
-        Src.getElementType(), Src.getPointer(),
+        Src.getElementType(), Src.emitRawPointer(*this),
         Align(Src.getAlignment().getQuantity()), Stride, IsVolatile,
-        ResultTy->getNumRows(), ResultTy->getNumColumns(),
-        "matrix");
+        ResultTy->getNumRows(), ResultTy->getNumColumns(), "matrix");
     return RValue::get(Result);
   }
 
@@ -3868,11 +3868,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     assert(PtrTy && "arg1 must be of pointer type");
     bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
 
-    EmitNonNullArgCheck(RValue::get(Dst.getPointer()), E->getArg(1)->getType(),
-                        E->getArg(1)->getExprLoc(), FD, 0);
+    EmitNonNullArgCheck(RValue::get(Dst.emitRawPointer(*this)),
+                        E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD,
+                        0);
     Value *Result = MB.CreateColumnMajorStore(
-        Matrix, Dst.getPointer(), Align(Dst.getAlignment().getQuantity()),
-        Stride, IsVolatile, MatrixTy->getNumRows(), MatrixTy->getNumColumns());
+        Matrix, Dst.emitRawPointer(*this),
+        Align(Dst.getAlignment().getQuantity()), Stride, IsVolatile,
+        MatrixTy->getNumRows(), MatrixTy->getNumColumns());
     return RValue::get(Result);
   }
 
@@ -4031,7 +4033,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   case Builtin::BI__builtin_bzero: {
     Address Dest = EmitPointerWithAlignment(E->getArg(0));
     Value *SizeVal = EmitScalarExpr(E->getArg(1));
-    EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
+    EmitNonNullArgCheck(Dest, E->getArg(0)->getType(),
                         E->getArg(0)->getExprLoc(), FD, 0);
     Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false);
     return RValue::get(nullptr);
@@ -4042,10 +4044,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Address Src = EmitPointerWithAlignment(E->getArg(0));
     Address Dest = EmitPointerWithAlignment(E->getArg(1));
     Value *SizeVal = EmitScalarExpr(E->getArg(2));
-    EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(0)->getType(),
-                        E->getArg(0)->getExprLoc(), FD, 0);
-    EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(1)->getType(),
-                        E->getArg(1)->getExprLoc(), FD, 0);
+    EmitNonNullArgCheck(RValue::get(Src.emitRawPointer(*this)),
+                        E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD,
+                        0);
+    EmitNonNullArgCheck(RValue::get(Dest.emitRawPointer(*this)),
+                        E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD,
+                        0);
     Builder.CreateMemMove(Dest, Src, SizeVal, false);
     return RValue::get(nullptr);
   }
@@ -4062,10 +4066,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Builder.CreateMemCpy(Dest, Src, SizeVal, false);
     if (BuiltinID == Builtin::BImempcpy ||
         BuiltinID == Builtin::BI__builtin_mempcpy)
-      return RValue::get(Builder.CreateInBoundsGEP(Dest.getElementType(),
-                                                   Dest.getPointer(), SizeVal));
+      return RValue::get(Builder.CreateInBoundsGEP(
+          Dest.getElementType(), Dest.emitRawPointer(*this), SizeVal));
     else
-      return RValue::get(Dest.getPointer());
+      return RValue::get(Dest, *this);
   }
 
   case Builtin::BI__builtin_memcpy_inline: {
@@ -4097,7 +4101,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Address Src = EmitPointerWithAlignment(E->getArg(1));
     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
     Builder.CreateMemCpy(Dest, Src, SizeVal, false);
-    return RValue::get(Dest.getPointer());
+    return RValue::get(Dest, *this);
   }
 
   case Builtin::BI__builtin_objc_memmove_collectable: {
@@ -4106,7 +4110,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Value *SizeVal = EmitScalarExpr(E->getArg(2));
     CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this,
                                                   DestAddr, SrcAddr, SizeVal);
-    return RValue::get(DestAddr.getPointer());
+    return RValue::get(DestAddr, *this);
   }
 
   case Builtin::BI__builtin___memmove_chk: {
@@ -4123,7 +4127,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Address Src = EmitPointerWithAlignment(E->getArg(1));
     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
     Builder.CreateMemMove(Dest, Src, SizeVal, false);
-    return RValue::get(Dest.getPointer());
+    return RValue::get(Dest, *this);
   }
 
   case Builtin::BImemmove:
@@ -4134,7 +4138,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     EmitArgCheck(TCK_Store, Dest, E->getArg(0), 0);
     EmitArgCheck(TCK_Load, Src, E->getArg(1), 1);
     Builder.CreateMemMove(Dest, Src, SizeVal, false);
-    return RValue::get(Dest.getPointer());
+    return RValue::get(Dest, *this);
   }
   case Builtin::BImemset:
   case Builtin::BI__builtin_memset: {
@@ -4142,10 +4146,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
                                          Builder.getInt8Ty());
     Value *SizeVal = EmitScalarExpr(E->getArg(2));
-    EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
+    EmitNonNullArgCheck(Dest, E->getArg(0)->getType(),
                         E->getArg(0)->getExprLoc(), FD, 0);
     Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
-    return RValue::get(Dest.getPointer());
+    return RValue::get(Dest, *this);
   }
   case Builtin::BI__builtin_memset_inline: {
     Address Dest = EmitPointerWithAlignment(E->getArg(0));
@@ -4153,8 +4157,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
         Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), Builder.getInt8Ty());
     uint64_t Size =
         E->getArg(2)->EvaluateKnownConstInt(getContext()).getZExtValue();
-    EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
-                        E->getArg(0)->getExprLoc(), FD, 0);
+    EmitNonNullArgCheck(RValue::get(Dest.emitRawPointer(*this)),
+                        E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD,
+                        0);
     Builder.CreateMemSetInline(Dest, ByteVal, Size);
     return RValue::get(nullptr);
   }
@@ -4173,7 +4178,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
                                          Builder.getInt8Ty());
     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
     Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
-    return RValue::get(Dest.getPointer());
+    return RValue::get(Dest, *this);
   }
   case Builtin::BI__builtin_wmemchr: {
     // The MSVC runtime library does not provide a definition of wmemchr, so we
@@ -4395,14 +4400,14 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
 
     // Store the stack pointer to the setjmp buffer.
     Value *StackAddr = Builder.CreateStackSave();
-    assert(Buf.getPointer()->getType() == StackAddr->getType());
+    assert(Buf.emitRawPointer(*this)->getType() == StackAddr->getType());
 
     Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Buf, 2);
     Builder.CreateStore(StackAddr, StackSaveSlot);
 
     // Call LLVM's EH setjmp, which is lightweight.
     Function *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
-    return RValue::get(Builder.CreateCall(F, Buf.getPointer()));
+    return RValue::get(Builder.CreateCall(F, Buf.emitRawPointer(*this)));
   }
   case Builtin::BI__builtin_longjmp: {
     Value *Buf = EmitScalarExpr(E->getArg(0));
@@ -5575,7 +5580,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     llvm::Value *Queue = EmitScalarExpr(E->getArg(0));
     llvm::Value *Flags = EmitScalarExpr(E->getArg(1));
     LValue NDRangeL = EmitAggExprToLValue(E->getArg(2));
-    llvm::Value *Range = NDRangeL.getAddress(*this).getPointer();
+    llvm::Value *Range = NDRangeL.getAddress(*this).emitRawPointer(*this);
     llvm::Type *RangeTy = NDRangeL.getAddress(*this).getType();
 
     if (NumArgs == 4) {
@@ -5684,9 +5689,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
               getContext(), Expr::NPC_ValueDependentIsNotNull)) {
         EventWaitList = llvm::ConstantPointerNull::get(PtrTy);
       } else {
-        EventWaitList = E->getArg(4)->getType()->isArrayType()
-                        ? EmitArrayToPointerDecay(E->getArg(4)).getPointer()
-                        : EmitScalarExpr(E->getArg(4));
+        EventWaitList =
+            E->getArg(4)->getType()->isArrayType()
+                ? EmitArrayToPointerDecay(E->getArg(4)).emitRawPointer(*this)
+                : EmitScalarExpr(E->getArg(4));
         // Convert to generic address space.
         EventWaitList = Builder.CreatePointerCast(EventWaitList, PtrTy);
       }
@@ -5782,7 +5788,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
         getContext().getTargetAddressSpace(LangAS::opencl_generic));
     LValue NDRangeL = EmitAggExprToLValue(E->getArg(0));
-    llvm::Value *NDRange = NDRangeL.getAddress(*this).getPointer();
+    llvm::Value *NDRange = NDRangeL.getAddress(*this).emitRawPointer(*this);
     auto Info =
         CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(1));
     Value *Kernel =
@@ -5867,7 +5873,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     auto PTy0 = FTy->getParamType(0);
     if (PTy0 != Arg0Val->getType()) {
       if (Arg0Ty->isArrayType())
-        Arg0Val = EmitArrayToPointerDecay(Arg0).getPointer();
+        Arg0Val = EmitArrayToPointerDecay(Arg0).emitRawPointer(*this);
       else
         Arg0Val = Builder.CreatePointerCast(Arg0Val, PTy0);
     }
@@ -5905,7 +5911,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     auto PTy1 = FTy->getParamType(1);
     if (PTy1 != Arg1Val->getType()) {
       if (Arg1Ty->isArrayType())
-        Arg1Val = EmitArrayToPointerDecay(Arg1).getPointer();
+        Arg1Val = EmitArrayToPointerDecay(Arg1).emitRawPointer(*this);
       else
         Arg1Val = Builder.CreatePointerCast(Arg1Val, PTy1);
     }
@@ -5919,7 +5925,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   case Builtin::BI__builtin_ms_va_start:
   case Builtin::BI__builtin_ms_va_end:
     return RValue::get(
-        EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).getPointer(),
+        EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).emitRawPointer(*this),
                        BuiltinID == Builtin::BI__builtin_ms_va_start));
 
   case Builtin::BI__builtin_ms_va_copy: {
@@ -5961,8 +5967,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   // If this is a predefined lib function (e.g. malloc), emit the call
   // using exactly the normal call path.
   if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID))
-    return emitLibraryCall(*this, FD, E,
-                      cast<llvm::Constant>(EmitScalarExpr(E->getCallee())));
+    return emitLibraryCall(
+        *this, FD, E, cast<llvm::Constant>(EmitScalarExpr(E->getCallee())));
 
   // Check that a call to a target specific builtin has the correct target
   // features.
@@ -6079,7 +6085,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
         return RValue::get(nullptr);
       return RValue::get(V);
     case TEK_Aggregate:
-      return RValue::getAggregate(ReturnValue.getValue(),
+      return RValue::getAggregate(ReturnValue.getAddress(),
                                   ReturnValue.isVolatile());
     case TEK_Complex:
       llvm_unreachable("No current target builtin returns complex");
@@ -8849,7 +8855,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
         // Get the alignment for the argument in addition to the value;
         // we'll use it later.
         PtrOp0 = EmitPointerWithAlignment(E->getArg(0));
-        Ops.push_back(PtrOp0.getPointer());
+        Ops.push_back(PtrOp0.emitRawPointer(*this));
         continue;
       }
     }
@@ -8876,7 +8882,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
         // Get the alignment for the argument in addition to the value;
         // we'll use it later.
         PtrOp1 = EmitPointerWithAlignment(E->getArg(1));
-        Ops.push_back(PtrOp1.getPointer());
+        Ops.push_back(PtrOp1.emitRawPointer(*this));
         continue;
       }
     }
@@ -9297,7 +9303,7 @@ Value *CodeGenFunction::EmitARMMVEBuiltinExpr(unsigned BuiltinID,
     if (ReturnValue.isNull())
       return MvecOut;
     else
-      return Builder.CreateStore(MvecOut, ReturnValue.getValue());
+      return Builder.CreateStore(MvecOut, ReturnValue.getAddress());
   }
 
   case CustomCodeGen::VST24: {
@@ -11477,7 +11483,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
         // Get the alignment for the argument in addition to the value;
         // we'll use it later.
         PtrOp0 = EmitPointerWithAlignment(E->getArg(0));
-        Ops.push_back(PtrOp0.getPointer());
+        Ops.push_back(PtrOp0.emitRawPointer(*this));
         continue;
       }
     }
@@ -13343,15 +13349,15 @@ Value *CodeGenFunction::EmitBPFBuiltinExpr(unsigned BuiltinID,
     if (!getDebugInfo()) {
       CGM.Error(E->getExprLoc(),
                 "using __builtin_preserve_field_info() without -g");
-      return IsBitField ? EmitLValue(Arg).getBitFieldPointer()
-                        : EmitLValue(Arg).getPointer(*this);
+      return IsBitField ? EmitLValue(Arg).getRawBitFieldPointer(*this)
+                        : EmitLValue(Arg).emitRawPointer(*this);
     }
 
     // Enable underlying preserve_*_access_index() generation.
     bool OldIsInPreservedAIRegion = IsInPreservedAIRegion;
     IsInPreservedAIRegion = true;
-    Value *FieldAddr = IsBitField ? EmitLValue(Arg).getBitFieldPointer()
-                                  : EmitLValue(Arg).getPointer(*this);
+    Value *FieldAddr = IsBitField ? EmitLValue(Arg).getRawBitFieldPointer(*this)
+                                  : EmitLValue(Arg).emitRawPointer(*this);
     IsInPreservedAIRegion = OldIsInPreservedAIRegion;
 
     ConstantInt *C = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
@@ -14343,14 +14349,14 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
   }
   case X86::BI_mm_setcsr:
   case X86::BI__builtin_ia32_ldmxcsr: {
-    Address Tmp = CreateMemTemp(E->getArg(0)->getType());
+    RawAddress Tmp = CreateMemTemp(E->getArg(0)->getType());
     Builder.CreateStore(Ops[0], Tmp);
     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr),
                               Tmp.getPointer());
   }
   case X86::BI_mm_getcsr:
   case X86::BI__builtin_ia32_stmxcsr: {
-    Address Tmp = CreateMemTemp(E->getType());
+    RawAddress Tmp = CreateMemTemp(E->getType());
     Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr),
                        Tmp.getPointer());
     return Builder.CreateLoad(Tmp, "stmxcsr");
@@ -17625,7 +17631,8 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
     SmallVector<Value *, 4> Ops;
     for (unsigned i = 0, e = E->getNumArgs(); i != e; i++)
       if (E->getArg(i)->getType()->isArrayType())
-        Ops.push_back(EmitArrayToPointerDecay(E->getArg(i)).getPointer());
+        Ops.push_back(
+            EmitArrayToPointerDecay(E->getArg(i)).emitRawPointer(*this));
       else
         Ops.push_back(EmitScalarExpr(E->getArg(i)));
     // The first argument of these two builtins is a pointer used to store their
@@ -20099,14 +20106,14 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
     // Save returned values.
     assert(II.NumResults);
     if (II.NumResults == 1) {
-      Builder.CreateAlignedStore(Result, Dst.getPointer(),
+      Builder.CreateAlignedStore(Result, Dst.emitRawPointer(*this),
                                  CharUnits::fromQuantity(4));
     } else {
       for (unsigned i = 0; i < II.NumResults; ++i) {
         Builder.CreateAlignedStore(
             Builder.CreateBitCast(Builder.CreateExtractValue(Result, i),
                                   Dst.getElementType()),
-            Builder.CreateGEP(Dst.getElementType(), Dst.getPointer(),
+            Builder.CreateGEP(Dst.getElementType(), Dst.emitRawPointer(*this),
                               llvm::ConstantInt::get(IntTy, i)),
             CharUnits::fromQuantity(4));
       }
@@ -20146,7 +20153,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
     for (unsigned i = 0; i < II.NumResults; ++i) {
       Value *V = Builder.CreateAlignedLoad(
           Src.getElementType(),
-          Builder.CreateGEP(Src.getElementType(), Src.getPointer(),
+          Builder.CreateGEP(Src.getElementType(), Src.emitRawPointer(*this),
                             llvm::ConstantInt::get(IntTy, i)),
           CharUnits::fromQuantity(4));
       Values.push_back(Builder.CreateBitCast(V, ParamType));
@@ -20218,7 +20225,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
     for (unsigned i = 0; i < MI.NumEltsA; ++i) {
       Value *V = Builder.CreateAlignedLoad(
           SrcA.getElementType(),
-          Builder.CreateGEP(SrcA.getElementType(), SrcA.getPointer(),
+          Builder.CreateGEP(SrcA.getElementType(), SrcA.emitRawPointer(*this),
                             llvm::ConstantInt::get(IntTy, i)),
           CharUnits::fromQuantity(4));
       Values.push_back(Builder.CreateBitCast(V, AType));
@@ -20228,7 +20235,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
     for (unsigned i = 0; i < MI.NumEltsB; ++i) {
       Value *V = Builder.CreateAlignedLoad(
           SrcB.getElementType(),
-          Builder.CreateGEP(SrcB.getElementType(), SrcB.getPointer(),
+          Builder.CreateGEP(SrcB.getElementType(), SrcB.emitRawPointer(*this),
                             llvm::ConstantInt::get(IntTy, i)),
           CharUnits::fromQuantity(4));
       Values.push_back(Builder.CreateBitCast(V, BType));
@@ -20239,7 +20246,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
     for (unsigned i = 0; i < MI.NumEltsC; ++i) {
       Value *V = Builder.CreateAlignedLoad(
           SrcC.getElementType(),
-          Builder.CreateGEP(SrcC.getElementType(), SrcC.getPointer(),
+          Builder.CreateGEP(SrcC.getElementType(), SrcC.emitRawPointer(*this),
                             llvm::ConstantInt::get(IntTy, i)),
           CharUnits::fromQuantity(4));
       Values.push_back(Builder.CreateBitCast(V, CType));
@@ -20249,7 +20256,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
     for (unsigned i = 0; i < MI.NumEltsD; ++i)
       Builder.CreateAlignedStore(
           Builder.CreateBitCast(Builder.CreateExtractValue(Result, i), DType),
-          Builder.CreateGEP(Dst.getElementType(), Dst.getPointer(),
+          Builder.CreateGEP(Dst.getElementType(), Dst.emitRawPointer(*this),
                             llvm::ConstantInt::get(IntTy, i)),
           CharUnits::fromQuantity(4));
     return Result;
@@ -20507,7 +20514,7 @@ struct BuiltinAlignArgs {
   BuiltinAlignArgs(const CallExpr *E, CodeGenFunction &CGF) {
     QualType AstType = E->getArg(0)->getType();
     if (AstType->isArrayType())
-      Src = CGF.EmitArrayToPointerDecay(E->getArg(0)).getPointer();
+      Src = CGF.EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(CGF);
     else
       Src = CGF.EmitScalarExpr(E->getArg(0));
     SrcType = Src->getType();
@@ -21125,7 +21132,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
   }
   case WebAssembly::BI__builtin_wasm_table_get: {
     assert(E->getArg(0)->getType()->isArrayType());
-    Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer();
+    Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
     Value *Index = EmitScalarExpr(E->getArg(1));
     Function *Callee;
     if (E->getType().isWebAssemblyExternrefType())
@@ -21139,7 +21146,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
   }
   case WebAssembly::BI__builtin_wasm_table_set: {
     assert(E->getArg(0)->getType()->isArrayType());
-    Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer();
+    Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
     Value *Index = EmitScalarExpr(E->getArg(1));
     Value *Val = EmitScalarExpr(E->getArg(2));
     Function *Callee;
@@ -21154,13 +21161,13 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
   }
   case WebAssembly::BI__builtin_wasm_table_size: {
     assert(E->getArg(0)->getType()->isArrayType());
-    Value *Value = EmitArrayToPointerDecay(E->getArg(0)).getPointer();
+    Value *Value = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_table_size);
     return Builder.CreateCall(Callee, Value);
   }
   case WebAssembly::BI__builtin_wasm_table_grow: {
     assert(E->getArg(0)->getType()->isArrayType());
-    Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer();
+    Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
     Value *Val = EmitScalarExpr(E->getArg(1));
     Value *NElems = EmitScalarExpr(E->getArg(2));
 
@@ -21177,7 +21184,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
   }
   case WebAssembly::BI__builtin_wasm_table_fill: {
     assert(E->getArg(0)->getType()->isArrayType());
-    Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer();
+    Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
     Value *Index = EmitScalarExpr(E->getArg(1));
     Value *Val = EmitScalarExpr(E->getArg(2));
     Value *NElems = EmitScalarExpr(E->getArg(3));
@@ -21195,8 +21202,8 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
   }
   case WebAssembly::BI__builtin_wasm_table_copy: {
     assert(E->getArg(0)->getType()->isArrayType());
-    Value *TableX = EmitArrayToPointerDecay(E->getArg(0)).getPointer();
-    Value *TableY = EmitArrayToPointerDecay(E->getArg(1)).getPointer();
+    Value *TableX = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
+    Value *TableY = EmitArrayToPointerDecay(E->getArg(1)).emitRawPointer(*this);
     Value *DstIdx = EmitScalarExpr(E->getArg(2));
     Value *SrcIdx = EmitScalarExpr(E->getArg(3));
     Value *NElems = EmitScalarExpr(E->getArg(4));
@@ -21275,7 +21282,7 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID,
   auto MakeCircOp = [this, E](unsigned IntID, bool IsLoad) {
     // The base pointer is passed by address, so it needs to be loaded.
     Address A = EmitPointerWithAlignment(E->getArg(0));
-    Address BP = Address(A.getPointer(), Int8PtrTy, A.getAlignment());
+    Address BP = Address(A.emitRawPointer(*this), Int8PtrTy, A.getAlignment());
     llvm::Value *Base = Builder.CreateLoad(BP);
     // The treatment of both loads and stores is the same: the arguments for
     // the builtin are the same as the arguments for the intrinsic.
@@ -21316,8 +21323,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID,
     // EmitPointerWithAlignment and EmitScalarExpr evaluates the expression
     // per call.
     Address DestAddr = EmitPointerWithAlignment(E->getArg(1));
-    DestAddr = Address(DestAddr.getPointer(), Int8Ty, DestAddr.getAlignment());
-    llvm::Value *DestAddress = DestAddr.getPointer();
+    DestAddr = DestAddr.withElementType(Int8Ty);
+    llvm::Value *DestAddress = DestAddr.emitRawPointer(*this);
 
     // Operands are Base, Dest, Modifier.
     // The intrinsic format in LLVM IR is defined as
@@ -21368,8 +21375,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID,
         {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn});
 
     llvm::Value *PredOut = Builder.CreateExtractValue(Result, 1);
-    Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.getPointer(),
-        PredAddr.getAlignment());
+    Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.emitRawPointer(*this),
+                               PredAddr.getAlignment());
     return Builder.CreateExtractValue(Result, 0);
   }
   // These are identical to the builtins above, except they don't consume
@@ -21387,8 +21394,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID,
         {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
 
     llvm::Value *PredOut = Builder.CreateExtractValue(Result, 1);
-    Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.getPointer(),
-        PredAddr.getAlignment());
+    Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.emitRawPointer(*this),
+                               PredAddr.getAlignment());
     return Builder.CreateExtractValue(Result, 0);
   }
 
diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp
index b756318c46a9..0cb5b06a519c 100644
--- a/clang/lib/CodeGen/CGCUDANV.cpp
+++ b/clang/lib/CodeGen/CGCUDANV.cpp
@@ -331,11 +331,11 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF,
       llvm::ConstantInt::get(SizeTy, std::max<size_t>(1, Args.size())));
   // Store pointers to the arguments in a locally allocated launch_args.
   for (unsigned i = 0; i < Args.size(); ++i) {
-    llvm::Value* VarPtr = CGF.GetAddrOfLocalVar(Args[i]).getPointer();
+    llvm::Value *VarPtr = CGF.GetAddrOfLocalVar(Args[i]).emitRawPointer(CGF);
     llvm::Value *VoidVarPtr = CGF.Builder.CreatePointerCast(VarPtr, PtrTy);
     CGF.Builder.CreateDefaultAlignedStore(
-        VoidVarPtr,
-        CGF.Builder.CreateConstGEP1_32(PtrTy, KernelArgs.getPointer(), i));
+        VoidVarPtr, CGF.Builder.CreateConstGEP1_32(
+                        PtrTy, KernelArgs.emitRawPointer(CGF), i));
   }
 
   llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end");
@@ -393,9 +393,10 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF,
                               /*isVarArg=*/false),
       addUnderscoredPrefixToName("PopCallConfiguration"));
 
-  CGF.EmitRuntimeCallOrInvoke(cudaPopConfigFn,
-                              {GridDim.getPointer(), BlockDim.getPointer(),
-                               ShmemSize.getPointer(), Stream.getPointer()});
+  CGF.EmitRuntimeCallOrInvoke(cudaPopConfigFn, {GridDim.emitRawPointer(CGF),
+                                                BlockDim.emitRawPointer(CGF),
+                                                ShmemSize.emitRawPointer(CGF),
+                                                Stream.emitRawPointer(CGF)});
 
   // Emit the call to cudaLaunch
   llvm::Value *Kernel =
@@ -405,7 +406,7 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF,
                        cudaLaunchKernelFD->getParamDecl(0)->getType());
   LaunchKernelArgs.add(RValue::getAggregate(GridDim), Dim3Ty);
   LaunchKernelArgs.add(RValue::getAggregate(BlockDim), Dim3Ty);
-  LaunchKernelArgs.add(RValue::get(KernelArgs.getPointer()),
+  LaunchKernelArgs.add(RValue::get(KernelArgs, CGF),
                        cudaLaunchKernelFD->getParamDecl(3)->getType());
   LaunchKernelArgs.add(RValue::get(CGF.Builder.CreateLoad(ShmemSize)),
                        cudaLaunchKernelFD->getParamDecl(4)->getType());
@@ -438,8 +439,8 @@ void CGNVCUDARuntime::emitDeviceStubBodyLegacy(CodeGenFunction &CGF,
     auto TInfo = CGM.getContext().getTypeInfoInChars(A->getType());
     Offset = Offset.alignTo(TInfo.Align);
     llvm::Value *Args[] = {
-        CGF.Builder.CreatePointerCast(CGF.GetAddrOfLocalVar(A).getPointer(),
-                                      PtrTy),
+        CGF.Builder.CreatePointerCast(
+            CGF.GetAddrOfLocalVar(A).emitRawPointer(CGF), PtrTy),
         llvm::ConstantInt::get(SizeTy, TInfo.Width.getQuantity()),
         llvm::ConstantInt::get(SizeTy, Offset.getQuantity()),
     };
diff --git a/clang/lib/CodeGen/CGCXXABI.cpp b/clang/lib/CodeGen/CGCXXABI.cpp
index a8bf57a277e9..7c6dfc3e59d8 100644
--- a/clang/lib/CodeGen/CGCXXABI.cpp
+++ b/clang/lib/CodeGen/CGCXXABI.cpp
@@ -20,6 +20,12 @@ using namespace CodeGen;
 
 CGCXXABI::~CGCXXABI() { }
 
+Address CGCXXABI::getThisAddress(CodeGenFunction &CGF) {
+  return CGF.makeNaturalAddressForPointer(
+      CGF.CXXABIThisValue, CGF.CXXABIThisDecl->getType()->getPointeeType(),
+      CGF.CXXABIThisAlignment);
+}
+
 void CGCXXABI::ErrorUnsupportedABI(CodeGenFunction &CGF, StringRef S) {
   DiagnosticsEngine &Diags = CGF.CGM.getDiags();
   unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error,
@@ -44,8 +50,12 @@ CGCallee CGCXXABI::EmitLoadOfMemberFunctionPointer(
     llvm::Value *MemPtr, const MemberPointerType *MPT) {
   ErrorUnsupportedABI(CGF, "calls through member pointers");
 
-  ThisPtrForCall = This.getPointer();
-  const auto *FPT = MPT->getPointeeType()->castAs<FunctionProtoType>();
+  const auto *RD =
+      cast<CXXRecordDecl>(MPT->getClass()->castAs<RecordType>()->getDecl());
+  ThisPtrForCall =
+      CGF.getAsNaturalPointerTo(This, CGF.getContext().getRecordType(RD));
+  const FunctionProtoType *FPT =
+      MPT->getPointeeType()->getAs<FunctionProtoType>();
   llvm::Constant *FnPtr = llvm::Constant::getNullValue(
       llvm::PointerType::getUnqual(CGM.getLLVMContext()));
   return CGCallee::forDirect(FnPtr, FPT);
@@ -251,16 +261,15 @@ void CGCXXABI::ReadArrayCookie(CodeGenFunction &CGF, Address ptr,
 
   // If we don't need an array cookie, bail out early.
   if (!requiresArrayCookie(expr, eltTy)) {
-    allocPtr = ptr.getPointer();
+    allocPtr = ptr.emitRawPointer(CGF);
     numElements = nullptr;
     cookieSize = CharUnits::Zero();
     return;
   }
 
   cookieSize = getArrayCookieSizeImpl(eltTy);
-  Address allocAddr =
-    CGF.Builder.CreateConstInBoundsByteGEP(ptr, -cookieSize);
-  allocPtr = allocAddr.getPointer();
+  Address allocAddr = CGF.Builder.CreateConstInBoundsByteGEP(ptr, -cookieSize);
+  allocPtr = allocAddr.emitRawPointer(CGF);
   numElements = readArrayCookieImpl(CGF, allocAddr, cookieSize);
 }
 
diff --git a/clang/lib/CodeGen/CGCXXABI.h b/clang/lib/CodeGen/CGCXXABI.h
index ad1ad08d0856..c7eccbd0095a 100644
--- a/clang/lib/CodeGen/CGCXXABI.h
+++ b/clang/lib/CodeGen/CGCXXABI.h
@@ -57,12 +57,8 @@ protected:
   llvm::Value *getThisValue(CodeGenFunction &CGF) {
     return CGF.CXXABIThisValue;
   }
-  Address getThisAddress(CodeGenFunction &CGF) {
-    return Address(
-        CGF.CXXABIThisValue,
-        CGF.ConvertTypeForMem(CGF.CXXABIThisDecl->getType()->getPointeeType()),
-        CGF.CXXABIThisAlignment);
-  }
+
+  Address getThisAddress(CodeGenFunction &CGF);
 
   /// Issue a diagnostic about unsupported features in the ABI.
   void ErrorUnsupportedABI(CodeGenFunction &CGF, StringRef S);
@@ -475,12 +471,6 @@ public:
                                   BaseSubobject Base,
                                   const CXXRecordDecl *NearestVBase) = 0;
 
-  /// Get the address point of the vtable for the given base subobject while
-  /// building a constexpr.
-  virtual llvm::Constant *
-  getVTableAddressPointForConstExpr(BaseSubobject Base,
-                                    const CXXRecordDecl *VTableClass) = 0;
-
   /// Get the address of the vtable for the given record decl which should be
   /// used for the vptr at the given offset in RD.
   virtual llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD,
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index a28d7888715d..e21ed396dc8d 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -1031,15 +1031,9 @@ static void forConstantArrayExpansion(CodeGenFunction &CGF,
                                       ConstantArrayExpansion *CAE,
                                       Address BaseAddr,
                                       llvm::function_ref<void(Address)> Fn) {
-  CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy);
-  CharUnits EltAlign =
-    BaseAddr.getAlignment().alignmentOfArrayElement(EltSize);
-  llvm::Type *EltTy = CGF.ConvertTypeForMem(CAE->EltTy);
-
   for (int i = 0, n = CAE->NumElts; i < n; i++) {
-    llvm::Value *EltAddr = CGF.Builder.CreateConstGEP2_32(
-        BaseAddr.getElementType(), BaseAddr.getPointer(), 0, i);
-    Fn(Address(EltAddr, EltTy, EltAlign));
+    Address EltAddr = CGF.Builder.CreateConstGEP2_32(BaseAddr, 0, i);
+    Fn(EltAddr);
   }
 }
 
@@ -1154,9 +1148,10 @@ void CodeGenFunction::ExpandTypeToArgs(
 }
 
 /// Create a temporary allocation for the purposes of coercion.
-static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty,
-                                           CharUnits MinAlign,
-                                           const Twine &Name = "tmp") {
+static RawAddress CreateTempAllocaForCoercion(CodeGenFunction &CGF,
+                                              llvm::Type *Ty,
+                                              CharUnits MinAlign,
+                                              const Twine &Name = "tmp") {
   // Don't use an alignment that's worse than what LLVM would prefer.
   auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(Ty);
   CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign));
@@ -1326,11 +1321,11 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
   }
 
   // Otherwise do coercion through memory. This is stupid, but simple.
-  Address Tmp =
+  RawAddress Tmp =
       CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment(), Src.getName());
   CGF.Builder.CreateMemCpy(
-      Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), Src.getPointer(),
-      Src.getAlignment().getAsAlign(),
+      Tmp.getPointer(), Tmp.getAlignment().getAsAlign(),
+      Src.emitRawPointer(CGF), Src.getAlignment().getAsAlign(),
       llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize.getKnownMinValue()));
   return CGF.Builder.CreateLoad(Tmp);
 }
@@ -1414,11 +1409,12 @@ static void CreateCoercedStore(llvm::Value *Src,
     //
     // FIXME: Assert that we aren't truncating non-padding bits when have access
     // to that information.
-    Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment());
+    RawAddress Tmp =
+        CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment());
     CGF.Builder.CreateStore(Src, Tmp);
     CGF.Builder.CreateMemCpy(
-        Dst.getPointer(), Dst.getAlignment().getAsAlign(), Tmp.getPointer(),
-        Tmp.getAlignment().getAsAlign(),
+        Dst.emitRawPointer(CGF), Dst.getAlignment().getAsAlign(),
+        Tmp.getPointer(), Tmp.getAlignment().getAsAlign(),
         llvm::ConstantInt::get(CGF.IntPtrTy, DstSize.getFixedValue()));
   }
 }
@@ -3018,17 +3014,17 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
     case ABIArgInfo::Indirect:
     case ABIArgInfo::IndirectAliased: {
       assert(NumIRArgs == 1);
-      Address ParamAddr = Address(Fn->getArg(FirstIRArg), ConvertTypeForMem(Ty),
-                                  ArgI.getIndirectAlign(), KnownNonNull);
+      Address ParamAddr = makeNaturalAddressForPointer(
+          Fn->getArg(FirstIRArg), Ty, ArgI.getIndirectAlign(), false, nullptr,
+          nullptr, KnownNonNull);
 
       if (!hasScalarEvaluationKind(Ty)) {
         // Aggregates and complex variables are accessed by reference. All we
         // need to do is realign the value, if requested. Also, if the address
         // may be aliased, copy it to ensure that the parameter variable is
         // mutable and has a unique adress, as C requires.
-        Address V = ParamAddr;
         if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) {
-          Address AlignedTemp = CreateMemTemp(Ty, "coerce");
+          RawAddress AlignedTemp = CreateMemTemp(Ty, "coerce");
 
           // Copy from the incoming argument pointer to the temporary with the
           // appropriate alignment.
@@ -3038,11 +3034,12 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
           CharUnits Size = getContext().getTypeSizeInChars(Ty);
           Builder.CreateMemCpy(
               AlignedTemp.getPointer(), AlignedTemp.getAlignment().getAsAlign(),
-              ParamAddr.getPointer(), ParamAddr.getAlignment().getAsAlign(),
+              ParamAddr.emitRawPointer(*this),
+              ParamAddr.getAlignment().getAsAlign(),
               llvm::ConstantInt::get(IntPtrTy, Size.getQuantity()));
-          V = AlignedTemp;
+          ParamAddr = AlignedTemp;
         }
-        ArgVals.push_back(ParamValue::forIndirect(V));
+        ArgVals.push_back(ParamValue::forIndirect(ParamAddr));
       } else {
         // Load scalar value from indirect argument.
         llvm::Value *V =
@@ -3156,10 +3153,10 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
               == ParameterABI::SwiftErrorResult) {
           QualType pointeeTy = Ty->getPointeeType();
           assert(pointeeTy->isPointerType());
-          Address temp =
-            CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
-          Address arg(V, ConvertTypeForMem(pointeeTy),
-                      getContext().getTypeAlignInChars(pointeeTy));
+          RawAddress temp =
+              CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
+          Address arg = makeNaturalAddressForPointer(
+              V, pointeeTy, getContext().getTypeAlignInChars(pointeeTy));
           llvm::Value *incomingErrorValue = Builder.CreateLoad(arg);
           Builder.CreateStore(incomingErrorValue, temp);
           V = temp.getPointer();
@@ -3496,7 +3493,7 @@ static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
   llvm::LoadInst *load =
     dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());
   if (!load || load->isAtomic() || load->isVolatile() ||
-      load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer())
+      load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getBasePointer())
     return nullptr;
 
   // Okay!  Burn it all down.  This relies for correctness on the
@@ -3533,12 +3530,15 @@ static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
 
 /// Heuristically search for a dominating store to the return-value slot.
 static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
+  llvm::Value *ReturnValuePtr = CGF.ReturnValue.getBasePointer();
+
   // Check if a User is a store which pointerOperand is the ReturnValue.
   // We are looking for stores to the ReturnValue, not for stores of the
   // ReturnValue to some other location.
-  auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * {
+  auto GetStoreIfValid = [&CGF,
+                          ReturnValuePtr](llvm::User *U) -> llvm::StoreInst * {
     auto *SI = dyn_cast<llvm::StoreInst>(U);
-    if (!SI || SI->getPointerOperand() != CGF.ReturnValue.getPointer() ||
+    if (!SI || SI->getPointerOperand() != ReturnValuePtr ||
         SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType())
       return nullptr;
     // These aren't actually possible for non-coerced returns, and we
@@ -3552,7 +3552,7 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
   // for something immediately preceding the IP.  Sometimes this can
   // happen with how we generate implicit-returns; it can also happen
   // with noreturn cleanups.
-  if (!CGF.ReturnValue.getPointer()->hasOneUse()) {
+  if (!ReturnValuePtr->hasOneUse()) {
     llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
     if (IP->empty()) return nullptr;
 
@@ -3570,8 +3570,7 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
     return nullptr;
   }
 
-  llvm::StoreInst *store =
-      GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back());
+  llvm::StoreInst *store = GetStoreIfValid(ReturnValuePtr->user_back());
   if (!store) return nullptr;
 
   // Now do a first-and-dirty dominance check: just walk up the
@@ -4115,7 +4114,11 @@ void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
 }
 
 static bool isProvablyNull(llvm::Value *addr) {
-  return isa<llvm::ConstantPointerNull>(addr);
+  return llvm::isa_and_nonnull<llvm::ConstantPointerNull>(addr);
+}
+
+static bool isProvablyNonNull(Address Addr, CodeGenFunction &CGF) {
+  return llvm::isKnownNonZero(Addr.getBasePointer(), CGF.CGM.getDataLayout());
 }
 
 /// Emit the actual writing-back of a writeback.
@@ -4123,21 +4126,20 @@ static void emitWriteback(CodeGenFunction &CGF,
                           const CallArgList::Writeback &writeback) {
   const LValue &srcLV = writeback.Source;
   Address srcAddr = srcLV.getAddress(CGF);
-  assert(!isProvablyNull(srcAddr.getPointer()) &&
+  assert(!isProvablyNull(srcAddr.getBasePointer()) &&
          "shouldn't have writeback for provably null argument");
 
   llvm::BasicBlock *contBB = nullptr;
 
   // If the argument wasn't provably non-null, we need to null check
   // before doing the store.
-  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
-                                              CGF.CGM.getDataLayout());
+  bool provablyNonNull = isProvablyNonNull(srcAddr, CGF);
+
   if (!provablyNonNull) {
     llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
     contBB = CGF.createBasicBlock("icr.done");
 
-    llvm::Value *isNull =
-      CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
+    llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
     CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
     CGF.EmitBlock(writebackBB);
   }
@@ -4241,7 +4243,7 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
       CGF.ConvertTypeForMem(CRE->getType()->getPointeeType());
 
   // If the address is a constant null, just pass the appropriate null.
-  if (isProvablyNull(srcAddr.getPointer())) {
+  if (isProvablyNull(srcAddr.getBasePointer())) {
     args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
              CRE->getType());
     return;
@@ -4270,17 +4272,16 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
   // If the address is *not* known to be non-null, we need to switch.
   llvm::Value *finalArgument;
 
-  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
-                                              CGF.CGM.getDataLayout());
+  bool provablyNonNull = isProvablyNonNull(srcAddr, CGF);
+
   if (provablyNonNull) {
-    finalArgument = temp.getPointer();
+    finalArgument = temp.emitRawPointer(CGF);
   } else {
-    llvm::Value *isNull =
-      CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
+    llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
 
-    finalArgument = CGF.Builder.CreateSelect(isNull,
-                                   llvm::ConstantPointerNull::get(destType),
-                                             temp.getPointer(), "icr.argument");
+    finalArgument = CGF.Builder.CreateSelect(
+        isNull, llvm::ConstantPointerNull::get(destType),
+        temp.emitRawPointer(CGF), "icr.argument");
 
     // If we need to copy, then the load has to be conditional, which
     // means we need control flow.
@@ -4404,6 +4405,16 @@ void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
   EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, std::nullopt);
 }
 
+void CodeGenFunction::EmitNonNullArgCheck(Address Addr, QualType ArgType,
+                                          SourceLocation ArgLoc,
+                                          AbstractCallee AC, unsigned ParmNum) {
+  if (!AC.getDecl() || !(SanOpts.has(SanitizerKind::NonnullAttribute) ||
+                         SanOpts.has(SanitizerKind::NullabilityArg)))
+    return;
+
+  EmitNonNullArgCheck(RValue::get(Addr, *this), ArgType, ArgLoc, AC, ParmNum);
+}
+
 // Check if the call is going to use the inalloca convention. This needs to
 // agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
 // later, so we can't check it directly.
@@ -4744,10 +4755,20 @@ CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
 llvm::CallInst *
 CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
                                          const llvm::Twine &name) {
-  return EmitNounwindRuntimeCall(callee, std::nullopt, name);
+  return EmitNounwindRuntimeCall(callee, ArrayRef<llvm::Value *>(), name);
 }
 
 /// Emits a call to the given nounwind runtime function.
+llvm::CallInst *
+CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
+                                         ArrayRef<Address> args,
+                                         const llvm::Twine &name) {
+  SmallVector<llvm::Value *, 3> values;
+  for (auto arg : args)
+    values.push_back(arg.emitRawPointer(*this));
+  return EmitNounwindRuntimeCall(callee, values, name);
+}
+
 llvm::CallInst *
 CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
                                          ArrayRef<llvm::Value *> args,
@@ -5026,7 +5047,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
 
   // If we're using inalloca, insert the allocation after the stack save.
   // FIXME: Do this earlier rather than hacking it in here!
-  Address ArgMemory = Address::invalid();
+  RawAddress ArgMemory = RawAddress::invalid();
   if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
     const llvm::DataLayout &DL = CGM.getDataLayout();
     llvm::Instruction *IP = CallArgs.getStackBase();
@@ -5042,7 +5063,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
     AI->setAlignment(Align.getAsAlign());
     AI->setUsedWithInAlloca(true);
     assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
-    ArgMemory = Address(AI, ArgStruct, Align);
+    ArgMemory = RawAddress(AI, ArgStruct, Align);
   }
 
   ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
@@ -5051,11 +5072,11 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
   // If the call returns a temporary with struct return, create a temporary
   // alloca to hold the result, unless one is given to us.
   Address SRetPtr = Address::invalid();
-  Address SRetAlloca = Address::invalid();
+  RawAddress SRetAlloca = RawAddress::invalid();
   llvm::Value *UnusedReturnSizePtr = nullptr;
   if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) {
     if (!ReturnValue.isNull()) {
-      SRetPtr = ReturnValue.getValue();
+      SRetPtr = ReturnValue.getAddress();
     } else {
       SRetPtr = CreateMemTemp(RetTy, "tmp", &SRetAlloca);
       if (HaveInsertPoint() && ReturnValue.isUnused()) {
@@ -5065,15 +5086,16 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       }
     }
     if (IRFunctionArgs.hasSRetArg()) {
-      IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer();
+      IRCallArgs[IRFunctionArgs.getSRetArgNo()] =
+          getAsNaturalPointerTo(SRetPtr, RetTy);
     } else if (RetAI.isInAlloca()) {
       Address Addr =
           Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex());
-      Builder.CreateStore(SRetPtr.getPointer(), Addr);
+      Builder.CreateStore(getAsNaturalPointerTo(SRetPtr, RetTy), Addr);
     }
   }
 
-  Address swiftErrorTemp = Address::invalid();
+  RawAddress swiftErrorTemp = RawAddress::invalid();
   Address swiftErrorArg = Address::invalid();
 
   // When passing arguments using temporary allocas, we need to add the
@@ -5106,9 +5128,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       assert(NumIRArgs == 0);
       assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
       if (I->isAggregate()) {
-        Address Addr = I->hasLValue()
-                           ? I->getKnownLValue().getAddress(*this)
-                           : I->getKnownRValue().getAggregateAddress();
+        RawAddress Addr = I->hasLValue()
+                              ? I->getKnownLValue().getAddress(*this)
+                              : I->getKnownRValue().getAggregateAddress();
         llvm::Instruction *Placeholder =
             cast<llvm::Instruction>(Addr.getPointer());
 
@@ -5132,7 +5154,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       } else if (ArgInfo.getInAllocaIndirect()) {
         // Make a temporary alloca and store the address of it into the argument
         // struct.
-        Address Addr = CreateMemTempWithoutCast(
+        RawAddress Addr = CreateMemTempWithoutCast(
             I->Ty, getContext().getTypeAlignInChars(I->Ty),
             "indirect-arg-temp");
         I->copyInto(*this, Addr);
@@ -5154,12 +5176,12 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       assert(NumIRArgs == 1);
       if (!I->isAggregate()) {
         // Make a temporary alloca to pass the argument.
-        Address Addr = CreateMemTempWithoutCast(
+        RawAddress Addr = CreateMemTempWithoutCast(
             I->Ty, ArgInfo.getIndirectAlign(), "indirect-arg-temp");
 
-        llvm::Value *Val = Addr.getPointer();
+        llvm::Value *Val = getAsNaturalPointerTo(Addr, I->Ty);
         if (ArgHasMaybeUndefAttr)
-          Val = Builder.CreateFreeze(Addr.getPointer());
+          Val = Builder.CreateFreeze(Val);
         IRCallArgs[FirstIRArg] = Val;
 
         I->copyInto(*this, Addr);
@@ -5175,7 +5197,6 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
         Address Addr = I->hasLValue()
                            ? I->getKnownLValue().getAddress(*this)
                            : I->getKnownRValue().getAggregateAddress();
-        llvm::Value *V = Addr.getPointer();
         CharUnits Align = ArgInfo.getIndirectAlign();
         const llvm::DataLayout *TD = &CGM.getDataLayout();
 
@@ -5186,8 +5207,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
 
         bool NeedCopy = false;
         if (Addr.getAlignment() < Align &&
-            llvm::getOrEnforceKnownAlignment(V, Align.getAsAlign(), *TD) <
-                Align.getAsAlign()) {
+            llvm::getOrEnforceKnownAlignment(Addr.emitRawPointer(*this),
+                                             Align.getAsAlign(),
+                                             *TD) < Align.getAsAlign()) {
           NeedCopy = true;
         } else if (I->hasLValue()) {
           auto LV = I->getKnownLValue();
@@ -5218,11 +5240,11 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
 
         if (NeedCopy) {
           // Create an aligned temporary, and copy to it.
-          Address AI = CreateMemTempWithoutCast(
+          RawAddress AI = CreateMemTempWithoutCast(
               I->Ty, ArgInfo.getIndirectAlign(), "byval-temp");
-          llvm::Value *Val = AI.getPointer();
+          llvm::Value *Val = getAsNaturalPointerTo(AI, I->Ty);
           if (ArgHasMaybeUndefAttr)
-            Val = Builder.CreateFreeze(AI.getPointer());
+            Val = Builder.CreateFreeze(Val);
           IRCallArgs[FirstIRArg] = Val;
 
           // Emit lifetime markers for the temporary alloca.
@@ -5239,6 +5261,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
           I->copyInto(*this, AI);
         } else {
           // Skip the extra memcpy call.
+          llvm::Value *V = getAsNaturalPointerTo(Addr, I->Ty);
           auto *T = llvm::PointerType::get(
               CGM.getLLVMContext(), CGM.getDataLayout().getAllocaAddrSpace());
 
@@ -5278,8 +5301,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
           assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
 
           QualType pointeeTy = I->Ty->getPointeeType();
-          swiftErrorArg = Address(V, ConvertTypeForMem(pointeeTy),
-                                  getContext().getTypeAlignInChars(pointeeTy));
+          swiftErrorArg = makeNaturalAddressForPointer(
+              V, pointeeTy, getContext().getTypeAlignInChars(pointeeTy));
 
           swiftErrorTemp =
             CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
@@ -5416,7 +5439,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
 
       llvm::Value *tempSize = nullptr;
       Address addr = Address::invalid();
-      Address AllocaAddr = Address::invalid();
+      RawAddress AllocaAddr = RawAddress::invalid();
       if (I->isAggregate()) {
         addr = I->hasLValue() ? I->getKnownLValue().getAddress(*this)
                               : I->getKnownRValue().getAggregateAddress();
@@ -5850,7 +5873,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
           return RValue::getComplex(std::make_pair(Real, Imag));
         }
         case TEK_Aggregate: {
-          Address DestPtr = ReturnValue.getValue();
+          Address DestPtr = ReturnValue.getAddress();
           bool DestIsVolatile = ReturnValue.isVolatile();
 
           if (!DestPtr.isValid()) {
diff --git a/clang/lib/CodeGen/CGCall.h b/clang/lib/CodeGen/CGCall.h
index 1bd48a072593..6b676ac196db 100644
--- a/clang/lib/CodeGen/CGCall.h
+++ b/clang/lib/CodeGen/CGCall.h
@@ -377,6 +377,7 @@ public:
   Address getValue() const { return Addr; }
   bool isUnused() const { return IsUnused; }
   bool isExternallyDestructed() const { return IsExternallyDestructed; }
+  Address getAddress() const { return Addr; }
 };
 
 /// Adds attributes to \p F according to our \p CodeGenOpts and \p LangOpts, as
diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp
index 34319381901a..8c1c8ee455d2 100644
--- a/clang/lib/CodeGen/CGClass.cpp
+++ b/clang/lib/CodeGen/CGClass.cpp
@@ -139,8 +139,9 @@ Address CodeGenFunction::LoadCXXThisAddress() {
     CXXThisAlignment = CGM.getClassPointerAlignment(MD->getParent());
   }
 
-  llvm::Type *Ty = ConvertType(MD->getFunctionObjectParameterType());
-  return Address(LoadCXXThis(), Ty, CXXThisAlignment, KnownNonNull);
+  return makeNaturalAddressForPointer(
+      LoadCXXThis(), MD->getFunctionObjectParameterType(), CXXThisAlignment,
+      false, nullptr, nullptr, KnownNonNull);
 }
 
 /// Emit the address of a field using a member data pointer.
@@ -270,7 +271,7 @@ ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr,
   }
 
   // Apply the base offset.
-  llvm::Value *ptr = addr.getPointer();
+  llvm::Value *ptr = addr.emitRawPointer(CGF);
   ptr = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ptr, baseOffset, "add.ptr");
 
   // If we have a virtual component, the alignment of the result will
@@ -338,8 +339,8 @@ Address CodeGenFunction::GetAddressOfBaseClass(
     if (sanitizePerformTypeCheck()) {
       SanitizerSet SkippedChecks;
       SkippedChecks.set(SanitizerKind::Null, !NullCheckValue);
-      EmitTypeCheck(TCK_Upcast, Loc, Value.getPointer(),
-                    DerivedTy, DerivedAlign, SkippedChecks);
+      EmitTypeCheck(TCK_Upcast, Loc, Value.emitRawPointer(*this), DerivedTy,
+                    DerivedAlign, SkippedChecks);
     }
     return Value.withElementType(BaseValueTy);
   }
@@ -354,7 +355,7 @@ Address CodeGenFunction::GetAddressOfBaseClass(
     llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull");
     endBB = createBasicBlock("cast.end");
 
-    llvm::Value *isNull = Builder.CreateIsNull(Value.getPointer());
+    llvm::Value *isNull = Builder.CreateIsNull(Value);
     Builder.CreateCondBr(isNull, endBB, notNullBB);
     EmitBlock(notNullBB);
   }
@@ -363,14 +364,15 @@ Address CodeGenFunction::GetAddressOfBaseClass(
     SanitizerSet SkippedChecks;
     SkippedChecks.set(SanitizerKind::Null, true);
     EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc,
-                  Value.getPointer(), DerivedTy, DerivedAlign, SkippedChecks);
+                  Value.emitRawPointer(*this), DerivedTy, DerivedAlign,
+                  SkippedChecks);
   }
 
   // Compute the virtual offset.
   llvm::Value *VirtualOffset = nullptr;
   if (VBase) {
     VirtualOffset =
-      CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase);
+        CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase);
   }
 
   // Apply both offsets.
@@ -387,7 +389,7 @@ Address CodeGenFunction::GetAddressOfBaseClass(
     EmitBlock(endBB);
 
     llvm::PHINode *PHI = Builder.CreatePHI(PtrTy, 2, "cast.result");
-    PHI->addIncoming(Value.getPointer(), notNullBB);
+    PHI->addIncoming(Value.emitRawPointer(*this), notNullBB);
     PHI->addIncoming(llvm::Constant::getNullValue(PtrTy), origBB);
     Value = Value.withPointer(PHI, NotKnownNonNull);
   }
@@ -424,15 +426,19 @@ CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,
     CastNotNull = createBasicBlock("cast.notnull");
     CastEnd = createBasicBlock("cast.end");
 
-    llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr.getPointer());
+    llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr);
     Builder.CreateCondBr(IsNull, CastNull, CastNotNull);
     EmitBlock(CastNotNull);
   }
 
   // Apply the offset.
-  llvm::Value *Value = BaseAddr.getPointer();
-  Value = Builder.CreateInBoundsGEP(
-      Int8Ty, Value, Builder.CreateNeg(NonVirtualOffset), "sub.ptr");
+  Address Addr = BaseAddr.withElementType(Int8Ty);
+  Addr = Builder.CreateInBoundsGEP(
+      Addr, Builder.CreateNeg(NonVirtualOffset), Int8Ty,
+      CGM.getClassPointerAlignment(Derived), "sub.ptr");
+
+  // Just cast.
+  Addr = Addr.withElementType(DerivedValueTy);
 
   // Produce a PHI if we had a null-check.
   if (NullCheckValue) {
@@ -441,13 +447,15 @@ CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,
     Builder.CreateBr(CastEnd);
     EmitBlock(CastEnd);
 
+    llvm::Value *Value = Addr.emitRawPointer(*this);
     llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2);
     PHI->addIncoming(Value, CastNotNull);
     PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull);
-    Value = PHI;
+    return Address(PHI, Addr.getElementType(),
+                   CGM.getClassPointerAlignment(Derived));
   }
 
-  return Address(Value, DerivedValueTy, CGM.getClassPointerAlignment(Derived));
+  return Addr;
 }
 
 llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,
@@ -1719,7 +1727,7 @@ namespace {
       // Use the base class declaration location as inline DebugLocation. All
       // fields of the class are destroyed.
       DeclAsInlineDebugLocation InlineHere(CGF, *BaseClass);
-      EmitSanitizerDtorFieldsCallback(CGF, Addr.getPointer(),
+      EmitSanitizerDtorFieldsCallback(CGF, Addr.emitRawPointer(CGF),
                                       BaseSize.getQuantity());
 
       // Prevent the current stack frame from disappearing from the stack trace.
@@ -2022,7 +2030,7 @@ void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
 
   // Find the end of the array.
   llvm::Type *elementType = arrayBase.getElementType();
-  llvm::Value *arrayBegin = arrayBase.getPointer();
+  llvm::Value *arrayBegin = arrayBase.emitRawPointer(*this);
   llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(
       elementType, arrayBegin, numElements, "arrayctor.end");
 
@@ -2118,14 +2126,15 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
   Address This = ThisAVS.getAddress();
   LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace();
   LangAS ThisAS = D->getFunctionObjectParameterType().getAddressSpace();
-  llvm::Value *ThisPtr = This.getPointer();
+  llvm::Value *ThisPtr =
+      getAsNaturalPointerTo(This, D->getThisType()->getPointeeType());
 
   if (SlotAS != ThisAS) {
     unsigned TargetThisAS = getContext().getTargetAddressSpace(ThisAS);
     llvm::Type *NewType =
         llvm::PointerType::get(getLLVMContext(), TargetThisAS);
-    ThisPtr = getTargetHooks().performAddrSpaceCast(*this, This.getPointer(),
-                                                    ThisAS, SlotAS, NewType);
+    ThisPtr = getTargetHooks().performAddrSpaceCast(*this, ThisPtr, ThisAS,
+                                                    SlotAS, NewType);
   }
 
   // Push the this ptr.
@@ -2194,7 +2203,7 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
   const CXXRecordDecl *ClassDecl = D->getParent();
 
   if (!NewPointerIsChecked)
-    EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This.getPointer(),
+    EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This,
                   getContext().getRecordType(ClassDecl), CharUnits::Zero());
 
   if (D->isTrivial() && D->isDefaultConstructor()) {
@@ -2207,10 +2216,9 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
   // model that copy.
   if (isMemcpyEquivalentSpecialMember(D)) {
     assert(Args.size() == 2 && "unexpected argcount for trivial ctor");
-
     QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType();
-    Address Src = Address(Args[1].getRValue(*this).getScalarVal(), ConvertTypeForMem(SrcTy),
-                                      CGM.getNaturalTypeAlignment(SrcTy));
+    Address Src = makeNaturalAddressForPointer(
+        Args[1].getRValue(*this).getScalarVal(), SrcTy);
     LValue SrcLVal = MakeAddrLValue(Src, SrcTy);
     QualType DestTy = getContext().getTypeDeclType(ClassDecl);
     LValue DestLVal = MakeAddrLValue(This, DestTy);
@@ -2263,7 +2271,9 @@ void CodeGenFunction::EmitInheritedCXXConstructorCall(
     const CXXConstructorDecl *D, bool ForVirtualBase, Address This,
     bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) {
   CallArgList Args;
-  CallArg ThisArg(RValue::get(This.getPointer()), D->getThisType());
+  CallArg ThisArg(RValue::get(getAsNaturalPointerTo(
+                      This, D->getThisType()->getPointeeType())),
+                  D->getThisType());
 
   // Forward the parameters.
   if (InheritedFromVBase &&
@@ -2388,12 +2398,14 @@ CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
   CallArgList Args;
 
   // Push the this ptr.
-  Args.add(RValue::get(This.getPointer()), D->getThisType());
+  Args.add(RValue::get(getAsNaturalPointerTo(This, D->getThisType())),
+           D->getThisType());
 
   // Push the src ptr.
   QualType QT = *(FPT->param_type_begin());
   llvm::Type *t = CGM.getTypes().ConvertType(QT);
-  llvm::Value *SrcVal = Builder.CreateBitCast(Src.getPointer(), t);
+  llvm::Value *Val = getAsNaturalPointerTo(Src, D->getThisType());
+  llvm::Value *SrcVal = Builder.CreateBitCast(Val, t);
   Args.add(RValue::get(SrcVal), QT);
 
   // Skip over first argument (Src).
@@ -2418,7 +2430,9 @@ CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
 
   // this
   Address This = LoadCXXThisAddress();
-  DelegateArgs.add(RValue::get(This.getPointer()), (*I)->getType());
+  DelegateArgs.add(RValue::get(getAsNaturalPointerTo(
+                       This, (*I)->getType()->getPointeeType())),
+                   (*I)->getType());
   ++I;
 
   // FIXME: The location of the VTT parameter in the parameter list is
@@ -2775,7 +2789,7 @@ void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, Address Derived,
 
   if (MayBeNull) {
     llvm::Value *DerivedNotNull =
-        Builder.CreateIsNotNull(Derived.getPointer(), "cast.nonnull");
+        Builder.CreateIsNotNull(Derived.emitRawPointer(*this), "cast.nonnull");
 
     llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check");
     ContBlock = createBasicBlock("cast.cont");
@@ -2976,7 +2990,7 @@ void CodeGenFunction::EmitLambdaBlockInvokeBody() {
 
   QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
   Address ThisPtr = GetAddrOfBlockDecl(variable);
-  CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType);
+  CallArgs.add(RValue::get(getAsNaturalPointerTo(ThisPtr, ThisType)), ThisType);
 
   // Add the rest of the parameters.
   for (auto *param : BD->parameters())
@@ -3004,7 +3018,7 @@ void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) {
   QualType LambdaType = getContext().getRecordType(Lambda);
   QualType ThisType = getContext().getPointerType(LambdaType);
   Address ThisPtr = CreateMemTemp(LambdaType, "unused.capture");
-  CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType);
+  CallArgs.add(RValue::get(ThisPtr.emitRawPointer(*this)), ThisType);
 
   EmitLambdaDelegatingInvokeBody(MD, CallArgs);
 }
diff --git a/clang/lib/CodeGen/CGCleanup.cpp b/clang/lib/CodeGen/CGCleanup.cpp
index f87caf050eea..e6f8e6873004 100644
--- a/clang/lib/CodeGen/CGCleanup.cpp
+++ b/clang/lib/CodeGen/CGCleanup.cpp
@@ -27,7 +27,7 @@ bool DominatingValue<RValue>::saved_type::needsSaving(RValue rv) {
   if (rv.isScalar())
     return DominatingLLVMValue::needsSaving(rv.getScalarVal());
   if (rv.isAggregate())
-    return DominatingLLVMValue::needsSaving(rv.getAggregatePointer());
+    return DominatingValue<Address>::needsSaving(rv.getAggregateAddress());
   return true;
 }
 
@@ -35,69 +35,40 @@ DominatingValue<RValue>::saved_type
 DominatingValue<RValue>::saved_type::save(CodeGenFunction &CGF, RValue rv) {
   if (rv.isScalar()) {
     llvm::Value *V = rv.getScalarVal();
-
-    // These automatically dominate and don't need to be saved.
-    if (!DominatingLLVMValue::needsSaving(V))
-      return saved_type(V, nullptr, ScalarLiteral);
-
-    // Everything else needs an alloca.
-    Address addr =
-      CGF.CreateDefaultAlignTempAlloca(V->getType(), "saved-rvalue");
-    CGF.Builder.CreateStore(V, addr);
-    return saved_type(addr.getPointer(), nullptr, ScalarAddress);
+    return saved_type(DominatingLLVMValue::save(CGF, V),
+                      DominatingLLVMValue::needsSaving(V) ? ScalarAddress
+                                                          : ScalarLiteral);
   }
 
   if (rv.isComplex()) {
     CodeGenFunction::ComplexPairTy V = rv.getComplexVal();
-    llvm::Type *ComplexTy =
-        llvm::StructType::get(V.first->getType(), V.second->getType());
-    Address addr = CGF.CreateDefaultAlignTempAlloca(ComplexTy, "saved-complex");
-    CGF.Builder.CreateStore(V.first, CGF.Builder.CreateStructGEP(addr, 0));
-    CGF.Builder.CreateStore(V.second, CGF.Builder.CreateStructGEP(addr, 1));
-    return saved_type(addr.getPointer(), nullptr, ComplexAddress);
+    return saved_type(DominatingLLVMValue::save(CGF, V.first),
+                      DominatingLLVMValue::save(CGF, V.second));
   }
 
   assert(rv.isAggregate());
-  Address V = rv.getAggregateAddress(); // TODO: volatile?
-  if (!DominatingLLVMValue::needsSaving(V.getPointer()))
-    return saved_type(V.getPointer(), V.getElementType(), AggregateLiteral,
-                      V.getAlignment().getQuantity());
-
-  Address addr =
-    CGF.CreateTempAlloca(V.getType(), CGF.getPointerAlign(), "saved-rvalue");
-  CGF.Builder.CreateStore(V.getPointer(), addr);
-  return saved_type(addr.getPointer(), V.getElementType(), AggregateAddress,
-                    V.getAlignment().getQuantity());
+  Address V = rv.getAggregateAddress();
+  return saved_type(
+      DominatingValue<Address>::save(CGF, V), rv.isVolatileQualified(),
+      DominatingValue<Address>::needsSaving(V) ? AggregateAddress
+                                               : AggregateLiteral);
 }
 
 /// Given a saved r-value produced by SaveRValue, perform the code
 /// necessary to restore it to usability at the current insertion
 /// point.
 RValue DominatingValue<RValue>::saved_type::restore(CodeGenFunction &CGF) {
-  auto getSavingAddress = [&](llvm::Value *value) {
-    auto *AI = cast<llvm::AllocaInst>(value);
-    return Address(value, AI->getAllocatedType(),
-                   CharUnits::fromQuantity(AI->getAlign().value()));
-  };
   switch (K) {
   case ScalarLiteral:
-    return RValue::get(Value);
   case ScalarAddress:
-    return RValue::get(CGF.Builder.CreateLoad(getSavingAddress(Value)));
+    return RValue::get(DominatingLLVMValue::restore(CGF, Vals.first));
   case AggregateLiteral:
+  case AggregateAddress:
     return RValue::getAggregate(
-        Address(Value, ElementType, CharUnits::fromQuantity(Align)));
-  case AggregateAddress: {
-    auto addr = CGF.Builder.CreateLoad(getSavingAddress(Value));
-    return RValue::getAggregate(
-        Address(addr, ElementType, CharUnits::fromQuantity(Align)));
-  }
+        DominatingValue<Address>::restore(CGF, AggregateAddr), IsVolatile);
   case ComplexAddress: {
-    Address address = getSavingAddress(Value);
-    llvm::Value *real =
-        CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(address, 0));
-    llvm::Value *imag =
-        CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(address, 1));
+    llvm::Value *real = DominatingLLVMValue::restore(CGF, Vals.first);
+    llvm::Value *imag = DominatingLLVMValue::restore(CGF, Vals.second);
     return RValue::getComplex(real, imag);
   }
   }
@@ -294,14 +265,14 @@ void EHScopeStack::popNullFixups() {
     BranchFixups.pop_back();
 }
 
-Address CodeGenFunction::createCleanupActiveFlag() {
+RawAddress CodeGenFunction::createCleanupActiveFlag() {
   // Create a variable to decide whether the cleanup needs to be run.
-  Address active = CreateTempAllocaWithoutCast(
+  RawAddress active = CreateTempAllocaWithoutCast(
       Builder.getInt1Ty(), CharUnits::One(), "cleanup.cond");
 
   // Initialize it to false at a site that's guaranteed to be run
   // before each evaluation.
-  setBeforeOutermostConditional(Builder.getFalse(), active);
+  setBeforeOutermostConditional(Builder.getFalse(), active, *this);
 
   // Initialize it to true at the current location.
   Builder.CreateStore(Builder.getTrue(), active);
@@ -309,7 +280,7 @@ Address CodeGenFunction::createCleanupActiveFlag() {
   return active;
 }
 
-void CodeGenFunction::initFullExprCleanupWithFlag(Address ActiveFlag) {
+void CodeGenFunction::initFullExprCleanupWithFlag(RawAddress ActiveFlag) {
   // Set that as the active flag in the cleanup.
   EHCleanupScope &cleanup = cast<EHCleanupScope>(*EHStack.begin());
   assert(!cleanup.hasActiveFlag() && "cleanup already has active flag?");
@@ -322,15 +293,17 @@ void CodeGenFunction::initFullExprCleanupWithFlag(Address ActiveFlag) {
 void EHScopeStack::Cleanup::anchor() {}
 
 static void createStoreInstBefore(llvm::Value *value, Address addr,
-                                  llvm::Instruction *beforeInst) {
-  auto store = new llvm::StoreInst(value, addr.getPointer(), beforeInst);
+                                  llvm::Instruction *beforeInst,
+                                  CodeGenFunction &CGF) {
+  auto store = new llvm::StoreInst(value, addr.emitRawPointer(CGF), beforeInst);
   store->setAlignment(addr.getAlignment().getAsAlign());
 }
 
 static llvm::LoadInst *createLoadInstBefore(Address addr, const Twine &name,
-                                            llvm::Instruction *beforeInst) {
-  return new llvm::LoadInst(addr.getElementType(), addr.getPointer(), name,
-                            false, addr.getAlignment().getAsAlign(),
+                                            llvm::Instruction *beforeInst,
+                                            CodeGenFunction &CGF) {
+  return new llvm::LoadInst(addr.getElementType(), addr.emitRawPointer(CGF),
+                            name, false, addr.getAlignment().getAsAlign(),
                             beforeInst);
 }
 
@@ -357,8 +330,8 @@ static void ResolveAllBranchFixups(CodeGenFunction &CGF,
     // entry which we're currently popping.
     if (Fixup.OptimisticBranchBlock == nullptr) {
       createStoreInstBefore(CGF.Builder.getInt32(Fixup.DestinationIndex),
-                            CGF.getNormalCleanupDestSlot(),
-                            Fixup.InitialBranch);
+                            CGF.getNormalCleanupDestSlot(), Fixup.InitialBranch,
+                            CGF);
       Fixup.InitialBranch->setSuccessor(0, CleanupEntry);
     }
 
@@ -385,7 +358,7 @@ static llvm::SwitchInst *TransitionToCleanupSwitch(CodeGenFunction &CGF,
   if (llvm::BranchInst *Br = dyn_cast<llvm::BranchInst>(Term)) {
     assert(Br->isUnconditional());
     auto Load = createLoadInstBefore(CGF.getNormalCleanupDestSlot(),
-                                     "cleanup.dest", Term);
+                                     "cleanup.dest", Term, CGF);
     llvm::SwitchInst *Switch =
       llvm::SwitchInst::Create(Load, Br->getSuccessor(0), 4, Block);
     Br->eraseFromParent();
@@ -513,8 +486,8 @@ void CodeGenFunction::PopCleanupBlocks(
     I += Header.getSize();
 
     if (Header.isConditional()) {
-      Address ActiveFlag =
-          reinterpret_cast<Address &>(LifetimeExtendedCleanupStack[I]);
+      RawAddress ActiveFlag =
+          reinterpret_cast<RawAddress &>(LifetimeExtendedCleanupStack[I]);
       initFullExprCleanupWithFlag(ActiveFlag);
       I += sizeof(ActiveFlag);
     }
@@ -888,7 +861,7 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
         if (NormalCleanupDestSlot->hasOneUse()) {
           NormalCleanupDestSlot->user_back()->eraseFromParent();
           NormalCleanupDestSlot->eraseFromParent();
-          NormalCleanupDest = Address::invalid();
+          NormalCleanupDest = RawAddress::invalid();
         }
 
         llvm::BasicBlock *BranchAfter = Scope.getBranchAfterBlock(0);
@@ -912,9 +885,8 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
         // pass the abnormal exit flag to Fn (SEH cleanup)
         cleanupFlags.setHasExitSwitch();
 
-        llvm::LoadInst *Load =
-          createLoadInstBefore(getNormalCleanupDestSlot(), "cleanup.dest",
-                               nullptr);
+        llvm::LoadInst *Load = createLoadInstBefore(
+            getNormalCleanupDestSlot(), "cleanup.dest", nullptr, *this);
         llvm::SwitchInst *Switch =
           llvm::SwitchInst::Create(Load, Default, SwitchCapacity);
 
@@ -961,8 +933,8 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
         if (!Fixup.Destination) continue;
         if (!Fixup.OptimisticBranchBlock) {
           createStoreInstBefore(Builder.getInt32(Fixup.DestinationIndex),
-                                getNormalCleanupDestSlot(),
-                                Fixup.InitialBranch);
+                                getNormalCleanupDestSlot(), Fixup.InitialBranch,
+                                *this);
           Fixup.InitialBranch->setSuccessor(0, NormalEntry);
         }
         Fixup.OptimisticBranchBlock = NormalExit;
@@ -1135,7 +1107,7 @@ void CodeGenFunction::EmitBranchThroughCleanup(JumpDest Dest) {
 
   // Store the index at the start.
   llvm::ConstantInt *Index = Builder.getInt32(Dest.getDestIndex());
-  createStoreInstBefore(Index, getNormalCleanupDestSlot(), BI);
+  createStoreInstBefore(Index, getNormalCleanupDestSlot(), BI, *this);
 
   // Adjust BI to point to the first cleanup block.
   {
@@ -1269,9 +1241,9 @@ static void SetupCleanupBlockActivation(CodeGenFunction &CGF,
     // If we're in a conditional block, ignore the dominating IP and
     // use the outermost conditional branch.
     if (CGF.isInConditionalBranch()) {
-      CGF.setBeforeOutermostConditional(value, var);
+      CGF.setBeforeOutermostConditional(value, var, CGF);
     } else {
-      createStoreInstBefore(value, var, dominatingIP);
+      createStoreInstBefore(value, var, dominatingIP, CGF);
     }
   }
 
@@ -1321,7 +1293,7 @@ void CodeGenFunction::DeactivateCleanupBlock(EHScopeStack::stable_iterator C,
   Scope.setActive(false);
 }
 
-Address CodeGenFunction::getNormalCleanupDestSlot() {
+RawAddress CodeGenFunction::getNormalCleanupDestSlot() {
   if (!NormalCleanupDest.isValid())
     NormalCleanupDest =
       CreateDefaultAlignTempAlloca(Builder.getInt32Ty(), "cleanup.dest.slot");
diff --git a/clang/lib/CodeGen/CGCleanup.h b/clang/lib/CodeGen/CGCleanup.h
index 7a7344c07160..03e4a29d7b3d 100644
--- a/clang/lib/CodeGen/CGCleanup.h
+++ b/clang/lib/CodeGen/CGCleanup.h
@@ -333,7 +333,7 @@ public:
   Address getActiveFlag() const {
     return ActiveFlag;
   }
-  void setActiveFlag(Address Var) {
+  void setActiveFlag(RawAddress Var) {
     assert(Var.getAlignment().isOne());
     ActiveFlag = Var;
   }
diff --git a/clang/lib/CodeGen/CGCoroutine.cpp b/clang/lib/CodeGen/CGCoroutine.cpp
index b7142ec08af9..93ca711f716f 100644
--- a/clang/lib/CodeGen/CGCoroutine.cpp
+++ b/clang/lib/CodeGen/CGCoroutine.cpp
@@ -867,8 +867,8 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
     EmitStmt(S.getPromiseDeclStmt());
 
     Address PromiseAddr = GetAddrOfLocalVar(S.getPromiseDecl());
-    auto *PromiseAddrVoidPtr =
-        new llvm::BitCastInst(PromiseAddr.getPointer(), VoidPtrTy, "", CoroId);
+    auto *PromiseAddrVoidPtr = new llvm::BitCastInst(
+        PromiseAddr.emitRawPointer(*this), VoidPtrTy, "", CoroId);
     // Update CoroId to refer to the promise. We could not do it earlier because
     // promise local variable was not emitted yet.
     CoroId->setArgOperand(1, PromiseAddrVoidPtr);
diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index 2ef5ed04af30..267f2e40a7bb 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -1461,7 +1461,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
   bool EmitDebugInfo = DI && CGM.getCodeGenOpts().hasReducedDebugInfo();
 
   Address address = Address::invalid();
-  Address AllocaAddr = Address::invalid();
+  RawAddress AllocaAddr = RawAddress::invalid();
   Address OpenMPLocalAddr = Address::invalid();
   if (CGM.getLangOpts().OpenMPIRBuilder)
     OpenMPLocalAddr = OMPBuilderCBHelpers::getAddressOfLocalVariable(*this, &D);
@@ -1524,7 +1524,10 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
       // return slot, so that we can elide the copy when returning this
       // variable (C++0x [class.copy]p34).
       address = ReturnValue;
-      AllocaAddr = ReturnValue;
+      AllocaAddr =
+          RawAddress(ReturnValue.emitRawPointer(*this),
+                     ReturnValue.getElementType(), ReturnValue.getAlignment());
+      ;
 
       if (const RecordType *RecordTy = Ty->getAs<RecordType>()) {
         const auto *RD = RecordTy->getDecl();
@@ -1535,7 +1538,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
           // to this variable. Set it to zero to indicate that NRVO was not
           // applied.
           llvm::Value *Zero = Builder.getFalse();
-          Address NRVOFlag =
+          RawAddress NRVOFlag =
               CreateTempAlloca(Zero->getType(), CharUnits::One(), "nrvo");
           EnsureInsertPoint();
           Builder.CreateStore(Zero, NRVOFlag);
@@ -1678,7 +1681,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
   }
 
   if (D.hasAttr<AnnotateAttr>() && HaveInsertPoint())
-    EmitVarAnnotations(&D, address.getPointer());
+    EmitVarAnnotations(&D, address.emitRawPointer(*this));
 
   // Make sure we call @llvm.lifetime.end.
   if (emission.useLifetimeMarkers())
@@ -1851,12 +1854,13 @@ void CodeGenFunction::emitZeroOrPatternForAutoVarInit(QualType type,
     llvm::Value *BaseSizeInChars =
         llvm::ConstantInt::get(IntPtrTy, EltSize.getQuantity());
     Address Begin = Loc.withElementType(Int8Ty);
-    llvm::Value *End = Builder.CreateInBoundsGEP(
-        Begin.getElementType(), Begin.getPointer(), SizeVal, "vla.end");
+    llvm::Value *End = Builder.CreateInBoundsGEP(Begin.getElementType(),
+                                                 Begin.emitRawPointer(*this),
+                                                 SizeVal, "vla.end");
     llvm::BasicBlock *OriginBB = Builder.GetInsertBlock();
     EmitBlock(LoopBB);
     llvm::PHINode *Cur = Builder.CreatePHI(Begin.getType(), 2, "vla.cur");
-    Cur->addIncoming(Begin.getPointer(), OriginBB);
+    Cur->addIncoming(Begin.emitRawPointer(*this), OriginBB);
     CharUnits CurAlign = Loc.getAlignment().alignmentOfArrayElement(EltSize);
     auto *I =
         Builder.CreateMemCpy(Address(Cur, Int8Ty, CurAlign),
@@ -2283,7 +2287,7 @@ void CodeGenFunction::emitDestroy(Address addr, QualType type,
     checkZeroLength = false;
   }
 
-  llvm::Value *begin = addr.getPointer();
+  llvm::Value *begin = addr.emitRawPointer(*this);
   llvm::Value *end =
       Builder.CreateInBoundsGEP(addr.getElementType(), begin, length);
   emitArrayDestroy(begin, end, type, elementAlign, destroyer,
@@ -2543,7 +2547,7 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg,
   }
 
   Address DeclPtr = Address::invalid();
-  Address AllocaPtr = Address::invalid();
+  RawAddress AllocaPtr = Address::invalid();
   bool DoStore = false;
   bool IsScalar = hasScalarEvaluationKind(Ty);
   bool UseIndirectDebugAddress = false;
@@ -2555,8 +2559,8 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg,
     // Indirect argument is in alloca address space, which may be different
     // from the default address space.
     auto AllocaAS = CGM.getASTAllocaAddressSpace();
-    auto *V = DeclPtr.getPointer();
-    AllocaPtr = DeclPtr;
+    auto *V = DeclPtr.emitRawPointer(*this);
+    AllocaPtr = RawAddress(V, DeclPtr.getElementType(), DeclPtr.getAlignment());
 
     // For truly ABI indirect arguments -- those that are not `byval` -- store
     // the address of the argument on the stack to preserve debug information.
@@ -2695,7 +2699,7 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg,
   }
 
   if (D.hasAttr<AnnotateAttr>())
-    EmitVarAnnotations(&D, DeclPtr.getPointer());
+    EmitVarAnnotations(&D, DeclPtr.emitRawPointer(*this));
 
   // We can only check return value nullability if all arguments to the
   // function satisfy their nullability preconditions. This makes it necessary
diff --git a/clang/lib/CodeGen/CGException.cpp b/clang/lib/CodeGen/CGException.cpp
index 5a9d06da12de..34f289334a7d 100644
--- a/clang/lib/CodeGen/CGException.cpp
+++ b/clang/lib/CodeGen/CGException.cpp
@@ -397,7 +397,7 @@ namespace {
 void CodeGenFunction::EmitAnyExprToExn(const Expr *e, Address addr) {
   // Make sure the exception object is cleaned up if there's an
   // exception during initialization.
-  pushFullExprCleanup<FreeException>(EHCleanup, addr.getPointer());
+  pushFullExprCleanup<FreeException>(EHCleanup, addr.emitRawPointer(*this));
   EHScopeStack::stable_iterator cleanup = EHStack.stable_begin();
 
   // __cxa_allocate_exception returns a void*;  we need to cast this
@@ -416,8 +416,8 @@ void CodeGenFunction::EmitAnyExprToExn(const Expr *e, Address addr) {
                    /*IsInit*/ true);
 
   // Deactivate the cleanup block.
-  DeactivateCleanupBlock(cleanup,
-                         cast<llvm::Instruction>(typedAddr.getPointer()));
+  DeactivateCleanupBlock(
+      cleanup, cast<llvm::Instruction>(typedAddr.emitRawPointer(*this)));
 }
 
 Address CodeGenFunction::getExceptionSlot() {
@@ -1834,7 +1834,8 @@ Address CodeGenFunction::recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,
                                                    llvm::Value *ParentFP) {
   llvm::CallInst *RecoverCall = nullptr;
   CGBuilderTy Builder(*this, AllocaInsertPt);
-  if (auto *ParentAlloca = dyn_cast<llvm::AllocaInst>(ParentVar.getPointer())) {
+  if (auto *ParentAlloca =
+          dyn_cast_or_null<llvm::AllocaInst>(ParentVar.getBasePointer())) {
     // Mark the variable escaped if nobody else referenced it and compute the
     // localescape index.
     auto InsertPair = ParentCGF.EscapedLocals.insert(
@@ -1851,8 +1852,8 @@ Address CodeGenFunction::recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,
     // If the parent didn't have an alloca, we're doing some nested outlining.
     // Just clone the existing localrecover call, but tweak the FP argument to
     // use our FP value. All other arguments are constants.
-    auto *ParentRecover =
-        cast<llvm::IntrinsicInst>(ParentVar.getPointer()->stripPointerCasts());
+    auto *ParentRecover = cast<llvm::IntrinsicInst>(
+        ParentVar.emitRawPointer(*this)->stripPointerCasts());
     assert(ParentRecover->getIntrinsicID() == llvm::Intrinsic::localrecover &&
            "expected alloca or localrecover in parent LocalDeclMap");
     RecoverCall = cast<llvm::CallInst>(ParentRecover->clone());
@@ -1925,7 +1926,8 @@ void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF,
         if (isa<ImplicitParamDecl>(D) &&
             D->getType() == getContext().VoidPtrTy) {
           assert(D->getName().starts_with("frame_pointer"));
-          FramePtrAddrAlloca = cast<llvm::AllocaInst>(I.second.getPointer());
+          FramePtrAddrAlloca =
+              cast<llvm::AllocaInst>(I.second.getBasePointer());
           break;
         }
       }
@@ -1986,7 +1988,8 @@ void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF,
         LValue ThisFieldLValue =
             EmitLValueForLambdaField(LambdaThisCaptureField);
         if (!LambdaThisCaptureField->getType()->isPointerType()) {
-          CXXThisValue = ThisFieldLValue.getAddress(*this).getPointer();
+          CXXThisValue =
+              ThisFieldLValue.getAddress(*this).emitRawPointer(*this);
         } else {
           CXXThisValue = EmitLoadOfLValue(ThisFieldLValue, SourceLocation())
                              .getScalarVal();
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 85f5d739cef4..59f7cd452610 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -65,21 +65,21 @@ static llvm::cl::opt<bool> ClSanitizeDebugDeoptimization(
 
 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
 /// block.
-Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty,
-                                                     CharUnits Align,
-                                                     const Twine &Name,
-                                                     llvm::Value *ArraySize) {
+RawAddress
+CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits Align,
+                                             const Twine &Name,
+                                             llvm::Value *ArraySize) {
   auto Alloca = CreateTempAlloca(Ty, Name, ArraySize);
   Alloca->setAlignment(Align.getAsAlign());
-  return Address(Alloca, Ty, Align, KnownNonNull);
+  return RawAddress(Alloca, Ty, Align, KnownNonNull);
 }
 
 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
 /// block. The alloca is casted to default address space if necessary.
-Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align,
-                                          const Twine &Name,
-                                          llvm::Value *ArraySize,
-                                          Address *AllocaAddr) {
+RawAddress CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align,
+                                             const Twine &Name,
+                                             llvm::Value *ArraySize,
+                                             RawAddress *AllocaAddr) {
   auto Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize);
   if (AllocaAddr)
     *AllocaAddr = Alloca;
@@ -101,7 +101,7 @@ Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align,
         Ty->getPointerTo(DestAddrSpace), /*non-null*/ true);
   }
 
-  return Address(V, Ty, Align, KnownNonNull);
+  return RawAddress(V, Ty, Align, KnownNonNull);
 }
 
 /// CreateTempAlloca - This creates an alloca and inserts it into the entry
@@ -120,28 +120,29 @@ llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
 /// default alignment of the corresponding LLVM type, which is *not*
 /// guaranteed to be related in any way to the expected alignment of
 /// an AST type that might have been lowered to Ty.
-Address CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty,
-                                                      const Twine &Name) {
+RawAddress CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty,
+                                                         const Twine &Name) {
   CharUnits Align =
       CharUnits::fromQuantity(CGM.getDataLayout().getPrefTypeAlign(Ty));
   return CreateTempAlloca(Ty, Align, Name);
 }
 
-Address CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) {
+RawAddress CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) {
   CharUnits Align = getContext().getTypeAlignInChars(Ty);
   return CreateTempAlloca(ConvertType(Ty), Align, Name);
 }
 
-Address CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name,
-                                       Address *Alloca) {
+RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name,
+                                          RawAddress *Alloca) {
   // FIXME: Should we prefer the preferred type alignment here?
   return CreateMemTemp(Ty, getContext().getTypeAlignInChars(Ty), Name, Alloca);
 }
 
-Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align,
-                                       const Twine &Name, Address *Alloca) {
-  Address Result = CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name,
-                                    /*ArraySize=*/nullptr, Alloca);
+RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align,
+                                          const Twine &Name,
+                                          RawAddress *Alloca) {
+  RawAddress Result = CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name,
+                                       /*ArraySize=*/nullptr, Alloca);
 
   if (Ty->isConstantMatrixType()) {
     auto *ArrayTy = cast<llvm::ArrayType>(Result.getElementType());
@@ -154,13 +155,14 @@ Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align,
   return Result;
 }
 
-Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, CharUnits Align,
-                                                  const Twine &Name) {
+RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
+                                                     CharUnits Align,
+                                                     const Twine &Name) {
   return CreateTempAllocaWithoutCast(ConvertTypeForMem(Ty), Align, Name);
 }
 
-Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
-                                                  const Twine &Name) {
+RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
+                                                     const Twine &Name) {
   return CreateMemTempWithoutCast(Ty, getContext().getTypeAlignInChars(Ty),
                                   Name);
 }
@@ -359,7 +361,7 @@ pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M,
     } else {
       CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor(
           GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete));
-      CleanupArg = cast<llvm::Constant>(ReferenceTemporary.getPointer());
+      CleanupArg = cast<llvm::Constant>(ReferenceTemporary.emitRawPointer(CGF));
     }
     CGF.CGM.getCXXABI().registerGlobalDtor(
         CGF, *cast<VarDecl>(M->getExtendingDecl()), CleanupFn, CleanupArg);
@@ -384,10 +386,10 @@ pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M,
   }
 }
 
-static Address createReferenceTemporary(CodeGenFunction &CGF,
-                                        const MaterializeTemporaryExpr *M,
-                                        const Expr *Inner,
-                                        Address *Alloca = nullptr) {
+static RawAddress createReferenceTemporary(CodeGenFunction &CGF,
+                                           const MaterializeTemporaryExpr *M,
+                                           const Expr *Inner,
+                                           RawAddress *Alloca = nullptr) {
   auto &TCG = CGF.getTargetHooks();
   switch (M->getStorageDuration()) {
   case SD_FullExpression:
@@ -416,7 +418,7 @@ static Address createReferenceTemporary(CodeGenFunction &CGF,
               GV->getValueType()->getPointerTo(
                   CGF.getContext().getTargetAddressSpace(LangAS::Default)));
         // FIXME: Should we put the new global into a COMDAT?
-        return Address(C, GV->getValueType(), alignment);
+        return RawAddress(C, GV->getValueType(), alignment);
       }
     return CGF.CreateMemTemp(Ty, "ref.tmp", Alloca);
   }
@@ -448,7 +450,7 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
   auto ownership = M->getType().getObjCLifetime();
   if (ownership != Qualifiers::OCL_None &&
       ownership != Qualifiers::OCL_ExplicitNone) {
-    Address Object = createReferenceTemporary(*this, M, E);
+    RawAddress Object = createReferenceTemporary(*this, M, E);
     if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) {
       llvm::Type *Ty = ConvertTypeForMem(E->getType());
       Object = Object.withElementType(Ty);
@@ -502,8 +504,8 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
   }
 
   // Create and initialize the reference temporary.
-  Address Alloca = Address::invalid();
-  Address Object = createReferenceTemporary(*this, M, E, &Alloca);
+  RawAddress Alloca = Address::invalid();
+  RawAddress Object = createReferenceTemporary(*this, M, E, &Alloca);
   if (auto *Var = dyn_cast<llvm::GlobalVariable>(
           Object.getPointer()->stripPointerCasts())) {
     llvm::Type *TemporaryType = ConvertTypeForMem(E->getType());
@@ -1111,12 +1113,12 @@ llvm::Value *CodeGenFunction::EmitCountedByFieldExpr(
   } else if (const MemberExpr *ME = dyn_cast<MemberExpr>(StructBase)) {
     LValue LV = EmitMemberExpr(ME);
     Address Addr = LV.getAddress(*this);
-    Res = Addr.getPointer();
+    Res = Addr.emitRawPointer(*this);
   } else if (StructBase->getType()->isPointerType()) {
     LValueBaseInfo BaseInfo;
     TBAAAccessInfo TBAAInfo;
     Address Addr = EmitPointerWithAlignment(StructBase, &BaseInfo, &TBAAInfo);
-    Res = Addr.getPointer();
+    Res = Addr.emitRawPointer(*this);
   } else {
     return nullptr;
   }
@@ -1282,8 +1284,7 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo,
           if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) {
             if (BaseInfo)
               BaseInfo->mergeForCast(TargetTypeBaseInfo);
-            Addr = Address(Addr.getPointer(), Addr.getElementType(), Align,
-                           IsKnownNonNull);
+            Addr.setAlignment(Align);
           }
         }
 
@@ -1300,8 +1301,8 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo,
             CGF.ConvertTypeForMem(E->getType()->getPointeeType());
         Addr = Addr.withElementType(ElemTy);
         if (CE->getCastKind() == CK_AddressSpaceConversion)
-          Addr = CGF.Builder.CreateAddrSpaceCast(Addr,
-                                                 CGF.ConvertType(E->getType()));
+          Addr = CGF.Builder.CreateAddrSpaceCast(
+              Addr, CGF.ConvertType(E->getType()), ElemTy);
         return Addr;
       }
       break;
@@ -1364,10 +1365,9 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo,
   // TODO: conditional operators, comma.
 
   // Otherwise, use the alignment of the type.
-  CharUnits Align =
-      CGF.CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo);
-  llvm::Type *ElemTy = CGF.ConvertTypeForMem(E->getType()->getPointeeType());
-  return Address(CGF.EmitScalarExpr(E), ElemTy, Align, IsKnownNonNull);
+  return CGF.makeNaturalAddressForPointer(
+      CGF.EmitScalarExpr(E), E->getType()->getPointeeType(), CharUnits(),
+      /*ForPointeeType=*/true, BaseInfo, TBAAInfo, IsKnownNonNull);
 }
 
 /// EmitPointerWithAlignment - Given an expression of pointer type, try to
@@ -1468,8 +1468,7 @@ LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
       if (IsBaseCXXThis || isa<DeclRefExpr>(ME->getBase()))
         SkippedChecks.set(SanitizerKind::Null, true);
     }
-    EmitTypeCheck(TCK, E->getExprLoc(), LV.getPointer(*this), E->getType(),
-                  LV.getAlignment(), SkippedChecks);
+    EmitTypeCheck(TCK, E->getExprLoc(), LV, E->getType(), SkippedChecks);
   }
   return LV;
 }
@@ -1581,11 +1580,11 @@ LValue CodeGenFunction::EmitLValueHelper(const Expr *E,
       // Defend against branches out of gnu statement expressions surrounded by
       // cleanups.
       Address Addr = LV.getAddress(*this);
-      llvm::Value *V = Addr.getPointer();
+      llvm::Value *V = Addr.getBasePointer();
       Scope.ForceCleanup({&V});
-      return LValue::MakeAddr(Addr.withPointer(V, Addr.isKnownNonNull()),
-                              LV.getType(), getContext(), LV.getBaseInfo(),
-                              LV.getTBAAInfo());
+      Addr.replaceBasePointer(V);
+      return LValue::MakeAddr(Addr, LV.getType(), getContext(),
+                              LV.getBaseInfo(), LV.getTBAAInfo());
     }
     // FIXME: Is it possible to create an ExprWithCleanups that produces a
     // bitfield lvalue or some other non-simple lvalue?
@@ -1929,7 +1928,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
                                                LValueBaseInfo BaseInfo,
                                                TBAAAccessInfo TBAAInfo,
                                                bool isNontemporal) {
-  if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer()))
+  if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getBasePointer()))
     if (GV->isThreadLocal())
       Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV),
                               NotKnownNonNull);
@@ -2039,8 +2038,9 @@ llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
 
 // Convert the pointer of \p Addr to a pointer to a vector (the value type of
 // MatrixType), if it points to a array (the memory type of MatrixType).
-static Address MaybeConvertMatrixAddress(Address Addr, CodeGenFunction &CGF,
-                                         bool IsVector = true) {
+static RawAddress MaybeConvertMatrixAddress(RawAddress Addr,
+                                            CodeGenFunction &CGF,
+                                            bool IsVector = true) {
   auto *ArrayTy = dyn_cast<llvm::ArrayType>(Addr.getElementType());
   if (ArrayTy && IsVector) {
     auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
@@ -2077,7 +2077,7 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr,
                                         LValueBaseInfo BaseInfo,
                                         TBAAAccessInfo TBAAInfo,
                                         bool isInit, bool isNontemporal) {
-  if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer()))
+  if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getBasePointer()))
     if (GV->isThreadLocal())
       Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV),
                               NotKnownNonNull);
@@ -2432,14 +2432,12 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
       assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
       llvm::Type *ResultType = IntPtrTy;
       Address dst = EmitPointerWithAlignment(Dst.getBaseIvarExp());
-      llvm::Value *RHS = dst.getPointer();
+      llvm::Value *RHS = dst.emitRawPointer(*this);
       RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast");
-      llvm::Value *LHS =
-        Builder.CreatePtrToInt(LvalueDst.getPointer(), ResultType,
-                               "sub.ptr.lhs.cast");
+      llvm::Value *LHS = Builder.CreatePtrToInt(LvalueDst.emitRawPointer(*this),
+                                                ResultType, "sub.ptr.lhs.cast");
       llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset");
-      CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst,
-                                              BytesBetween);
+      CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst, BytesBetween);
     } else if (Dst.isGlobalObjCRef()) {
       CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst,
                                                 Dst.isThreadLocalRef());
@@ -2770,12 +2768,9 @@ CodeGenFunction::EmitLoadOfReference(LValue RefLVal,
   llvm::LoadInst *Load =
       Builder.CreateLoad(RefLVal.getAddress(*this), RefLVal.isVolatile());
   CGM.DecorateInstructionWithTBAA(Load, RefLVal.getTBAAInfo());
-
-  QualType PointeeType = RefLVal.getType()->getPointeeType();
-  CharUnits Align = CGM.getNaturalTypeAlignment(
-      PointeeType, PointeeBaseInfo, PointeeTBAAInfo,
-      /* forPointeeType= */ true);
-  return Address(Load, ConvertTypeForMem(PointeeType), Align);
+  return makeNaturalAddressForPointer(Load, RefLVal.getType()->getPointeeType(),
+                                      CharUnits(), /*ForPointeeType=*/true,
+                                      PointeeBaseInfo, PointeeTBAAInfo);
 }
 
 LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) {
@@ -2792,10 +2787,9 @@ Address CodeGenFunction::EmitLoadOfPointer(Address Ptr,
                                            LValueBaseInfo *BaseInfo,
                                            TBAAAccessInfo *TBAAInfo) {
   llvm::Value *Addr = Builder.CreateLoad(Ptr);
-  return Address(Addr, ConvertTypeForMem(PtrTy->getPointeeType()),
-                 CGM.getNaturalTypeAlignment(PtrTy->getPointeeType(), BaseInfo,
-                                             TBAAInfo,
-                                             /*forPointeeType=*/true));
+  return makeNaturalAddressForPointer(Addr, PtrTy->getPointeeType(),
+                                      CharUnits(), /*ForPointeeType=*/true,
+                                      BaseInfo, TBAAInfo);
 }
 
 LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr,
@@ -2991,7 +2985,7 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
                                         /* BaseInfo= */ nullptr,
                                         /* TBAAInfo= */ nullptr,
                                         /* forPointeeType= */ true);
-        Addr = Address(Val, ConvertTypeForMem(E->getType()), Alignment);
+        Addr = makeNaturalAddressForPointer(Val, T, Alignment);
       }
       return MakeAddrLValue(Addr, T, AlignmentSource::Decl);
     }
@@ -3023,11 +3017,12 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
             EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD),
                                     CapturedStmtInfo->getContextValue());
         Address LValueAddress = CapLVal.getAddress(*this);
-        CapLVal = MakeAddrLValue(
-            Address(LValueAddress.getPointer(), LValueAddress.getElementType(),
-                    getContext().getDeclAlign(VD)),
-            CapLVal.getType(), LValueBaseInfo(AlignmentSource::Decl),
-            CapLVal.getTBAAInfo());
+        CapLVal = MakeAddrLValue(Address(LValueAddress.emitRawPointer(*this),
+                                         LValueAddress.getElementType(),
+                                         getContext().getDeclAlign(VD)),
+                                 CapLVal.getType(),
+                                 LValueBaseInfo(AlignmentSource::Decl),
+                                 CapLVal.getTBAAInfo());
         // Mark lvalue as nontemporal if the variable is marked as nontemporal
         // in simd context.
         if (getLangOpts().OpenMP &&
@@ -3083,7 +3078,8 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
     // Handle threadlocal function locals.
     if (VD->getTLSKind() != VarDecl::TLS_None)
       addr = addr.withPointer(
-          Builder.CreateThreadLocalAddress(addr.getPointer()), NotKnownNonNull);
+          Builder.CreateThreadLocalAddress(addr.getBasePointer()),
+          NotKnownNonNull);
 
     // Check for OpenMP threadprivate variables.
     if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&
@@ -3351,7 +3347,7 @@ llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {
 
   // Pointers are passed directly, everything else is passed by address.
   if (!V->getType()->isPointerTy()) {
-    Address Ptr = CreateDefaultAlignTempAlloca(V->getType());
+    RawAddress Ptr = CreateDefaultAlignTempAlloca(V->getType());
     Builder.CreateStore(V, Ptr);
     V = Ptr.getPointer();
   }
@@ -3907,6 +3903,21 @@ static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF,
   }
 }
 
+static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
+                                     ArrayRef<llvm::Value *> indices,
+                                     llvm::Type *elementType, bool inbounds,
+                                     bool signedIndices, SourceLocation loc,
+                                     CharUnits align,
+                                     const llvm::Twine &name = "arrayidx") {
+  if (inbounds) {
+    return CGF.EmitCheckedInBoundsGEP(addr, indices, elementType, signedIndices,
+                                      CodeGenFunction::NotSubtraction, loc,
+                                      align, name);
+  } else {
+    return CGF.Builder.CreateGEP(addr, indices, elementType, align, name);
+  }
+}
+
 static CharUnits getArrayElementAlign(CharUnits arrayAlign,
                                       llvm::Value *idx,
                                       CharUnits eltSize) {
@@ -3954,7 +3965,7 @@ static Address wrapWithBPFPreserveStaticOffset(CodeGenFunction &CGF,
 
   llvm::Function *Fn =
       CGF.CGM.getIntrinsic(llvm::Intrinsic::preserve_static_offset);
-  llvm::CallInst *Call = CGF.Builder.CreateCall(Fn, {Addr.getPointer()});
+  llvm::CallInst *Call = CGF.Builder.CreateCall(Fn, {Addr.emitRawPointer(CGF)});
   return Address(Call, Addr.getElementType(), Addr.getAlignment());
 }
 
@@ -4017,7 +4028,7 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
   // We can use that to compute the best alignment of the element.
   CharUnits eltSize = CGF.getContext().getTypeSizeInChars(eltType);
   CharUnits eltAlign =
-    getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize);
+      getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize);
 
   if (hasBPFPreserveStaticOffset(Base))
     addr = wrapWithBPFPreserveStaticOffset(CGF, addr);
@@ -4026,19 +4037,19 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
   auto LastIndex = dyn_cast<llvm::ConstantInt>(indices.back());
   if (!LastIndex ||
       (!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, Base))) {
-    eltPtr = emitArraySubscriptGEP(
-        CGF, addr.getElementType(), addr.getPointer(), indices, inbounds,
-        signedIndices, loc, name);
+    addr = emitArraySubscriptGEP(CGF, addr, indices,
+                                 CGF.ConvertTypeForMem(eltType), inbounds,
+                                 signedIndices, loc, eltAlign, name);
+    return addr;
   } else {
     // Remember the original array subscript for bpf target
     unsigned idx = LastIndex->getZExtValue();
     llvm::DIType *DbgInfo = nullptr;
     if (arrayType)
       DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(*arrayType, loc);
-    eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(addr.getElementType(),
-                                                        addr.getPointer(),
-                                                        indices.size() - 1,
-                                                        idx, DbgInfo);
+    eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(
+        addr.getElementType(), addr.emitRawPointer(CGF), indices.size() - 1,
+        idx, DbgInfo);
   }
 
   return Address(eltPtr, CGF.ConvertTypeForMem(eltType), eltAlign);
@@ -4207,8 +4218,8 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
     CharUnits EltAlign =
       getArrayElementAlign(Addr.getAlignment(), Idx, InterfaceSize);
     llvm::Value *EltPtr =
-        emitArraySubscriptGEP(*this, Int8Ty, Addr.getPointer(), ScaledIdx,
-                              false, SignedIndices, E->getExprLoc());
+        emitArraySubscriptGEP(*this, Int8Ty, Addr.emitRawPointer(*this),
+                              ScaledIdx, false, SignedIndices, E->getExprLoc());
     Addr = Address(EltPtr, OrigBaseElemTy, EltAlign);
   } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
     // If this is A[i] where A is an array, the frontend will have decayed the
@@ -4254,7 +4265,7 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
 
             llvm::Type *CountTy = ConvertType(CountFD->getType());
             llvm::Value *Res = Builder.CreateInBoundsGEP(
-                Int8Ty, Addr.getPointer(),
+                Int8Ty, Addr.emitRawPointer(*this),
                 Builder.getInt32(OffsetDiff.getQuantity()), ".counted_by.gep");
             Res = Builder.CreateAlignedLoad(CountTy, Res, getIntAlign(),
                                             ".counted_by.load");
@@ -4500,9 +4511,9 @@ LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E,
     BaseInfo = ArrayLV.getBaseInfo();
     TBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, ResultExprTy);
   } else {
-    Address Base = emitOMPArraySectionBase(*this, E->getBase(), BaseInfo,
-                                           TBAAInfo, BaseTy, ResultExprTy,
-                                           IsLowerBound);
+    Address Base =
+        emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, TBAAInfo, BaseTy,
+                                ResultExprTy, IsLowerBound);
     EltPtr = emitArraySubscriptGEP(*this, Base, Idx, ResultExprTy,
                                    !getLangOpts().isSignedOverflowDefined(),
                                    /*signedIndices=*/false, E->getExprLoc());
@@ -4589,7 +4600,7 @@ LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
       SkippedChecks.set(SanitizerKind::Alignment, true);
     if (IsBaseCXXThis || isa<DeclRefExpr>(BaseExpr))
       SkippedChecks.set(SanitizerKind::Null, true);
-    EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr.getPointer(), PtrTy,
+    EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr, PtrTy,
                   /*Alignment=*/CharUnits::Zero(), SkippedChecks);
     BaseLV = MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo);
   } else
@@ -4638,8 +4649,8 @@ LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field,
       LambdaLV = EmitLoadOfReferenceLValue(AddrOfExplicitObject, D->getType(),
                                            AlignmentSource::Decl);
     else
-      LambdaLV = MakeNaturalAlignAddrLValue(AddrOfExplicitObject.getPointer(),
-                                            D->getType().getNonReferenceType());
+      LambdaLV = MakeAddrLValue(AddrOfExplicitObject,
+                                D->getType().getNonReferenceType());
   } else {
     QualType LambdaTagType = getContext().getTagDeclType(Field->getParent());
     LambdaLV = MakeNaturalAlignAddrLValue(ThisValue, LambdaTagType);
@@ -4829,7 +4840,8 @@ LValue CodeGenFunction::EmitLValueForField(LValue base,
       // information provided by invariant.group.  This is because accessing
       // fields may leak the real address of dynamic object, which could result
       // in miscompilation when leaked pointer would be compared.
-      auto *stripped = Builder.CreateStripInvariantGroup(addr.getPointer());
+      auto *stripped =
+          Builder.CreateStripInvariantGroup(addr.emitRawPointer(*this));
       addr = Address(stripped, addr.getElementType(), addr.getAlignment());
     }
   }
@@ -4848,10 +4860,11 @@ LValue CodeGenFunction::EmitLValueForField(LValue base,
       // Remember the original union field index
       llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(base.getType(),
           rec->getLocation());
-      addr = Address(
-          Builder.CreatePreserveUnionAccessIndex(
-              addr.getPointer(), getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo),
-          addr.getElementType(), addr.getAlignment());
+      addr =
+          Address(Builder.CreatePreserveUnionAccessIndex(
+                      addr.emitRawPointer(*this),
+                      getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo),
+                  addr.getElementType(), addr.getAlignment());
     }
 
     if (FieldType->isReferenceType())
@@ -5088,11 +5101,9 @@ LValue CodeGenFunction::EmitConditionalOperatorLValue(
   if (Info.LHS && Info.RHS) {
     Address lhsAddr = Info.LHS->getAddress(*this);
     Address rhsAddr = Info.RHS->getAddress(*this);
-    llvm::PHINode *phi = Builder.CreatePHI(lhsAddr.getType(), 2, "cond-lvalue");
-    phi->addIncoming(lhsAddr.getPointer(), Info.lhsBlock);
-    phi->addIncoming(rhsAddr.getPointer(), Info.rhsBlock);
-    Address result(phi, lhsAddr.getElementType(),
-                   std::min(lhsAddr.getAlignment(), rhsAddr.getAlignment()));
+    Address result = mergeAddressesInConditionalExpr(
+        lhsAddr, rhsAddr, Info.lhsBlock, Info.rhsBlock,
+        Builder.GetInsertBlock(), expr->getType());
     AlignmentSource alignSource =
         std::max(Info.LHS->getBaseInfo().getAlignmentSource(),
                  Info.RHS->getBaseInfo().getAlignmentSource());
@@ -5179,7 +5190,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
     LValue LV = EmitLValue(E->getSubExpr());
     Address V = LV.getAddress(*this);
     const auto *DCE = cast<CXXDynamicCastExpr>(E);
-    return MakeNaturalAlignAddrLValue(EmitDynamicCast(V, DCE), E->getType());
+    return MakeNaturalAlignRawAddrLValue(EmitDynamicCast(V, DCE), E->getType());
   }
 
   case CK_ConstructorConversion:
@@ -5244,8 +5255,8 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
     // C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is
     // performed and the object is not of the derived type.
     if (sanitizePerformTypeCheck())
-      EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(),
-                    Derived.getPointer(), E->getType());
+      EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(), Derived,
+                    E->getType());
 
     if (SanOpts.has(SanitizerKind::CFIDerivedCast))
       EmitVTablePtrCheckForCast(E->getType(), Derived,
@@ -5601,7 +5612,7 @@ LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
 
 LValue
 CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
-  return MakeNaturalAlignAddrLValue(EmitCXXTypeidExpr(E), E->getType());
+  return MakeNaturalAlignRawAddrLValue(EmitCXXTypeidExpr(E), E->getType());
 }
 
 Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) {
diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp
index 5190b22bcc16..143855aa84ca 100644
--- a/clang/lib/CodeGen/CGExprAgg.cpp
+++ b/clang/lib/CodeGen/CGExprAgg.cpp
@@ -294,10 +294,10 @@ void AggExprEmitter::withReturnValueSlot(
   // Otherwise, EmitCall will emit its own, notice that it's "unused", and end
   // its lifetime before we have the chance to emit a proper destructor call.
   bool UseTemp = Dest.isPotentiallyAliased() || Dest.requiresGCollection() ||
-                 (RequiresDestruction && !Dest.getAddress().isValid());
+                 (RequiresDestruction && Dest.isIgnored());
 
   Address RetAddr = Address::invalid();
-  Address RetAllocaAddr = Address::invalid();
+  RawAddress RetAllocaAddr = RawAddress::invalid();
 
   EHScopeStack::stable_iterator LifetimeEndBlock;
   llvm::Value *LifetimeSizePtr = nullptr;
@@ -329,7 +329,8 @@ void AggExprEmitter::withReturnValueSlot(
   if (!UseTemp)
     return;
 
-  assert(Dest.isIgnored() || Dest.getPointer() != Src.getAggregatePointer());
+  assert(Dest.isIgnored() || Dest.emitRawPointer(CGF) !=
+                                 Src.getAggregatePointer(E->getType(), CGF));
   EmitFinalDestCopy(E->getType(), Src);
 
   if (!RequiresDestruction && LifetimeStartInst) {
@@ -448,7 +449,8 @@ AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) {
   llvm::Value *Zero = llvm::ConstantInt::get(CGF.PtrDiffTy, 0);
   llvm::Value *IdxStart[] = { Zero, Zero };
   llvm::Value *ArrayStart = Builder.CreateInBoundsGEP(
-      ArrayPtr.getElementType(), ArrayPtr.getPointer(), IdxStart, "arraystart");
+      ArrayPtr.getElementType(), ArrayPtr.emitRawPointer(CGF), IdxStart,
+      "arraystart");
   CGF.EmitStoreThroughLValue(RValue::get(ArrayStart), Start);
   ++Field;
 
@@ -465,7 +467,8 @@ AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) {
     // End pointer.
     llvm::Value *IdxEnd[] = { Zero, Size };
     llvm::Value *ArrayEnd = Builder.CreateInBoundsGEP(
-        ArrayPtr.getElementType(), ArrayPtr.getPointer(), IdxEnd, "arrayend");
+        ArrayPtr.getElementType(), ArrayPtr.emitRawPointer(CGF), IdxEnd,
+        "arrayend");
     CGF.EmitStoreThroughLValue(RValue::get(ArrayEnd), EndOrLength);
   } else if (Ctx.hasSameType(Field->getType(), Ctx.getSizeType())) {
     // Length.
@@ -516,9 +519,9 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
   // down a level.
   llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0);
   llvm::Value *indices[] = { zero, zero };
-  llvm::Value *begin = Builder.CreateInBoundsGEP(
-      DestPtr.getElementType(), DestPtr.getPointer(), indices,
-      "arrayinit.begin");
+  llvm::Value *begin = Builder.CreateInBoundsGEP(DestPtr.getElementType(),
+                                                 DestPtr.emitRawPointer(CGF),
+                                                 indices, "arrayinit.begin");
 
   CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType);
   CharUnits elementAlign =
@@ -1059,7 +1062,7 @@ void AggExprEmitter::VisitBinCmp(const BinaryOperator *E) {
     if (RV.isScalar())
       return {RV.getScalarVal(), nullptr};
     if (RV.isAggregate())
-      return {RV.getAggregatePointer(), nullptr};
+      return {RV.getAggregatePointer(E->getType(), CGF), nullptr};
     assert(RV.isComplex());
     return RV.getComplexVal();
   };
@@ -1818,7 +1821,7 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr(
     // else, clean it up for -O0 builds and general tidiness.
     if (!pushedCleanup && LV.isSimple())
       if (llvm::GetElementPtrInst *GEP =
-              dyn_cast<llvm::GetElementPtrInst>(LV.getPointer(CGF)))
+              dyn_cast<llvm::GetElementPtrInst>(LV.emitRawPointer(CGF)))
         if (GEP->use_empty())
           GEP->eraseFromParent();
   }
@@ -1849,9 +1852,9 @@ void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
   // destPtr is an array*. Construct an elementType* by drilling down a level.
   llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0);
   llvm::Value *indices[] = {zero, zero};
-  llvm::Value *begin = Builder.CreateInBoundsGEP(
-      destPtr.getElementType(), destPtr.getPointer(), indices,
-      "arrayinit.begin");
+  llvm::Value *begin = Builder.CreateInBoundsGEP(destPtr.getElementType(),
+                                                 destPtr.emitRawPointer(CGF),
+                                                 indices, "arrayinit.begin");
 
   // Prepare to special-case multidimensional array initialization: we avoid
   // emitting multiple destructor loops in that case.
diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp
index 2adbef6d5512..1d2ca37a449b 100644
--- a/clang/lib/CodeGen/CGExprCXX.cpp
+++ b/clang/lib/CodeGen/CGExprCXX.cpp
@@ -280,7 +280,8 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr(
     LValueBaseInfo BaseInfo;
     TBAAAccessInfo TBAAInfo;
     Address ThisValue = EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo);
-    This = MakeAddrLValue(ThisValue, Base->getType(), BaseInfo, TBAAInfo);
+    This = MakeAddrLValue(ThisValue, Base->getType()->getPointeeType(),
+                          BaseInfo, TBAAInfo);
   } else {
     This = EmitLValue(Base);
   }
@@ -353,10 +354,8 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr(
     if (IsImplicitObjectCXXThis || isa<DeclRefExpr>(IOA))
       SkippedChecks.set(SanitizerKind::Null, true);
   }
-  EmitTypeCheck(CodeGenFunction::TCK_MemberCall, CallLoc,
-                This.getPointer(*this),
-                C.getRecordType(CalleeDecl->getParent()),
-                /*Alignment=*/CharUnits::Zero(), SkippedChecks);
+  EmitTypeCheck(CodeGenFunction::TCK_MemberCall, CallLoc, This,
+                C.getRecordType(CalleeDecl->getParent()), SkippedChecks);
 
   // C++ [class.virtual]p12:
   //   Explicit qualification with the scope operator (5.1) suppresses the
@@ -455,7 +454,7 @@ CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
   else
     This = EmitLValue(BaseExpr, KnownNonNull).getAddress(*this);
 
-  EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This.getPointer(),
+  EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This.emitRawPointer(*this),
                 QualType(MPT->getClass(), 0));
 
   // Get the member function pointer.
@@ -1110,9 +1109,10 @@ void CodeGenFunction::EmitNewArrayInitializer(
       // alloca.
       EndOfInit = CreateTempAlloca(BeginPtr.getType(), getPointerAlign(),
                                    "array.init.end");
-      CleanupDominator = Builder.CreateStore(BeginPtr.getPointer(), EndOfInit);
-      pushIrregularPartialArrayCleanup(BeginPtr.getPointer(), EndOfInit,
-                                       ElementType, ElementAlign,
+      CleanupDominator =
+          Builder.CreateStore(BeginPtr.emitRawPointer(*this), EndOfInit);
+      pushIrregularPartialArrayCleanup(BeginPtr.emitRawPointer(*this),
+                                       EndOfInit, ElementType, ElementAlign,
                                        getDestroyer(DtorKind));
       Cleanup = EHStack.stable_begin();
     }
@@ -1124,16 +1124,17 @@ void CodeGenFunction::EmitNewArrayInitializer(
       // element.  TODO: some of these stores can be trivially
       // observed to be unnecessary.
       if (EndOfInit.isValid()) {
-        Builder.CreateStore(CurPtr.getPointer(), EndOfInit);
+        Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit);
       }
       // FIXME: If the last initializer is an incomplete initializer list for
       // an array, and we have an array filler, we can fold together the two
       // initialization loops.
       StoreAnyExprIntoOneUnit(*this, IE, IE->getType(), CurPtr,
                               AggValueSlot::DoesNotOverlap);
-      CurPtr = Address(Builder.CreateInBoundsGEP(
-                           CurPtr.getElementType(), CurPtr.getPointer(),
-                           Builder.getSize(1), "array.exp.next"),
+      CurPtr = Address(Builder.CreateInBoundsGEP(CurPtr.getElementType(),
+                                                 CurPtr.emitRawPointer(*this),
+                                                 Builder.getSize(1),
+                                                 "array.exp.next"),
                        CurPtr.getElementType(),
                        StartAlign.alignmentAtOffset((++i) * ElementSize));
     }
@@ -1187,7 +1188,7 @@ void CodeGenFunction::EmitNewArrayInitializer(
     // FIXME: Share this cleanup with the constructor call emission rather than
     // having it create a cleanup of its own.
     if (EndOfInit.isValid())
-      Builder.CreateStore(CurPtr.getPointer(), EndOfInit);
+      Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit);
 
     // Emit a constructor call loop to initialize the remaining elements.
     if (InitListElements)
@@ -1250,15 +1251,15 @@ void CodeGenFunction::EmitNewArrayInitializer(
   llvm::BasicBlock *ContBB = createBasicBlock("new.loop.end");
 
   // Find the end of the array, hoisted out of the loop.
-  llvm::Value *EndPtr =
-    Builder.CreateInBoundsGEP(BeginPtr.getElementType(), BeginPtr.getPointer(),
-                              NumElements, "array.end");
+  llvm::Value *EndPtr = Builder.CreateInBoundsGEP(
+      BeginPtr.getElementType(), BeginPtr.emitRawPointer(*this), NumElements,
+      "array.end");
 
   // If the number of elements isn't constant, we have to now check if there is
   // anything left to initialize.
   if (!ConstNum) {
-    llvm::Value *IsEmpty =
-      Builder.CreateICmpEQ(CurPtr.getPointer(), EndPtr, "array.isempty");
+    llvm::Value *IsEmpty = Builder.CreateICmpEQ(CurPtr.emitRawPointer(*this),
+                                                EndPtr, "array.isempty");
     Builder.CreateCondBr(IsEmpty, ContBB, LoopBB);
   }
 
@@ -1268,19 +1269,20 @@ void CodeGenFunction::EmitNewArrayInitializer(
   // Set up the current-element phi.
   llvm::PHINode *CurPtrPhi =
       Builder.CreatePHI(CurPtr.getType(), 2, "array.cur");
-  CurPtrPhi->addIncoming(CurPtr.getPointer(), EntryBB);
+  CurPtrPhi->addIncoming(CurPtr.emitRawPointer(*this), EntryBB);
 
   CurPtr = Address(CurPtrPhi, CurPtr.getElementType(), ElementAlign);
 
   // Store the new Cleanup position for irregular Cleanups.
   if (EndOfInit.isValid())
-    Builder.CreateStore(CurPtr.getPointer(), EndOfInit);
+    Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit);
 
   // Enter a partial-destruction Cleanup if necessary.
   if (!CleanupDominator && needsEHCleanup(DtorKind)) {
-    pushRegularPartialArrayCleanup(BeginPtr.getPointer(), CurPtr.getPointer(),
-                                   ElementType, ElementAlign,
-                                   getDestroyer(DtorKind));
+    llvm::Value *BeginPtrRaw = BeginPtr.emitRawPointer(*this);
+    llvm::Value *CurPtrRaw = CurPtr.emitRawPointer(*this);
+    pushRegularPartialArrayCleanup(BeginPtrRaw, CurPtrRaw, ElementType,
+                                   ElementAlign, getDestroyer(DtorKind));
     Cleanup = EHStack.stable_begin();
     CleanupDominator = Builder.CreateUnreachable();
   }
@@ -1296,9 +1298,8 @@ void CodeGenFunction::EmitNewArrayInitializer(
   }
 
   // Advance to the next element by adjusting the pointer type as necessary.
-  llvm::Value *NextPtr =
-    Builder.CreateConstInBoundsGEP1_32(ElementTy, CurPtr.getPointer(), 1,
-                                       "array.next");
+  llvm::Value *NextPtr = Builder.CreateConstInBoundsGEP1_32(
+      ElementTy, CurPtr.emitRawPointer(*this), 1, "array.next");
 
   // Check whether we've gotten to the end of the array and, if so,
   // exit the loop.
@@ -1524,14 +1525,9 @@ static void EnterNewDeleteCleanup(CodeGenFunction &CGF,
 
     typedef CallDeleteDuringNew<DirectCleanupTraits> DirectCleanup;
 
-    DirectCleanup *Cleanup = CGF.EHStack
-      .pushCleanupWithExtra<DirectCleanup>(EHCleanup,
-                                           E->getNumPlacementArgs(),
-                                           E->getOperatorDelete(),
-                                           NewPtr.getPointer(),
-                                           AllocSize,
-                                           E->passAlignment(),
-                                           AllocAlign);
+    DirectCleanup *Cleanup = CGF.EHStack.pushCleanupWithExtra<DirectCleanup>(
+        EHCleanup, E->getNumPlacementArgs(), E->getOperatorDelete(),
+        NewPtr.emitRawPointer(CGF), AllocSize, E->passAlignment(), AllocAlign);
     for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I) {
       auto &Arg = NewArgs[I + NumNonPlacementArgs];
       Cleanup->setPlacementArg(I, Arg.getRValue(CGF), Arg.Ty);
@@ -1542,7 +1538,7 @@ static void EnterNewDeleteCleanup(CodeGenFunction &CGF,
 
   // Otherwise, we need to save all this stuff.
   DominatingValue<RValue>::saved_type SavedNewPtr =
-    DominatingValue<RValue>::save(CGF, RValue::get(NewPtr.getPointer()));
+      DominatingValue<RValue>::save(CGF, RValue::get(NewPtr, CGF));
   DominatingValue<RValue>::saved_type SavedAllocSize =
     DominatingValue<RValue>::save(CGF, RValue::get(AllocSize));
 
@@ -1620,14 +1616,14 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
     // In these cases, discard the computed alignment and use the
     // formal alignment of the allocated type.
     if (BaseInfo.getAlignmentSource() != AlignmentSource::Decl)
-      allocation = allocation.withAlignment(allocAlign);
+      allocation.setAlignment(allocAlign);
 
     // Set up allocatorArgs for the call to operator delete if it's not
     // the reserved global operator.
     if (E->getOperatorDelete() &&
         !E->getOperatorDelete()->isReservedGlobalPlacementOperator()) {
       allocatorArgs.add(RValue::get(allocSize), getContext().getSizeType());
-      allocatorArgs.add(RValue::get(allocation.getPointer()), arg->getType());
+      allocatorArgs.add(RValue::get(allocation, *this), arg->getType());
     }
 
   } else {
@@ -1715,8 +1711,7 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
     llvm::BasicBlock *notNullBB = createBasicBlock("new.notnull");
     contBB = createBasicBlock("new.cont");
 
-    llvm::Value *isNull =
-      Builder.CreateIsNull(allocation.getPointer(), "new.isnull");
+    llvm::Value *isNull = Builder.CreateIsNull(allocation, "new.isnull");
     Builder.CreateCondBr(isNull, contBB, notNullBB);
     EmitBlock(notNullBB);
   }
@@ -1762,12 +1757,12 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
   SkippedChecks.set(SanitizerKind::Null, nullCheck);
   EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall,
                 E->getAllocatedTypeSourceInfo()->getTypeLoc().getBeginLoc(),
-                result.getPointer(), allocType, result.getAlignment(),
-                SkippedChecks, numElements);
+                result, allocType, result.getAlignment(), SkippedChecks,
+                numElements);
 
   EmitNewInitializer(*this, E, allocType, elementTy, result, numElements,
                      allocSizeWithoutCookie);
-  llvm::Value *resultPtr = result.getPointer();
+  llvm::Value *resultPtr = result.emitRawPointer(*this);
   if (E->isArray()) {
     // NewPtr is a pointer to the base element type.  If we're
     // allocating an array of arrays, we'll need to cast back to the
@@ -1911,7 +1906,8 @@ static void EmitDestroyingObjectDelete(CodeGenFunction &CGF,
     CGF.CGM.getCXXABI().emitVirtualObjectDelete(CGF, DE, Ptr, ElementType,
                                                 Dtor);
   else
-    CGF.EmitDeleteCall(DE->getOperatorDelete(), Ptr.getPointer(), ElementType);
+    CGF.EmitDeleteCall(DE->getOperatorDelete(), Ptr.emitRawPointer(CGF),
+                       ElementType);
 }
 
 /// Emit the code for deleting a single object.
@@ -1927,8 +1923,7 @@ static bool EmitObjectDelete(CodeGenFunction &CGF,
   //   dynamic type, the static type shall be a base class of the dynamic type
   //   of the object to be deleted and the static type shall have a virtual
   //   destructor or the behavior is undefined.
-  CGF.EmitTypeCheck(CodeGenFunction::TCK_MemberCall,
-                    DE->getExprLoc(), Ptr.getPointer(),
+  CGF.EmitTypeCheck(CodeGenFunction::TCK_MemberCall, DE->getExprLoc(), Ptr,
                     ElementType);
 
   const FunctionDecl *OperatorDelete = DE->getOperatorDelete();
@@ -1977,9 +1972,8 @@ static bool EmitObjectDelete(CodeGenFunction &CGF,
   // Make sure that we call delete even if the dtor throws.
   // This doesn't have to a conditional cleanup because we're going
   // to pop it off in a second.
-  CGF.EHStack.pushCleanup<CallObjectDelete>(NormalAndEHCleanup,
-                                            Ptr.getPointer(),
-                                            OperatorDelete, ElementType);
+  CGF.EHStack.pushCleanup<CallObjectDelete>(
+      NormalAndEHCleanup, Ptr.emitRawPointer(CGF), OperatorDelete, ElementType);
 
   if (Dtor)
     CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
@@ -2066,7 +2060,7 @@ static void EmitArrayDelete(CodeGenFunction &CGF,
     CharUnits elementAlign =
       deletedPtr.getAlignment().alignmentOfArrayElement(elementSize);
 
-    llvm::Value *arrayBegin = deletedPtr.getPointer();
+    llvm::Value *arrayBegin = deletedPtr.emitRawPointer(CGF);
     llvm::Value *arrayEnd = CGF.Builder.CreateInBoundsGEP(
       deletedPtr.getElementType(), arrayBegin, numElements, "delete.end");
 
@@ -2097,7 +2091,7 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) {
   llvm::BasicBlock *DeleteNotNull = createBasicBlock("delete.notnull");
   llvm::BasicBlock *DeleteEnd = createBasicBlock("delete.end");
 
-  llvm::Value *IsNull = Builder.CreateIsNull(Ptr.getPointer(), "isnull");
+  llvm::Value *IsNull = Builder.CreateIsNull(Ptr, "isnull");
 
   Builder.CreateCondBr(IsNull, DeleteEnd, DeleteNotNull);
   EmitBlock(DeleteNotNull);
@@ -2132,10 +2126,8 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) {
       GEP.push_back(Zero);
     }
 
-    Ptr = Address(Builder.CreateInBoundsGEP(Ptr.getElementType(),
-                                            Ptr.getPointer(), GEP, "del.first"),
-                  ConvertTypeForMem(DeleteTy), Ptr.getAlignment(),
-                  Ptr.isKnownNonNull());
+    Ptr = Builder.CreateInBoundsGEP(Ptr, GEP, ConvertTypeForMem(DeleteTy),
+                                    Ptr.getAlignment(), "del.first");
   }
 
   assert(ConvertTypeForMem(DeleteTy) == Ptr.getElementType());
@@ -2193,7 +2185,7 @@ static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E,
   //   destruction and the static type of the operand is neither the constructor
   //   or destructor’s class nor one of its bases, the behavior is undefined.
   CGF.EmitTypeCheck(CodeGenFunction::TCK_DynamicOperation, E->getExprLoc(),
-                    ThisPtr.getPointer(), SrcRecordTy);
+                    ThisPtr, SrcRecordTy);
 
   // C++ [expr.typeid]p2:
   //   If the glvalue expression is obtained by applying the unary * operator to
@@ -2209,7 +2201,7 @@ static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E,
         CGF.createBasicBlock("typeid.bad_typeid");
     llvm::BasicBlock *EndBlock = CGF.createBasicBlock("typeid.end");
 
-    llvm::Value *IsNull = CGF.Builder.CreateIsNull(ThisPtr.getPointer());
+    llvm::Value *IsNull = CGF.Builder.CreateIsNull(ThisPtr);
     CGF.Builder.CreateCondBr(IsNull, BadTypeidBlock, EndBlock);
 
     CGF.EmitBlock(BadTypeidBlock);
@@ -2295,8 +2287,7 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr,
   //   construction or destruction and the static type of the operand is not a
   //   pointer to or object of the constructor or destructor’s own class or one
   //   of its bases, the dynamic_cast results in undefined behavior.
-  EmitTypeCheck(TCK_DynamicOperation, DCE->getExprLoc(), ThisAddr.getPointer(),
-                SrcRecordTy);
+  EmitTypeCheck(TCK_DynamicOperation, DCE->getExprLoc(), ThisAddr, SrcRecordTy);
 
   if (DCE->isAlwaysNull()) {
     if (llvm::Value *T = EmitDynamicCastToNull(*this, DestTy)) {
@@ -2331,7 +2322,7 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr,
     CastNull = createBasicBlock("dynamic_cast.null");
     CastNotNull = createBasicBlock("dynamic_cast.notnull");
 
-    llvm::Value *IsNull = Builder.CreateIsNull(ThisAddr.getPointer());
+    llvm::Value *IsNull = Builder.CreateIsNull(ThisAddr);
     Builder.CreateCondBr(IsNull, CastNull, CastNotNull);
     EmitBlock(CastNotNull);
   }
diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index 75286dceb13a..dbefd6d02e7b 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -800,8 +800,8 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD,
     // Add a vtable pointer, if we need one and it hasn't already been added.
     if (Layout.hasOwnVFPtr()) {
       llvm::Constant *VTableAddressPoint =
-          CGM.getCXXABI().getVTableAddressPointForConstExpr(
-              BaseSubobject(CD, Offset), VTableClass);
+          CGM.getCXXABI().getVTableAddressPoint(BaseSubobject(CD, Offset),
+                                                VTableClass);
       if (!AppendBytes(Offset, VTableAddressPoint))
         return false;
     }
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 8536570087ad..83247aa48f86 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -2250,7 +2250,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
     // performed and the object is not of the derived type.
     if (CGF.sanitizePerformTypeCheck())
       CGF.EmitTypeCheck(CodeGenFunction::TCK_DowncastPointer, CE->getExprLoc(),
-                        Derived.getPointer(), DestTy->getPointeeType());
+                        Derived, DestTy->getPointeeType());
 
     if (CGF.SanOpts.has(SanitizerKind::CFIDerivedCast))
       CGF.EmitVTablePtrCheckForCast(DestTy->getPointeeType(), Derived,
@@ -2258,13 +2258,14 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
                                     CodeGenFunction::CFITCK_DerivedCast,
                                     CE->getBeginLoc());
 
-    return Derived.getPointer();
+    return CGF.getAsNaturalPointerTo(Derived, CE->getType()->getPointeeType());
   }
   case CK_UncheckedDerivedToBase:
   case CK_DerivedToBase: {
     // The EmitPointerWithAlignment path does this fine; just discard
     // the alignment.
-    return CGF.EmitPointerWithAlignment(CE).getPointer();
+    return CGF.getAsNaturalPointerTo(CGF.EmitPointerWithAlignment(CE),
+                                     CE->getType()->getPointeeType());
   }
 
   case CK_Dynamic: {
@@ -2274,7 +2275,8 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
   }
 
   case CK_ArrayToPointerDecay:
-    return CGF.EmitArrayToPointerDecay(E).getPointer();
+    return CGF.getAsNaturalPointerTo(CGF.EmitArrayToPointerDecay(E),
+                                     CE->getType()->getPointeeType());
   case CK_FunctionToPointerDecay:
     return EmitLValue(E).getPointer(CGF);
 
@@ -5588,3 +5590,16 @@ CodeGenFunction::EmitCheckedInBoundsGEP(llvm::Type *ElemTy, Value *Ptr,
 
   return GEPVal;
 }
+
+Address CodeGenFunction::EmitCheckedInBoundsGEP(
+    Address Addr, ArrayRef<Value *> IdxList, llvm::Type *elementType,
+    bool SignedIndices, bool IsSubtraction, SourceLocation Loc, CharUnits Align,
+    const Twine &Name) {
+  if (!SanOpts.has(SanitizerKind::PointerOverflow))
+    return Builder.CreateInBoundsGEP(Addr, IdxList, elementType, Align, Name);
+
+  return RawAddress(
+      EmitCheckedInBoundsGEP(Addr.getElementType(), Addr.emitRawPointer(*this),
+                             IdxList, SignedIndices, IsSubtraction, Loc, Name),
+      elementType, Align);
+}
diff --git a/clang/lib/CodeGen/CGNonTrivialStruct.cpp b/clang/lib/CodeGen/CGNonTrivialStruct.cpp
index 75c1d7fbea84..8fade0fac21e 100644
--- a/clang/lib/CodeGen/CGNonTrivialStruct.cpp
+++ b/clang/lib/CodeGen/CGNonTrivialStruct.cpp
@@ -366,7 +366,7 @@ template <class Derived> struct GenFuncBase {
     llvm::Value *SizeInBytes =
         CGF.Builder.CreateNUWMul(BaseEltSizeVal, NumElts);
     llvm::Value *DstArrayEnd = CGF.Builder.CreateInBoundsGEP(
-        CGF.Int8Ty, DstAddr.getPointer(), SizeInBytes);
+        CGF.Int8Ty, DstAddr.emitRawPointer(CGF), SizeInBytes);
     llvm::BasicBlock *PreheaderBB = CGF.Builder.GetInsertBlock();
 
     // Create the header block and insert the phi instructions.
@@ -376,7 +376,7 @@ template <class Derived> struct GenFuncBase {
 
     for (unsigned I = 0; I < N; ++I) {
       PHIs[I] = CGF.Builder.CreatePHI(CGF.CGM.Int8PtrPtrTy, 2, "addr.cur");
-      PHIs[I]->addIncoming(StartAddrs[I].getPointer(), PreheaderBB);
+      PHIs[I]->addIncoming(StartAddrs[I].emitRawPointer(CGF), PreheaderBB);
     }
 
     // Create the exit and loop body blocks.
@@ -410,7 +410,7 @@ template <class Derived> struct GenFuncBase {
       // Instrs to update the destination and source addresses.
       // Update phi instructions.
       NewAddrs[I] = getAddrWithOffset(NewAddrs[I], EltSize);
-      PHIs[I]->addIncoming(NewAddrs[I].getPointer(), LoopBB);
+      PHIs[I]->addIncoming(NewAddrs[I].emitRawPointer(CGF), LoopBB);
     }
 
     // Insert an unconditional branch to the header block.
@@ -488,7 +488,7 @@ template <class Derived> struct GenFuncBase {
 
     for (unsigned I = 0; I < N; ++I) {
       Alignments[I] = Addrs[I].getAlignment();
-      Ptrs[I] = Addrs[I].getPointer();
+      Ptrs[I] = Addrs[I].emitRawPointer(CallerCGF);
     }
 
     if (llvm::Function *F =
diff --git a/clang/lib/CodeGen/CGObjC.cpp b/clang/lib/CodeGen/CGObjC.cpp
index f3a948cf13f9..c7f497a7c845 100644
--- a/clang/lib/CodeGen/CGObjC.cpp
+++ b/clang/lib/CodeGen/CGObjC.cpp
@@ -94,8 +94,8 @@ CodeGenFunction::EmitObjCBoxedExpr(const ObjCBoxedExpr *E) {
     // and cast value to correct type
     Address Temporary = CreateMemTemp(SubExpr->getType());
     EmitAnyExprToMem(SubExpr, Temporary, Qualifiers(), /*isInit*/ true);
-    llvm::Value *BitCast =
-        Builder.CreateBitCast(Temporary.getPointer(), ConvertType(ArgQT));
+    llvm::Value *BitCast = Builder.CreateBitCast(
+        Temporary.emitRawPointer(*this), ConvertType(ArgQT));
     Args.add(RValue::get(BitCast), ArgQT);
 
     // Create char array to store type encoding
@@ -204,11 +204,11 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E,
   ObjCMethodDecl::param_const_iterator PI = MethodWithObjects->param_begin();
   const ParmVarDecl *argDecl = *PI++;
   QualType ArgQT = argDecl->getType().getUnqualifiedType();
-  Args.add(RValue::get(Objects.getPointer()), ArgQT);
+  Args.add(RValue::get(Objects, *this), ArgQT);
   if (DLE) {
     argDecl = *PI++;
     ArgQT = argDecl->getType().getUnqualifiedType();
-    Args.add(RValue::get(Keys.getPointer()), ArgQT);
+    Args.add(RValue::get(Keys, *this), ArgQT);
   }
   argDecl = *PI;
   ArgQT = argDecl->getType().getUnqualifiedType();
@@ -827,7 +827,7 @@ static void emitStructGetterCall(CodeGenFunction &CGF, ObjCIvarDecl *ivar,
   //                  sizeof (Type of Ivar), isAtomic, false);
   CallArgList args;
 
-  llvm::Value *dest = CGF.ReturnValue.getPointer();
+  llvm::Value *dest = CGF.ReturnValue.emitRawPointer(CGF);
   args.add(RValue::get(dest), Context.VoidPtrTy);
   args.add(RValue::get(src), Context.VoidPtrTy);
 
@@ -1147,8 +1147,8 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
       callCStructCopyConstructor(Dst, Src);
     } else {
       ObjCIvarDecl *ivar = propImpl->getPropertyIvarDecl();
-      emitCPPObjectAtomicGetterCall(*this, ReturnValue.getPointer(), ivar,
-                                    AtomicHelperFn);
+      emitCPPObjectAtomicGetterCall(*this, ReturnValue.emitRawPointer(*this),
+                                    ivar, AtomicHelperFn);
     }
     return;
   }
@@ -1163,7 +1163,7 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
     }
     else {
       ObjCIvarDecl *ivar = propImpl->getPropertyIvarDecl();
-      emitCPPObjectAtomicGetterCall(*this, ReturnValue.getPointer(),
+      emitCPPObjectAtomicGetterCall(*this, ReturnValue.emitRawPointer(*this),
                                     ivar, AtomicHelperFn);
     }
     return;
@@ -1287,7 +1287,7 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
     case TEK_Scalar: {
       llvm::Value *value;
       if (propType->isReferenceType()) {
-        value = LV.getAddress(*this).getPointer();
+        value = LV.getAddress(*this).emitRawPointer(*this);
       } else {
         // We want to load and autoreleaseReturnValue ARC __weak ivars.
         if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) {
@@ -1821,16 +1821,14 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
   CallArgList Args;
 
   // The first argument is a temporary of the enumeration-state type.
-  Args.add(RValue::get(StatePtr.getPointer()),
-           getContext().getPointerType(StateTy));
+  Args.add(RValue::get(StatePtr, *this), getContext().getPointerType(StateTy));
 
   // The second argument is a temporary array with space for NumItems
   // pointers.  We'll actually be loading elements from the array
   // pointer written into the control state; this buffer is so that
   // collections that *aren't* backed by arrays can still queue up
   // batches of elements.
-  Args.add(RValue::get(ItemsPtr.getPointer()),
-           getContext().getPointerType(ItemsTy));
+  Args.add(RValue::get(ItemsPtr, *this), getContext().getPointerType(ItemsTy));
 
   // The third argument is the capacity of that temporary array.
   llvm::Type *NSUIntegerTy = ConvertType(getContext().getNSUIntegerType());
@@ -2198,7 +2196,7 @@ static llvm::Value *emitARCLoadOperation(CodeGenFunction &CGF, Address addr,
   if (!fn)
     fn = getARCIntrinsic(IntID, CGF.CGM);
 
-  return CGF.EmitNounwindRuntimeCall(fn, addr.getPointer());
+  return CGF.EmitNounwindRuntimeCall(fn, addr.emitRawPointer(CGF));
 }
 
 /// Perform an operation having the following signature:
@@ -2216,9 +2214,8 @@ static llvm::Value *emitARCStoreOperation(CodeGenFunction &CGF, Address addr,
   llvm::Type *origType = value->getType();
 
   llvm::Value *args[] = {
-    CGF.Builder.CreateBitCast(addr.getPointer(), CGF.Int8PtrPtrTy),
-    CGF.Builder.CreateBitCast(value, CGF.Int8PtrTy)
-  };
+      CGF.Builder.CreateBitCast(addr.emitRawPointer(CGF), CGF.Int8PtrPtrTy),
+      CGF.Builder.CreateBitCast(value, CGF.Int8PtrTy)};
   llvm::CallInst *result = CGF.EmitNounwindRuntimeCall(fn, args);
 
   if (ignored) return nullptr;
@@ -2237,9 +2234,8 @@ static void emitARCCopyOperation(CodeGenFunction &CGF, Address dst, Address src,
     fn = getARCIntrinsic(IntID, CGF.CGM);
 
   llvm::Value *args[] = {
-    CGF.Builder.CreateBitCast(dst.getPointer(), CGF.Int8PtrPtrTy),
-    CGF.Builder.CreateBitCast(src.getPointer(), CGF.Int8PtrPtrTy)
-  };
+      CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), CGF.Int8PtrPtrTy),
+      CGF.Builder.CreateBitCast(src.emitRawPointer(CGF), CGF.Int8PtrPtrTy)};
   CGF.EmitNounwindRuntimeCall(fn, args);
 }
 
@@ -2490,9 +2486,8 @@ llvm::Value *CodeGenFunction::EmitARCStoreStrongCall(Address addr,
     fn = getARCIntrinsic(llvm::Intrinsic::objc_storeStrong, CGM);
 
   llvm::Value *args[] = {
-    Builder.CreateBitCast(addr.getPointer(), Int8PtrPtrTy),
-    Builder.CreateBitCast(value, Int8PtrTy)
-  };
+      Builder.CreateBitCast(addr.emitRawPointer(*this), Int8PtrPtrTy),
+      Builder.CreateBitCast(value, Int8PtrTy)};
   EmitNounwindRuntimeCall(fn, args);
 
   if (ignored) return nullptr;
@@ -2643,7 +2638,7 @@ void CodeGenFunction::EmitARCDestroyWeak(Address addr) {
   if (!fn)
     fn = getARCIntrinsic(llvm::Intrinsic::objc_destroyWeak, CGM);
 
-  EmitNounwindRuntimeCall(fn, addr.getPointer());
+  EmitNounwindRuntimeCall(fn, addr.emitRawPointer(*this));
 }
 
 /// void \@objc_moveWeak(i8** %dest, i8** %src)
diff --git a/clang/lib/CodeGen/CGObjCGNU.cpp b/clang/lib/CodeGen/CGObjCGNU.cpp
index a36b0cdddaf0..4e7f777ba1d9 100644
--- a/clang/lib/CodeGen/CGObjCGNU.cpp
+++ b/clang/lib/CodeGen/CGObjCGNU.cpp
@@ -706,7 +706,8 @@ protected:
                               llvm::Value *cmd, MessageSendInfo &MSI) override {
     CGBuilderTy &Builder = CGF.Builder;
     llvm::Value *lookupArgs[] = {
-        EnforceType(Builder, ObjCSuper.getPointer(), PtrToObjCSuperTy), cmd};
+        EnforceType(Builder, ObjCSuper.emitRawPointer(CGF), PtrToObjCSuperTy),
+        cmd};
     return CGF.EmitNounwindRuntimeCall(MsgLookupSuperFn, lookupArgs);
   }
 
@@ -761,8 +762,8 @@ class CGObjCGNUstep : public CGObjCGNU {
       llvm::FunctionCallee LookupFn = SlotLookupFn;
 
       // Store the receiver on the stack so that we can reload it later
-      Address ReceiverPtr =
-        CGF.CreateTempAlloca(Receiver->getType(), CGF.getPointerAlign());
+      RawAddress ReceiverPtr =
+          CGF.CreateTempAlloca(Receiver->getType(), CGF.getPointerAlign());
       Builder.CreateStore(Receiver, ReceiverPtr);
 
       llvm::Value *self;
@@ -778,9 +779,9 @@ class CGObjCGNUstep : public CGObjCGNU {
         LookupFn2->addParamAttr(0, llvm::Attribute::NoCapture);
 
       llvm::Value *args[] = {
-              EnforceType(Builder, ReceiverPtr.getPointer(), PtrToIdTy),
-              EnforceType(Builder, cmd, SelectorTy),
-              EnforceType(Builder, self, IdTy) };
+          EnforceType(Builder, ReceiverPtr.getPointer(), PtrToIdTy),
+          EnforceType(Builder, cmd, SelectorTy),
+          EnforceType(Builder, self, IdTy)};
       llvm::CallBase *slot = CGF.EmitRuntimeCallOrInvoke(LookupFn, args);
       slot->setOnlyReadsMemory();
       slot->setMetadata(msgSendMDKind, node);
@@ -800,7 +801,7 @@ class CGObjCGNUstep : public CGObjCGNU {
                                 llvm::Value *cmd,
                                 MessageSendInfo &MSI) override {
       CGBuilderTy &Builder = CGF.Builder;
-      llvm::Value *lookupArgs[] = {ObjCSuper.getPointer(), cmd};
+      llvm::Value *lookupArgs[] = {ObjCSuper.emitRawPointer(CGF), cmd};
 
       llvm::CallInst *slot =
         CGF.EmitNounwindRuntimeCall(SlotLookupSuperFn, lookupArgs);
@@ -1221,10 +1222,10 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
                               llvm::Value *cmd, MessageSendInfo &MSI) override {
     // Don't access the slot unless we're trying to cache the result.
     CGBuilderTy &Builder = CGF.Builder;
-    llvm::Value *lookupArgs[] = {CGObjCGNU::EnforceType(Builder,
-                                                        ObjCSuper.getPointer(),
-                                                        PtrToObjCSuperTy),
-                                 cmd};
+    llvm::Value *lookupArgs[] = {
+        CGObjCGNU::EnforceType(Builder, ObjCSuper.emitRawPointer(CGF),
+                               PtrToObjCSuperTy),
+        cmd};
     return CGF.EmitNounwindRuntimeCall(MsgLookupSuperFn, lookupArgs);
   }
 
@@ -2186,7 +2187,8 @@ protected:
                               llvm::Value *cmd, MessageSendInfo &MSI) override {
     CGBuilderTy &Builder = CGF.Builder;
     llvm::Value *lookupArgs[] = {
-        EnforceType(Builder, ObjCSuper.getPointer(), PtrToObjCSuperTy), cmd,
+        EnforceType(Builder, ObjCSuper.emitRawPointer(CGF), PtrToObjCSuperTy),
+        cmd,
     };
 
     if (CGM.ReturnTypeUsesSRet(MSI.CallInfo))
@@ -4201,15 +4203,15 @@ void CGObjCGNU::EmitThrowStmt(CodeGenFunction &CGF,
 llvm::Value * CGObjCGNU::EmitObjCWeakRead(CodeGenFunction &CGF,
                                           Address AddrWeakObj) {
   CGBuilderTy &B = CGF.Builder;
-  return B.CreateCall(WeakReadFn,
-                      EnforceType(B, AddrWeakObj.getPointer(), PtrToIdTy));
+  return B.CreateCall(
+      WeakReadFn, EnforceType(B, AddrWeakObj.emitRawPointer(CGF), PtrToIdTy));
 }
 
 void CGObjCGNU::EmitObjCWeakAssign(CodeGenFunction &CGF,
                                    llvm::Value *src, Address dst) {
   CGBuilderTy &B = CGF.Builder;
   src = EnforceType(B, src, IdTy);
-  llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy);
+  llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), PtrToIdTy);
   B.CreateCall(WeakAssignFn, {src, dstVal});
 }
 
@@ -4218,7 +4220,7 @@ void CGObjCGNU::EmitObjCGlobalAssign(CodeGenFunction &CGF,
                                      bool threadlocal) {
   CGBuilderTy &B = CGF.Builder;
   src = EnforceType(B, src, IdTy);
-  llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy);
+  llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), PtrToIdTy);
   // FIXME. Add threadloca assign API
   assert(!threadlocal && "EmitObjCGlobalAssign - Threal Local API NYI");
   B.CreateCall(GlobalAssignFn, {src, dstVal});
@@ -4229,7 +4231,7 @@ void CGObjCGNU::EmitObjCIvarAssign(CodeGenFunction &CGF,
                                    llvm::Value *ivarOffset) {
   CGBuilderTy &B = CGF.Builder;
   src = EnforceType(B, src, IdTy);
-  llvm::Value *dstVal = EnforceType(B, dst.getPointer(), IdTy);
+  llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), IdTy);
   B.CreateCall(IvarAssignFn, {src, dstVal, ivarOffset});
 }
 
@@ -4237,7 +4239,7 @@ void CGObjCGNU::EmitObjCStrongCastAssign(CodeGenFunction &CGF,
                                          llvm::Value *src, Address dst) {
   CGBuilderTy &B = CGF.Builder;
   src = EnforceType(B, src, IdTy);
-  llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy);
+  llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), PtrToIdTy);
   B.CreateCall(StrongCastAssignFn, {src, dstVal});
 }
 
@@ -4246,8 +4248,8 @@ void CGObjCGNU::EmitGCMemmoveCollectable(CodeGenFunction &CGF,
                                          Address SrcPtr,
                                          llvm::Value *Size) {
   CGBuilderTy &B = CGF.Builder;
-  llvm::Value *DestPtrVal = EnforceType(B, DestPtr.getPointer(), PtrTy);
-  llvm::Value *SrcPtrVal = EnforceType(B, SrcPtr.getPointer(), PtrTy);
+  llvm::Value *DestPtrVal = EnforceType(B, DestPtr.emitRawPointer(CGF), PtrTy);
+  llvm::Value *SrcPtrVal = EnforceType(B, SrcPtr.emitRawPointer(CGF), PtrTy);
 
   B.CreateCall(MemMoveFn, {DestPtrVal, SrcPtrVal, Size});
 }
diff --git a/clang/lib/CodeGen/CGObjCMac.cpp b/clang/lib/CodeGen/CGObjCMac.cpp
index e815e097e1fb..5d0e27b4c15e 100644
--- a/clang/lib/CodeGen/CGObjCMac.cpp
+++ b/clang/lib/CodeGen/CGObjCMac.cpp
@@ -1310,7 +1310,7 @@ private:
   /// EmitSelector - Return a Value*, of type ObjCTypes.SelectorPtrTy,
   /// for the given selector.
   llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel);
-  Address EmitSelectorAddr(Selector Sel);
+  ConstantAddress EmitSelectorAddr(Selector Sel);
 
 public:
   CGObjCMac(CodeGen::CodeGenModule &cgm);
@@ -1538,7 +1538,7 @@ private:
   /// EmitSelector - Return a Value*, of type ObjCTypes.SelectorPtrTy,
   /// for the given selector.
   llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel);
-  Address EmitSelectorAddr(Selector Sel);
+  ConstantAddress EmitSelectorAddr(Selector Sel);
 
   /// GetInterfaceEHType - Get the cached ehtype for the given Objective-C
   /// interface. The return value has type EHTypePtrTy.
@@ -2064,9 +2064,8 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF,
                                     const ObjCMethodDecl *Method) {
   // Create and init a super structure; this is a (receiver, class)
   // pair we will pass to objc_msgSendSuper.
-  Address ObjCSuper =
-    CGF.CreateTempAlloca(ObjCTypes.SuperTy, CGF.getPointerAlign(),
-                         "objc_super");
+  RawAddress ObjCSuper = CGF.CreateTempAlloca(
+      ObjCTypes.SuperTy, CGF.getPointerAlign(), "objc_super");
   llvm::Value *ReceiverAsObject =
     CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy);
   CGF.Builder.CreateStore(ReceiverAsObject,
@@ -4259,7 +4258,7 @@ namespace {
 
       CGF.EmitBlock(FinallyCallExit);
       CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryExitFn(),
-                                  ExceptionData.getPointer());
+                                  ExceptionData.emitRawPointer(CGF));
 
       CGF.EmitBlock(FinallyNoCallExit);
 
@@ -4425,7 +4424,9 @@ void FragileHazards::emitHazardsInNewBlocks() {
 }
 
 static void addIfPresent(llvm::DenseSet<llvm::Value*> &S, Address V) {
-  if (V.isValid()) S.insert(V.getPointer());
+  if (V.isValid())
+    if (llvm::Value *Ptr = V.getBasePointer())
+      S.insert(Ptr);
 }
 
 void FragileHazards::collectLocals() {
@@ -4628,13 +4629,13 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
   //  - Call objc_exception_try_enter to push ExceptionData on top of
   //    the EH stack.
   CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(),
-                              ExceptionData.getPointer());
+                              ExceptionData.emitRawPointer(CGF));
 
   //  - Call setjmp on the exception data buffer.
   llvm::Constant *Zero = llvm::ConstantInt::get(CGF.Builder.getInt32Ty(), 0);
   llvm::Value *GEPIndexes[] = { Zero, Zero, Zero };
   llvm::Value *SetJmpBuffer = CGF.Builder.CreateGEP(
-      ObjCTypes.ExceptionDataTy, ExceptionData.getPointer(), GEPIndexes,
+      ObjCTypes.ExceptionDataTy, ExceptionData.emitRawPointer(CGF), GEPIndexes,
       "setjmp_buffer");
   llvm::CallInst *SetJmpResult = CGF.EmitNounwindRuntimeCall(
       ObjCTypes.getSetJmpFn(), SetJmpBuffer, "setjmp_result");
@@ -4673,9 +4674,9 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
   } else {
     // Retrieve the exception object.  We may emit multiple blocks but
     // nothing can cross this so the value is already in SSA form.
-    llvm::CallInst *Caught =
-      CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(),
-                                  ExceptionData.getPointer(), "caught");
+    llvm::CallInst *Caught = CGF.EmitNounwindRuntimeCall(
+        ObjCTypes.getExceptionExtractFn(), ExceptionData.emitRawPointer(CGF),
+        "caught");
 
     // Push the exception to rethrow onto the EH value stack for the
     // benefit of any @throws in the handlers.
@@ -4698,7 +4699,7 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
       // Enter a new exception try block (in case a @catch block
       // throws an exception).
       CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(),
-                                  ExceptionData.getPointer());
+                                  ExceptionData.emitRawPointer(CGF));
 
       llvm::CallInst *SetJmpResult =
         CGF.EmitNounwindRuntimeCall(ObjCTypes.getSetJmpFn(),
@@ -4829,9 +4830,9 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
       // Extract the new exception and save it to the
       // propagating-exception slot.
       assert(PropagatingExnVar.isValid());
-      llvm::CallInst *NewCaught =
-        CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(),
-                                    ExceptionData.getPointer(), "caught");
+      llvm::CallInst *NewCaught = CGF.EmitNounwindRuntimeCall(
+          ObjCTypes.getExceptionExtractFn(), ExceptionData.emitRawPointer(CGF),
+          "caught");
       CGF.Builder.CreateStore(NewCaught, PropagatingExnVar);
 
       // Don't pop the catch handler; the throw already did.
@@ -4861,9 +4862,8 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
 
     // Otherwise, just look in the buffer for the exception to throw.
     } else {
-      llvm::CallInst *Caught =
-        CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(),
-                                    ExceptionData.getPointer());
+      llvm::CallInst *Caught = CGF.EmitNounwindRuntimeCall(
+          ObjCTypes.getExceptionExtractFn(), ExceptionData.emitRawPointer(CGF));
       PropagatingExn = Caught;
     }
 
@@ -4906,7 +4906,7 @@ llvm::Value * CGObjCMac::EmitObjCWeakRead(CodeGen::CodeGenFunction &CGF,
                                           Address AddrWeakObj) {
   llvm::Type* DestTy = AddrWeakObj.getElementType();
   llvm::Value *AddrWeakObjVal = CGF.Builder.CreateBitCast(
-      AddrWeakObj.getPointer(), ObjCTypes.PtrObjectPtrTy);
+      AddrWeakObj.emitRawPointer(CGF), ObjCTypes.PtrObjectPtrTy);
   llvm::Value *read_weak =
     CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcReadWeakFn(),
                                 AddrWeakObjVal, "weakread");
@@ -4928,8 +4928,8 @@ void CGObjCMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
     src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
   }
   src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
-  llvm::Value *dstVal =
-      CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
+  llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF),
+                                                  ObjCTypes.PtrObjectPtrTy);
   llvm::Value *args[] = { src, dstVal };
   CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignWeakFn(),
                               args, "weakassign");
@@ -4950,8 +4950,8 @@ void CGObjCMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
     src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
   }
   src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
-  llvm::Value *dstVal =
-      CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
+  llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF),
+                                                  ObjCTypes.PtrObjectPtrTy);
   llvm::Value *args[] = {src, dstVal};
   if (!threadlocal)
     CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignGlobalFn(),
@@ -4977,8 +4977,8 @@ void CGObjCMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
     src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
   }
   src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
-  llvm::Value *dstVal =
-      CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
+  llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF),
+                                                  ObjCTypes.PtrObjectPtrTy);
   llvm::Value *args[] = {src, dstVal, ivarOffset};
   CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignIvarFn(), args);
 }
@@ -4997,8 +4997,8 @@ void CGObjCMac::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF,
     src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
   }
   src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
-  llvm::Value *dstVal =
-      CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
+  llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF),
+                                                  ObjCTypes.PtrObjectPtrTy);
   llvm::Value *args[] = {src, dstVal};
   CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignStrongCastFn(),
                               args, "strongassign");
@@ -5007,7 +5007,8 @@ void CGObjCMac::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF,
 void CGObjCMac::EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF,
                                          Address DestPtr, Address SrcPtr,
                                          llvm::Value *size) {
-  llvm::Value *args[] = { DestPtr.getPointer(), SrcPtr.getPointer(), size };
+  llvm::Value *args[] = {DestPtr.emitRawPointer(CGF),
+                         SrcPtr.emitRawPointer(CGF), size};
   CGF.EmitNounwindRuntimeCall(ObjCTypes.GcMemmoveCollectableFn(), args);
 }
 
@@ -5243,7 +5244,7 @@ llvm::Value *CGObjCMac::EmitSelector(CodeGenFunction &CGF, Selector Sel) {
   return CGF.Builder.CreateLoad(EmitSelectorAddr(Sel));
 }
 
-Address CGObjCMac::EmitSelectorAddr(Selector Sel) {
+ConstantAddress CGObjCMac::EmitSelectorAddr(Selector Sel) {
   CharUnits Align = CGM.getPointerAlign();
 
   llvm::GlobalVariable *&Entry = SelectorReferences[Sel];
@@ -5254,7 +5255,7 @@ Address CGObjCMac::EmitSelectorAddr(Selector Sel) {
     Entry->setExternallyInitialized(true);
   }
 
-  return Address(Entry, ObjCTypes.SelectorPtrTy, Align);
+  return ConstantAddress(Entry, ObjCTypes.SelectorPtrTy, Align);
 }
 
 llvm::Constant *CGObjCCommonMac::GetClassName(StringRef RuntimeName) {
@@ -7323,7 +7324,7 @@ CGObjCNonFragileABIMac::EmitVTableMessageSend(CodeGenFunction &CGF,
             ObjCTypes.MessageRefTy, CGF.getPointerAlign());
 
   // Update the message ref argument.
-  args[1].setRValue(RValue::get(mref.getPointer()));
+  args[1].setRValue(RValue::get(mref, CGF));
 
   // Load the function to call from the message ref table.
   Address calleeAddr = CGF.Builder.CreateStructGEP(mref, 0);
@@ -7552,9 +7553,8 @@ CGObjCNonFragileABIMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF,
   // ...
   // Create and init a super structure; this is a (receiver, class)
   // pair we will pass to objc_msgSendSuper.
-  Address ObjCSuper =
-    CGF.CreateTempAlloca(ObjCTypes.SuperTy, CGF.getPointerAlign(),
-                         "objc_super");
+  RawAddress ObjCSuper = CGF.CreateTempAlloca(
+      ObjCTypes.SuperTy, CGF.getPointerAlign(), "objc_super");
 
   llvm::Value *ReceiverAsObject =
     CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy);
@@ -7594,7 +7594,7 @@ llvm::Value *CGObjCNonFragileABIMac::EmitSelector(CodeGenFunction &CGF,
   return LI;
 }
 
-Address CGObjCNonFragileABIMac::EmitSelectorAddr(Selector Sel) {
+ConstantAddress CGObjCNonFragileABIMac::EmitSelectorAddr(Selector Sel) {
   llvm::GlobalVariable *&Entry = SelectorReferences[Sel];
   CharUnits Align = CGM.getPointerAlign();
   if (!Entry) {
@@ -7610,7 +7610,7 @@ Address CGObjCNonFragileABIMac::EmitSelectorAddr(Selector Sel) {
     CGM.addCompilerUsedGlobal(Entry);
   }
 
-  return Address(Entry, ObjCTypes.SelectorPtrTy, Align);
+  return ConstantAddress(Entry, ObjCTypes.SelectorPtrTy, Align);
 }
 
 /// EmitObjCIvarAssign - Code gen for assigning to a __strong object.
@@ -7629,8 +7629,8 @@ void CGObjCNonFragileABIMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
     src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
   }
   src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
-  llvm::Value *dstVal =
-      CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
+  llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF),
+                                                  ObjCTypes.PtrObjectPtrTy);
   llvm::Value *args[] = {src, dstVal, ivarOffset};
   CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignIvarFn(), args);
 }
@@ -7650,8 +7650,8 @@ void CGObjCNonFragileABIMac::EmitObjCStrongCastAssign(
     src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
   }
   src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
-  llvm::Value *dstVal =
-      CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
+  llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF),
+                                                  ObjCTypes.PtrObjectPtrTy);
   llvm::Value *args[] = {src, dstVal};
   CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignStrongCastFn(),
                               args, "weakassign");
@@ -7660,7 +7660,8 @@ void CGObjCNonFragileABIMac::EmitObjCStrongCastAssign(
 void CGObjCNonFragileABIMac::EmitGCMemmoveCollectable(
     CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr,
     llvm::Value *Size) {
-  llvm::Value *args[] = { DestPtr.getPointer(), SrcPtr.getPointer(), Size };
+  llvm::Value *args[] = {DestPtr.emitRawPointer(CGF),
+                         SrcPtr.emitRawPointer(CGF), Size};
   CGF.EmitNounwindRuntimeCall(ObjCTypes.GcMemmoveCollectableFn(), args);
 }
 
@@ -7672,7 +7673,7 @@ llvm::Value * CGObjCNonFragileABIMac::EmitObjCWeakRead(
   Address AddrWeakObj) {
   llvm::Type *DestTy = AddrWeakObj.getElementType();
   llvm::Value *AddrWeakObjVal = CGF.Builder.CreateBitCast(
-      AddrWeakObj.getPointer(), ObjCTypes.PtrObjectPtrTy);
+      AddrWeakObj.emitRawPointer(CGF), ObjCTypes.PtrObjectPtrTy);
   llvm::Value *read_weak =
     CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcReadWeakFn(),
                                 AddrWeakObjVal, "weakread");
@@ -7694,8 +7695,8 @@ void CGObjCNonFragileABIMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
     src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
   }
   src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
-  llvm::Value *dstVal =
-      CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
+  llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF),
+                                                  ObjCTypes.PtrObjectPtrTy);
   llvm::Value *args[] = {src, dstVal};
   CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignWeakFn(),
                               args, "weakassign");
@@ -7716,8 +7717,8 @@ void CGObjCNonFragileABIMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
     src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
   }
   src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
-  llvm::Value *dstVal =
-      CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
+  llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF),
+                                                  ObjCTypes.PtrObjectPtrTy);
   llvm::Value *args[] = {src, dstVal};
   if (!threadlocal)
     CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignGlobalFn(),
diff --git a/clang/lib/CodeGen/CGObjCRuntime.cpp b/clang/lib/CodeGen/CGObjCRuntime.cpp
index 424564f97599..01d0f35da196 100644
--- a/clang/lib/CodeGen/CGObjCRuntime.cpp
+++ b/clang/lib/CodeGen/CGObjCRuntime.cpp
@@ -67,7 +67,7 @@ LValue CGObjCRuntime::EmitValueForIvarAtOffset(CodeGen::CodeGenFunction &CGF,
   V = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, V, Offset, "add.ptr");
 
   if (!Ivar->isBitField()) {
-    LValue LV = CGF.MakeNaturalAlignAddrLValue(V, IvarTy);
+    LValue LV = CGF.MakeNaturalAlignRawAddrLValue(V, IvarTy);
     return LV;
   }
 
@@ -233,7 +233,7 @@ void CGObjCRuntime::EmitTryCatchStmt(CodeGenFunction &CGF,
       llvm::Instruction *CPICandidate = Handler.Block->getFirstNonPHI();
       if (auto *CPI = dyn_cast_or_null<llvm::CatchPadInst>(CPICandidate)) {
         CGF.CurrentFuncletPad = CPI;
-        CPI->setOperand(2, CGF.getExceptionSlot().getPointer());
+        CPI->setOperand(2, CGF.getExceptionSlot().emitRawPointer(CGF));
         CGF.EHStack.pushCleanup<CatchRetScope>(NormalCleanup, CPI);
       }
     }
@@ -405,7 +405,7 @@ bool CGObjCRuntime::canMessageReceiverBeNull(CodeGenFunction &CGF,
     auto self = curMethod->getSelfDecl();
     if (self->getType().isConstQualified()) {
       if (auto LI = dyn_cast<llvm::LoadInst>(receiver->stripPointerCasts())) {
-        llvm::Value *selfAddr = CGF.GetAddrOfLocalVar(self).getPointer();
+        llvm::Value *selfAddr = CGF.GetAddrOfLocalVar(self).emitRawPointer(CGF);
         if (selfAddr == LI->getPointerOperand()) {
           return false;
         }
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index e8a68dbcc687..934d1e722d0d 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -622,7 +622,7 @@ static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
     auto *GV = new llvm::GlobalVariable(
         CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
         llvm::GlobalValue::PrivateLinkage, Init, Name);
-    LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
+    LValue LV = CGF.MakeNaturalAlignRawAddrLValue(GV, Ty);
     RValue InitRVal;
     switch (CGF.getEvaluationKind(Ty)) {
     case TEK_Scalar:
@@ -668,8 +668,8 @@ static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
 
   llvm::Value *SrcBegin = nullptr;
   if (DRD)
-    SrcBegin = SrcAddr.getPointer();
-  llvm::Value *DestBegin = DestAddr.getPointer();
+    SrcBegin = SrcAddr.emitRawPointer(CGF);
+  llvm::Value *DestBegin = DestAddr.emitRawPointer(CGF);
   // Cast from pointer to array type to pointer to single element.
   llvm::Value *DestEnd =
       CGF.Builder.CreateGEP(DestAddr.getElementType(), DestBegin, NumElements);
@@ -912,7 +912,7 @@ static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
 
 static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
                           Address OriginalBaseAddress, llvm::Value *Addr) {
-  Address Tmp = Address::invalid();
+  RawAddress Tmp = RawAddress::invalid();
   Address TopTmp = Address::invalid();
   Address MostTopTmp = Address::invalid();
   BaseTy = BaseTy.getNonReferenceType();
@@ -971,10 +971,10 @@ Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
     Address SharedAddr = SharedAddresses[N].first.getAddress(CGF);
     llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
         SharedAddr.getElementType(), BaseLValue.getPointer(CGF),
-        SharedAddr.getPointer());
+        SharedAddr.emitRawPointer(CGF));
     llvm::Value *PrivatePointer =
         CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
-            PrivateAddr.getPointer(), SharedAddr.getType());
+            PrivateAddr.emitRawPointer(CGF), SharedAddr.getType());
     llvm::Value *Ptr = CGF.Builder.CreateGEP(
         SharedAddr.getElementType(), PrivatePointer, Adjustment);
     return castToBase(CGF, OrigVD->getType(),
@@ -1557,7 +1557,7 @@ static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc(
   return OMPBuilder.getTargetEntryUniqueInfo(FileInfoCallBack, ParentName);
 }
 
-Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
+ConstantAddress CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
   auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(VD); };
 
   auto LinkageForVariable = [&VD, this]() {
@@ -1579,8 +1579,8 @@ Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
       LinkageForVariable);
 
   if (!addr)
-    return Address::invalid();
-  return Address(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(VD));
+    return ConstantAddress::invalid();
+  return ConstantAddress(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(VD));
 }
 
 llvm::Constant *
@@ -1604,7 +1604,7 @@ Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
   llvm::Type *VarTy = VDAddr.getElementType();
   llvm::Value *Args[] = {
       emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
-      CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy),
+      CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy),
       CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
       getOrCreateThreadPrivateCache(VD)};
   return Address(
@@ -1627,7 +1627,8 @@ void CGOpenMPRuntime::emitThreadPrivateVarInit(
   // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
   // to register constructor/destructor for variable.
   llvm::Value *Args[] = {
-      OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy),
+      OMPLoc,
+      CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.VoidPtrTy),
       Ctor, CopyCtor, Dtor};
   CGF.EmitRuntimeCall(
       OMPBuilder.getOrCreateRuntimeFunction(
@@ -1900,13 +1901,13 @@ void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
 
     // OutlinedFn(&GTid, &zero_bound, CapturedStruct);
     Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
-    Address ZeroAddrBound =
+    RawAddress ZeroAddrBound =
         CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
                                          /*Name=*/".bound.zero.addr");
     CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddrBound);
     llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
     // ThreadId for serialized parallels is 0.
-    OutlinedFnArgs.push_back(ThreadIDAddr.getPointer());
+    OutlinedFnArgs.push_back(ThreadIDAddr.emitRawPointer(CGF));
     OutlinedFnArgs.push_back(ZeroAddrBound.getPointer());
     OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
 
@@ -2272,7 +2273,7 @@ void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
         emitUpdateLocation(CGF, Loc), // ident_t *<loc>
         getThreadID(CGF, Loc),        // i32 <gtid>
         BufSize,                      // size_t <buf_size>
-        CL.getPointer(),              // void *<copyprivate list>
+        CL.emitRawPointer(CGF),       // void *<copyprivate list>
         CpyFn,                        // void (*) (void *, void *) <copy_func>
         DidItVal                      // i32 did_it
     };
@@ -2591,10 +2592,10 @@ static void emitForStaticInitCall(
       ThreadId,
       CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1,
                                                   M2)), // Schedule type
-      Values.IL.getPointer(),                           // &isLastIter
-      Values.LB.getPointer(),                           // &LB
-      Values.UB.getPointer(),                           // &UB
-      Values.ST.getPointer(),                           // &Stride
+      Values.IL.emitRawPointer(CGF),                    // &isLastIter
+      Values.LB.emitRawPointer(CGF),                    // &LB
+      Values.UB.emitRawPointer(CGF),                    // &UB
+      Values.ST.emitRawPointer(CGF),                    // &Stride
       CGF.Builder.getIntN(Values.IVSize, 1),            // Incr
       Chunk                                             // Chunk
   };
@@ -2697,12 +2698,11 @@ llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
   //          kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
   //          kmp_int[32|64] *p_stride);
   llvm::Value *Args[] = {
-      emitUpdateLocation(CGF, Loc),
-      getThreadID(CGF, Loc),
-      IL.getPointer(), // &isLastIter
-      LB.getPointer(), // &Lower
-      UB.getPointer(), // &Upper
-      ST.getPointer()  // &Stride
+      emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
+      IL.emitRawPointer(CGF), // &isLastIter
+      LB.emitRawPointer(CGF), // &Lower
+      UB.emitRawPointer(CGF), // &Upper
+      ST.emitRawPointer(CGF)  // &Stride
   };
   llvm::Value *Call = CGF.EmitRuntimeCall(
       OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), Args);
@@ -3047,7 +3047,7 @@ emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
       CGF.Builder
           .CreatePointerBitCastOrAddrSpaceCast(TDBase.getAddress(CGF),
                                                CGF.VoidPtrTy, CGF.Int8Ty)
-          .getPointer()};
+          .emitRawPointer(CGF)};
   SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
                                           std::end(CommonArgs));
   if (isOpenMPTaskLoopDirective(Kind)) {
@@ -3574,7 +3574,8 @@ getPointerAndSize(CodeGenFunction &CGF, const Expr *E) {
         CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false);
     Address UpAddrAddress = UpAddrLVal.getAddress(CGF);
     llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32(
-        UpAddrAddress.getElementType(), UpAddrAddress.getPointer(), /*Idx0=*/1);
+        UpAddrAddress.getElementType(), UpAddrAddress.emitRawPointer(CGF),
+        /*Idx0=*/1);
     llvm::Value *LowIntPtr = CGF.Builder.CreatePtrToInt(Addr, CGF.SizeTy);
     llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGF.SizeTy);
     SizeVal = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
@@ -3888,8 +3889,9 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
         llvm::Value *Size;
         std::tie(Addr, Size) = getPointerAndSize(CGF, E);
         llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
-        LValue Base = CGF.MakeAddrLValue(
-            CGF.Builder.CreateGEP(AffinitiesArray, Idx), KmpTaskAffinityInfoTy);
+        LValue Base =
+            CGF.MakeAddrLValue(CGF.Builder.CreateGEP(CGF, AffinitiesArray, Idx),
+                               KmpTaskAffinityInfoTy);
         // affs[i].base_addr = &<Affinities[i].second>;
         LValue BaseAddrLVal = CGF.EmitLValueForField(
             Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
@@ -3910,7 +3912,7 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
     llvm::Value *LocRef = emitUpdateLocation(CGF, Loc);
     llvm::Value *GTid = getThreadID(CGF, Loc);
     llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
-        AffinitiesArray.getPointer(), CGM.VoidPtrTy);
+        AffinitiesArray.emitRawPointer(CGF), CGM.VoidPtrTy);
     // FIXME: Emit the function and ignore its result for now unless the
     // runtime function is properly implemented.
     (void)CGF.EmitRuntimeCall(
@@ -3921,8 +3923,8 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
   llvm::Value *NewTaskNewTaskTTy =
       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
           NewTask, KmpTaskTWithPrivatesPtrTy);
-  LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
-                                               KmpTaskTWithPrivatesQTy);
+  LValue Base = CGF.MakeNaturalAlignRawAddrLValue(NewTaskNewTaskTTy,
+                                                  KmpTaskTWithPrivatesQTy);
   LValue TDBase =
       CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
   // Fill the data in the resulting kmp_task_t record.
@@ -4047,7 +4049,7 @@ CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal,
           CGF.ConvertTypeForMem(KmpDependInfoPtrTy)),
       KmpDependInfoPtrTy->castAs<PointerType>());
   Address DepObjAddr = CGF.Builder.CreateGEP(
-      Base.getAddress(CGF),
+      CGF, Base.getAddress(CGF),
       llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
   LValue NumDepsBase = CGF.MakeAddrLValue(
       DepObjAddr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo());
@@ -4097,7 +4099,7 @@ static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
       LValue &PosLVal = *Pos.get<LValue *>();
       llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
       Base = CGF.MakeAddrLValue(
-          CGF.Builder.CreateGEP(DependenciesArray, Idx), KmpDependInfoTy);
+          CGF.Builder.CreateGEP(CGF, DependenciesArray, Idx), KmpDependInfoTy);
     }
     // deps[i].base_addr = &<Dependencies[i].second>;
     LValue BaseAddrLVal = CGF.EmitLValueForField(
@@ -4195,7 +4197,7 @@ void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF,
           ElSize,
           CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false));
       llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
-      Address DepAddr = CGF.Builder.CreateGEP(DependenciesArray, Pos);
+      Address DepAddr = CGF.Builder.CreateGEP(CGF, DependenciesArray, Pos);
       CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(CGF), Size);
 
       // Increase pos.
@@ -4430,7 +4432,7 @@ void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
       Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy),
       CGF.ConvertTypeForMem(KmpDependInfoTy));
   llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
-      Addr.getElementType(), Addr.getPointer(),
+      Addr.getElementType(), Addr.emitRawPointer(CGF),
       llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
   DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr,
                                                                CGF.VoidPtrTy);
@@ -4460,8 +4462,8 @@ void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
 
   Address Begin = Base.getAddress(CGF);
   // Cast from pointer to array type to pointer to single element.
-  llvm::Value *End = CGF.Builder.CreateGEP(
-      Begin.getElementType(), Begin.getPointer(), NumDeps);
+  llvm::Value *End = CGF.Builder.CreateGEP(Begin.getElementType(),
+                                           Begin.emitRawPointer(CGF), NumDeps);
   // The basic structure here is a while-do loop.
   llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body");
   llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done");
@@ -4469,7 +4471,7 @@ void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
   CGF.EmitBlock(BodyBB);
   llvm::PHINode *ElementPHI =
       CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast");
-  ElementPHI->addIncoming(Begin.getPointer(), EntryBB);
+  ElementPHI->addIncoming(Begin.emitRawPointer(CGF), EntryBB);
   Begin = Begin.withPointer(ElementPHI, KnownNonNull);
   Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(),
                             Base.getTBAAInfo());
@@ -4483,12 +4485,12 @@ void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
       FlagsLVal);
 
   // Shift the address forward by one element.
-  Address ElementNext =
-      CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext");
-  ElementPHI->addIncoming(ElementNext.getPointer(),
-                          CGF.Builder.GetInsertBlock());
+  llvm::Value *ElementNext =
+      CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext")
+          .emitRawPointer(CGF);
+  ElementPHI->addIncoming(ElementNext, CGF.Builder.GetInsertBlock());
   llvm::Value *IsEmpty =
-      CGF.Builder.CreateICmpEQ(ElementNext.getPointer(), End, "omp.isempty");
+      CGF.Builder.CreateICmpEQ(ElementNext, End, "omp.isempty");
   CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
   // Done.
   CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
@@ -4531,7 +4533,7 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
     DepTaskArgs[1] = ThreadID;
     DepTaskArgs[2] = NewTask;
     DepTaskArgs[3] = NumOfElements;
-    DepTaskArgs[4] = DependenciesArray.getPointer();
+    DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF);
     DepTaskArgs[5] = CGF.Builder.getInt32(0);
     DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
   }
@@ -4563,7 +4565,7 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
     DepWaitTaskArgs[0] = UpLoc;
     DepWaitTaskArgs[1] = ThreadID;
     DepWaitTaskArgs[2] = NumOfElements;
-    DepWaitTaskArgs[3] = DependenciesArray.getPointer();
+    DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
     DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
     DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
     DepWaitTaskArgs[6] =
@@ -4725,8 +4727,8 @@ static void EmitOMPAggregateReduction(
   const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
   llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
 
-  llvm::Value *RHSBegin = RHSAddr.getPointer();
-  llvm::Value *LHSBegin = LHSAddr.getPointer();
+  llvm::Value *RHSBegin = RHSAddr.emitRawPointer(CGF);
+  llvm::Value *LHSBegin = LHSAddr.emitRawPointer(CGF);
   // Cast from pointer to array type to pointer to single element.
   llvm::Value *LHSEnd =
       CGF.Builder.CreateGEP(LHSAddr.getElementType(), LHSBegin, NumElements);
@@ -4990,7 +4992,7 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
   QualType ReductionArrayTy = C.getConstantArrayType(
       C.VoidPtrTy, ArraySize, nullptr, ArraySizeModifier::Normal,
       /*IndexTypeQuals=*/0);
-  Address ReductionList =
+  RawAddress ReductionList =
       CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
   const auto *IPriv = Privates.begin();
   unsigned Idx = 0;
@@ -5462,7 +5464,7 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
       C.getConstantArrayType(RDType, ArraySize, nullptr,
                              ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
   // kmp_task_red_input_t .rd_input.[Size];
-  Address TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input.");
+  RawAddress TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input.");
   ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionOrigs,
                        Data.ReductionCopies, Data.ReductionOps);
   for (unsigned Cnt = 0; Cnt < Size; ++Cnt) {
@@ -5473,7 +5475,7 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
         TaskRedInput.getElementType(), TaskRedInput.getPointer(), Idxs,
         /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc,
         ".rd_input.gep.");
-    LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType);
+    LValue ElemLVal = CGF.MakeNaturalAlignRawAddrLValue(GEP, RDType);
     // ElemLVal.reduce_shar = &Shareds[Cnt];
     LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD);
     RCG.emitSharedOrigLValue(CGF, Cnt);
@@ -5629,7 +5631,7 @@ void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
       DepWaitTaskArgs[0] = UpLoc;
       DepWaitTaskArgs[1] = ThreadID;
       DepWaitTaskArgs[2] = NumOfElements;
-      DepWaitTaskArgs[3] = DependenciesArray.getPointer();
+      DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
       DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
       DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
       DepWaitTaskArgs[6] =
@@ -5852,7 +5854,7 @@ void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF,
   AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, CGF.getContext().VoidPtrTy,
                                            AllocatorTraitsLVal.getBaseInfo(),
                                            AllocatorTraitsLVal.getTBAAInfo());
-  llvm::Value *Traits = Addr.getPointer();
+  llvm::Value *Traits = Addr.emitRawPointer(CGF);
 
   llvm::Value *AllocatorVal =
       CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
@@ -7312,17 +7314,19 @@ private:
                       CGF.EmitOMPSharedLValue(MC.getAssociatedExpression())
                           .getAddress(CGF);
                 }
-                Size = CGF.Builder.CreatePtrDiff(
-                    CGF.Int8Ty, ComponentLB.getPointer(), LB.getPointer());
+                llvm::Value *ComponentLBPtr = ComponentLB.emitRawPointer(CGF);
+                llvm::Value *LBPtr = LB.emitRawPointer(CGF);
+                Size = CGF.Builder.CreatePtrDiff(CGF.Int8Ty, ComponentLBPtr,
+                                                 LBPtr);
                 break;
               }
             }
             assert(Size && "Failed to determine structure size");
             CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
-            CombinedInfo.BasePointers.push_back(BP.getPointer());
+            CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF));
             CombinedInfo.DevicePtrDecls.push_back(nullptr);
             CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
-            CombinedInfo.Pointers.push_back(LB.getPointer());
+            CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));
             CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
                 Size, CGF.Int64Ty, /*isSigned=*/true));
             CombinedInfo.Types.push_back(Flags);
@@ -7332,13 +7336,14 @@ private:
             LB = CGF.Builder.CreateConstGEP(ComponentLB, 1);
           }
           CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
-          CombinedInfo.BasePointers.push_back(BP.getPointer());
+          CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF));
           CombinedInfo.DevicePtrDecls.push_back(nullptr);
           CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
-          CombinedInfo.Pointers.push_back(LB.getPointer());
+          CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));
+          llvm::Value *LBPtr = LB.emitRawPointer(CGF);
           Size = CGF.Builder.CreatePtrDiff(
-              CGF.Int8Ty, CGF.Builder.CreateConstGEP(HB, 1).getPointer(),
-              LB.getPointer());
+              CGF.Int8Ty, CGF.Builder.CreateConstGEP(HB, 1).emitRawPointer(CGF),
+              LBPtr);
           CombinedInfo.Sizes.push_back(
               CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true));
           CombinedInfo.Types.push_back(Flags);
@@ -7356,20 +7361,21 @@ private:
             (Next == CE && MapType != OMPC_MAP_unknown)) {
           if (!IsMappingWholeStruct) {
             CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
-            CombinedInfo.BasePointers.push_back(BP.getPointer());
+            CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF));
             CombinedInfo.DevicePtrDecls.push_back(nullptr);
             CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
-            CombinedInfo.Pointers.push_back(LB.getPointer());
+            CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));
             CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
                 Size, CGF.Int64Ty, /*isSigned=*/true));
             CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize
                                                                       : 1);
           } else {
             StructBaseCombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
-            StructBaseCombinedInfo.BasePointers.push_back(BP.getPointer());
+            StructBaseCombinedInfo.BasePointers.push_back(
+                BP.emitRawPointer(CGF));
             StructBaseCombinedInfo.DevicePtrDecls.push_back(nullptr);
             StructBaseCombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
-            StructBaseCombinedInfo.Pointers.push_back(LB.getPointer());
+            StructBaseCombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));
             StructBaseCombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
                 Size, CGF.Int64Ty, /*isSigned=*/true));
             StructBaseCombinedInfo.NonContigInfo.Dims.push_back(
@@ -8211,11 +8217,11 @@ public:
     }
     CombinedInfo.Exprs.push_back(VD);
     // Base is the base of the struct
-    CombinedInfo.BasePointers.push_back(PartialStruct.Base.getPointer());
+    CombinedInfo.BasePointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
     CombinedInfo.DevicePtrDecls.push_back(nullptr);
     CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
     // Pointer is the address of the lowest element
-    llvm::Value *LB = LBAddr.getPointer();
+    llvm::Value *LB = LBAddr.emitRawPointer(CGF);
     const CXXMethodDecl *MD =
         CGF.CurFuncDecl ? dyn_cast<CXXMethodDecl>(CGF.CurFuncDecl) : nullptr;
     const CXXRecordDecl *RD = MD ? MD->getParent() : nullptr;
@@ -8229,7 +8235,7 @@ public:
       // if the this[:1] expression had appeared in a map clause with a map-type
       // of tofrom.
       // Emit this[:1]
-      CombinedInfo.Pointers.push_back(PartialStruct.Base.getPointer());
+      CombinedInfo.Pointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
       QualType Ty = MD->getFunctionObjectParameterType();
       llvm::Value *Size =
           CGF.Builder.CreateIntCast(CGF.getTypeSize(Ty), CGF.Int64Ty,
@@ -8238,7 +8244,7 @@ public:
     } else {
       CombinedInfo.Pointers.push_back(LB);
       // Size is (addr of {highest+1} element) - (addr of lowest element)
-      llvm::Value *HB = HBAddr.getPointer();
+      llvm::Value *HB = HBAddr.emitRawPointer(CGF);
       llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32(
           HBAddr.getElementType(), HB, /*Idx0=*/1);
       llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy);
@@ -8747,7 +8753,7 @@ public:
         Address PtrAddr = CGF.EmitLoadOfReference(CGF.MakeAddrLValue(
             CV, ElementType, CGF.getContext().getDeclAlign(VD),
             AlignmentSource::Decl));
-        CombinedInfo.Pointers.push_back(PtrAddr.getPointer());
+        CombinedInfo.Pointers.push_back(PtrAddr.emitRawPointer(CGF));
       } else {
         CombinedInfo.Pointers.push_back(CV);
       }
@@ -9558,10 +9564,11 @@ static void emitTargetCallKernelLaunch(
     bool HasNoWait = D.hasClausesOfKind<OMPNowaitClause>();
     unsigned NumTargetItems = InputInfo.NumberOfTargetItems;
 
-    llvm::Value *BasePointersArray = InputInfo.BasePointersArray.getPointer();
-    llvm::Value *PointersArray = InputInfo.PointersArray.getPointer();
-    llvm::Value *SizesArray = InputInfo.SizesArray.getPointer();
-    llvm::Value *MappersArray = InputInfo.MappersArray.getPointer();
+    llvm::Value *BasePointersArray =
+        InputInfo.BasePointersArray.emitRawPointer(CGF);
+    llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF);
+    llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF);
+    llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF);
 
     auto &&EmitTargetCallFallbackCB =
         [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
@@ -10309,15 +10316,16 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
     // Source location for the ident struct
     llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc());
 
-    llvm::Value *OffloadingArgs[] = {RTLoc,
-                                     DeviceID,
-                                     PointerNum,
-                                     InputInfo.BasePointersArray.getPointer(),
-                                     InputInfo.PointersArray.getPointer(),
-                                     InputInfo.SizesArray.getPointer(),
-                                     MapTypesArray,
-                                     MapNamesArray,
-                                     InputInfo.MappersArray.getPointer()};
+    llvm::Value *OffloadingArgs[] = {
+        RTLoc,
+        DeviceID,
+        PointerNum,
+        InputInfo.BasePointersArray.emitRawPointer(CGF),
+        InputInfo.PointersArray.emitRawPointer(CGF),
+        InputInfo.SizesArray.emitRawPointer(CGF),
+        MapTypesArray,
+        MapNamesArray,
+        InputInfo.MappersArray.emitRawPointer(CGF)};
 
     // Select the right runtime function call for each standalone
     // directive.
@@ -11128,7 +11136,7 @@ void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF,
       getThreadID(CGF, D.getBeginLoc()),
       llvm::ConstantInt::getSigned(CGM.Int32Ty, NumIterations.size()),
       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
-          CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).getPointer(),
+          CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).emitRawPointer(CGF),
           CGM.VoidPtrTy)};
 
   llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction(
@@ -11162,7 +11170,8 @@ static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM,
                           /*Volatile=*/false, Int64Ty);
   }
   llvm::Value *Args[] = {
-      ULoc, ThreadID, CGF.Builder.CreateConstArrayGEP(CntAddr, 0).getPointer()};
+      ULoc, ThreadID,
+      CGF.Builder.CreateConstArrayGEP(CntAddr, 0).emitRawPointer(CGF)};
   llvm::FunctionCallee RTLFn;
   llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
   OMPDoacrossKind<T> ODK;
@@ -11332,7 +11341,7 @@ Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,
         Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID(
             CGF, SourceLocation::getFromRawEncoding(LocEncoding));
         Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
-            Addr.getPointer(), CGF.VoidPtrTy);
+            Addr.emitRawPointer(CGF), CGF.VoidPtrTy);
         llvm::Value *AllocVal = getAllocatorVal(CGF, AllocExpr);
         Args[2] = AllocVal;
         CGF.EmitRuntimeCall(RTLFn, Args);
@@ -11690,15 +11699,17 @@ void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF,
       LLIVTy, getName({UniqueDeclName, "iv"}));
   cast<llvm::GlobalVariable>(LastIV)->setAlignment(
       IVLVal.getAlignment().getAsAlign());
-  LValue LastIVLVal = CGF.MakeNaturalAlignAddrLValue(LastIV, IVLVal.getType());
+  LValue LastIVLVal =
+      CGF.MakeNaturalAlignRawAddrLValue(LastIV, IVLVal.getType());
 
   // Last value of the lastprivate conditional.
   // decltype(priv_a) last_a;
   llvm::GlobalVariable *Last = OMPBuilder.getOrCreateInternalVariable(
       CGF.ConvertTypeForMem(LVal.getType()), UniqueDeclName);
-  Last->setAlignment(LVal.getAlignment().getAsAlign());
-  LValue LastLVal = CGF.MakeAddrLValue(
-      Address(Last, Last->getValueType(), LVal.getAlignment()), LVal.getType());
+  cast<llvm::GlobalVariable>(Last)->setAlignment(
+      LVal.getAlignment().getAsAlign());
+  LValue LastLVal =
+      CGF.MakeRawAddrLValue(Last, LVal.getType(), LVal.getAlignment());
 
   // Global loop counter. Required to handle inner parallel-for regions.
   // iv
@@ -11871,9 +11882,8 @@ void CGOpenMPRuntime::emitLastprivateConditionalFinalUpdate(
   // The variable was not updated in the region - exit.
   if (!GV)
     return;
-  LValue LPLVal = CGF.MakeAddrLValue(
-      Address(GV, GV->getValueType(), PrivLVal.getAlignment()),
-      PrivLVal.getType().getNonReferenceType());
+  LValue LPLVal = CGF.MakeRawAddrLValue(
+      GV, PrivLVal.getType().getNonReferenceType(), PrivLVal.getAlignment());
   llvm::Value *Res = CGF.EmitLoadOfScalar(LPLVal, Loc);
   CGF.EmitStoreOfScalar(Res, PrivLVal);
 }
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index c3206427b143..522ae3d35d22 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -1068,13 +1068,12 @@ public:
   /// \param Loc Location of the reference to threadprivate var.
   /// \return Address of the threadprivate variable for the current thread.
   virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
-                                         const VarDecl *VD,
-                                         Address VDAddr,
+                                         const VarDecl *VD, Address VDAddr,
                                          SourceLocation Loc);
 
   /// Returns the address of the variable marked as declare target with link
   /// clause OR as declare target with to clause and unified memory.
-  virtual Address getAddrOfDeclareTargetVar(const VarDecl *VD);
+  virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD);
 
   /// Emit a code for initialization of threadprivate variable. It emits
   /// a call to runtime library which adds initial value to the newly created
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index 299ee1460b3d..5baac8f0e3e2 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -1096,7 +1096,8 @@ void CGOpenMPRuntimeGPU::emitGenericVarsProlog(CodeGenFunction &CGF,
     llvm::PointerType *VarPtrTy = CGF.ConvertTypeForMem(VarTy)->getPointerTo();
     llvm::Value *CastedVoidPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
         VoidPtr, VarPtrTy, VD->getName() + "_on_stack");
-    LValue VarAddr = CGF.MakeNaturalAlignAddrLValue(CastedVoidPtr, VarTy);
+    LValue VarAddr =
+        CGF.MakeNaturalAlignPointeeRawAddrLValue(CastedVoidPtr, VarTy);
     Rec.second.PrivateAddr = VarAddr.getAddress(CGF);
     Rec.second.GlobalizedVal = VoidPtr;
 
@@ -1206,8 +1207,8 @@ void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF,
 
   bool IsBareKernel = D.getSingleClause<OMPXBareClause>();
 
-  Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
-                                                      /*Name=*/".zero.addr");
+  RawAddress ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
+                                                         /*Name=*/".zero.addr");
   CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddr);
   llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
   // We don't emit any thread id function call in bare kernel, but because the
@@ -1215,7 +1216,7 @@ void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF,
   if (IsBareKernel)
     OutlinedFnArgs.push_back(llvm::ConstantPointerNull::get(CGM.VoidPtrTy));
   else
-    OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).getPointer());
+    OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).emitRawPointer(CGF));
   OutlinedFnArgs.push_back(ZeroAddr.getPointer());
   OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
   emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
@@ -1289,7 +1290,7 @@ void CGOpenMPRuntimeGPU::emitParallelCall(CodeGenFunction &CGF,
         llvm::ConstantInt::get(CGF.Int32Ty, -1),
         FnPtr,
         ID,
-        Bld.CreateBitOrPointerCast(CapturedVarsAddrs.getPointer(),
+        Bld.CreateBitOrPointerCast(CapturedVarsAddrs.emitRawPointer(CGF),
                                    CGF.VoidPtrPtrTy),
         llvm::ConstantInt::get(CGM.SizeTy, CapturedVars.size())};
     CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
@@ -1503,17 +1504,18 @@ static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr,
       CGF.EmitBlock(PreCondBB);
       llvm::PHINode *PhiSrc =
           Bld.CreatePHI(Ptr.getType(), /*NumReservedValues=*/2);
-      PhiSrc->addIncoming(Ptr.getPointer(), CurrentBB);
+      PhiSrc->addIncoming(Ptr.emitRawPointer(CGF), CurrentBB);
       llvm::PHINode *PhiDest =
           Bld.CreatePHI(ElemPtr.getType(), /*NumReservedValues=*/2);
-      PhiDest->addIncoming(ElemPtr.getPointer(), CurrentBB);
+      PhiDest->addIncoming(ElemPtr.emitRawPointer(CGF), CurrentBB);
       Ptr = Address(PhiSrc, Ptr.getElementType(), Ptr.getAlignment());
       ElemPtr =
           Address(PhiDest, ElemPtr.getElementType(), ElemPtr.getAlignment());
+      llvm::Value *PtrEndRaw = PtrEnd.emitRawPointer(CGF);
+      llvm::Value *PtrRaw = Ptr.emitRawPointer(CGF);
       llvm::Value *PtrDiff = Bld.CreatePtrDiff(
-          CGF.Int8Ty, PtrEnd.getPointer(),
-          Bld.CreatePointerBitCastOrAddrSpaceCast(Ptr.getPointer(),
-                                                  CGF.VoidPtrTy));
+          CGF.Int8Ty, PtrEndRaw,
+          Bld.CreatePointerBitCastOrAddrSpaceCast(PtrRaw, CGF.VoidPtrTy));
       Bld.CreateCondBr(Bld.CreateICmpSGT(PtrDiff, Bld.getInt64(IntSize - 1)),
                        ThenBB, ExitBB);
       CGF.EmitBlock(ThenBB);
@@ -1528,8 +1530,8 @@ static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr,
                             TBAAAccessInfo());
       Address LocalPtr = Bld.CreateConstGEP(Ptr, 1);
       Address LocalElemPtr = Bld.CreateConstGEP(ElemPtr, 1);
-      PhiSrc->addIncoming(LocalPtr.getPointer(), ThenBB);
-      PhiDest->addIncoming(LocalElemPtr.getPointer(), ThenBB);
+      PhiSrc->addIncoming(LocalPtr.emitRawPointer(CGF), ThenBB);
+      PhiDest->addIncoming(LocalElemPtr.emitRawPointer(CGF), ThenBB);
       CGF.EmitBranch(PreCondBB);
       CGF.EmitBlock(ExitBB);
     } else {
@@ -1676,10 +1678,10 @@ static void emitReductionListCopy(
     // scope and that of functions it invokes (i.e., reduce_function).
     // RemoteReduceData[i] = (void*)&RemoteElem
     if (UpdateDestListPtr) {
-      CGF.EmitStoreOfScalar(Bld.CreatePointerBitCastOrAddrSpaceCast(
-                                DestElementAddr.getPointer(), CGF.VoidPtrTy),
-                            DestElementPtrAddr, /*Volatile=*/false,
-                            C.VoidPtrTy);
+      CGF.EmitStoreOfScalar(
+          Bld.CreatePointerBitCastOrAddrSpaceCast(
+              DestElementAddr.emitRawPointer(CGF), CGF.VoidPtrTy),
+          DestElementPtrAddr, /*Volatile=*/false, C.VoidPtrTy);
     }
 
     ++Idx;
@@ -1830,7 +1832,7 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM,
       // elemptr = ((CopyType*)(elemptrptr)) + I
       Address ElemPtr(ElemPtrPtr, CopyType, Align);
       if (NumIters > 1)
-        ElemPtr = Bld.CreateGEP(ElemPtr, Cnt);
+        ElemPtr = Bld.CreateGEP(CGF, ElemPtr, Cnt);
 
       // Get pointer to location in transfer medium.
       // MediumPtr = &medium[warp_id]
@@ -1894,7 +1896,7 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM,
           TargetElemPtrPtr, /*Volatile=*/false, C.VoidPtrTy, Loc);
       Address TargetElemPtr(TargetElemPtrVal, CopyType, Align);
       if (NumIters > 1)
-        TargetElemPtr = Bld.CreateGEP(TargetElemPtr, Cnt);
+        TargetElemPtr = Bld.CreateGEP(CGF, TargetElemPtr, Cnt);
 
       // *TargetElemPtr = SrcMediumVal;
       llvm::Value *SrcMediumValue =
@@ -2105,9 +2107,9 @@ static llvm::Function *emitShuffleAndReduceFunction(
   CGF.EmitBlock(ThenBB);
   // reduce_function(LocalReduceList, RemoteReduceList)
   llvm::Value *LocalReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
-      LocalReduceList.getPointer(), CGF.VoidPtrTy);
+      LocalReduceList.emitRawPointer(CGF), CGF.VoidPtrTy);
   llvm::Value *RemoteReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
-      RemoteReduceList.getPointer(), CGF.VoidPtrTy);
+      RemoteReduceList.emitRawPointer(CGF), CGF.VoidPtrTy);
   CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
       CGF, Loc, ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr});
   Bld.CreateBr(MergeBB);
@@ -2218,9 +2220,9 @@ static llvm::Value *emitListToGlobalCopyFunction(
     llvm::Value *BufferPtr =
         Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs);
     LValue GlobLVal = CGF.EmitLValueForField(
-        CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD);
+        CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD);
     Address GlobAddr = GlobLVal.getAddress(CGF);
-    GlobLVal.setAddress(Address(GlobAddr.getPointer(),
+    GlobLVal.setAddress(Address(GlobAddr.emitRawPointer(CGF),
                                 CGF.ConvertTypeForMem(Private->getType()),
                                 GlobAddr.getAlignment()));
     switch (CGF.getEvaluationKind(Private->getType())) {
@@ -2304,7 +2306,7 @@ static llvm::Value *emitListToGlobalReduceFunction(
 
   // 1. Build a list of reduction variables.
   // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
-  Address ReductionList =
+  RawAddress ReductionList =
       CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
   auto IPriv = Privates.begin();
   llvm::Value *Idxs[] = {CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg),
@@ -2319,10 +2321,10 @@ static llvm::Value *emitListToGlobalReduceFunction(
     llvm::Value *BufferPtr =
         Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs);
     LValue GlobLVal = CGF.EmitLValueForField(
-        CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD);
+        CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD);
     Address GlobAddr = GlobLVal.getAddress(CGF);
-    CGF.EmitStoreOfScalar(GlobAddr.getPointer(), Elem, /*Volatile=*/false,
-                          C.VoidPtrTy);
+    CGF.EmitStoreOfScalar(GlobAddr.emitRawPointer(CGF), Elem,
+                          /*Volatile=*/false, C.VoidPtrTy);
     if ((*IPriv)->getType()->isVariablyModifiedType()) {
       // Store array size.
       ++Idx;
@@ -2425,9 +2427,9 @@ static llvm::Value *emitGlobalToListCopyFunction(
     llvm::Value *BufferPtr =
         Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs);
     LValue GlobLVal = CGF.EmitLValueForField(
-        CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD);
+        CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD);
     Address GlobAddr = GlobLVal.getAddress(CGF);
-    GlobLVal.setAddress(Address(GlobAddr.getPointer(),
+    GlobLVal.setAddress(Address(GlobAddr.emitRawPointer(CGF),
                                 CGF.ConvertTypeForMem(Private->getType()),
                                 GlobAddr.getAlignment()));
     switch (CGF.getEvaluationKind(Private->getType())) {
@@ -2526,10 +2528,10 @@ static llvm::Value *emitGlobalToListReduceFunction(
     llvm::Value *BufferPtr =
         Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs);
     LValue GlobLVal = CGF.EmitLValueForField(
-        CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD);
+        CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD);
     Address GlobAddr = GlobLVal.getAddress(CGF);
-    CGF.EmitStoreOfScalar(GlobAddr.getPointer(), Elem, /*Volatile=*/false,
-                          C.VoidPtrTy);
+    CGF.EmitStoreOfScalar(GlobAddr.emitRawPointer(CGF), Elem,
+                          /*Volatile=*/false, C.VoidPtrTy);
     if ((*IPriv)->getType()->isVariablyModifiedType()) {
       // Store array size.
       ++Idx;
@@ -2545,7 +2547,7 @@ static llvm::Value *emitGlobalToListReduceFunction(
   }
 
   // Call reduce_function(ReduceList, GlobalReduceList)
-  llvm::Value *GlobalReduceList = ReductionList.getPointer();
+  llvm::Value *GlobalReduceList = ReductionList.emitRawPointer(CGF);
   Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg);
   llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar(
       AddrReduceListArg, /*Volatile=*/false, C.VoidPtrTy, Loc);
@@ -2876,7 +2878,7 @@ void CGOpenMPRuntimeGPU::emitReduction(
   }
 
   llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
-      ReductionList.getPointer(), CGF.VoidPtrTy);
+      ReductionList.emitRawPointer(CGF), CGF.VoidPtrTy);
   llvm::Function *ReductionFn = emitReductionFunction(
       CGF.CurFn->getName(), Loc, CGF.ConvertTypeForMem(ReductionArrayTy),
       Privates, LHSExprs, RHSExprs, ReductionOps);
@@ -3106,15 +3108,15 @@ llvm::Function *CGOpenMPRuntimeGPU::createParallelDataSharingWrapper(
   // Get the array of arguments.
   SmallVector<llvm::Value *, 8> Args;
 
-  Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).getPointer());
-  Args.emplace_back(ZeroAddr.getPointer());
+  Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).emitRawPointer(CGF));
+  Args.emplace_back(ZeroAddr.emitRawPointer(CGF));
 
   CGBuilderTy &Bld = CGF.Builder;
   auto CI = CS.capture_begin();
 
   // Use global memory for data sharing.
   // Handle passing of global args to workers.
-  Address GlobalArgs =
+  RawAddress GlobalArgs =
       CGF.CreateDefaultAlignTempAlloca(CGF.VoidPtrPtrTy, "global_args");
   llvm::Value *GlobalArgsPtr = GlobalArgs.getPointer();
   llvm::Value *DataSharingArgs[] = {GlobalArgsPtr};
@@ -3400,7 +3402,7 @@ void CGOpenMPRuntimeGPU::adjustTargetSpecificDataForLambdas(
         VDAddr = CGF.EmitLoadOfReferenceLValue(VDAddr,
                                                VD->getType().getCanonicalType())
                      .getAddress(CGF);
-      CGF.EmitStoreOfScalar(VDAddr.getPointer(), VarLVal);
+      CGF.EmitStoreOfScalar(VDAddr.emitRawPointer(CGF), VarLVal);
     }
   }
 }
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index cb5a004e4f4a..576fe2f7a2d4 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -2294,7 +2294,7 @@ std::pair<llvm::Value*, llvm::Type *> CodeGenFunction::EmitAsmInputLValue(
 
   Address Addr = InputValue.getAddress(*this);
   ConstraintStr += '*';
-  return {Addr.getPointer(), Addr.getElementType()};
+  return {InputValue.getPointer(*this), Addr.getElementType()};
 }
 
 std::pair<llvm::Value *, llvm::Type *>
@@ -2701,7 +2701,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
 
       ArgTypes.push_back(DestAddr.getType());
       ArgElemTypes.push_back(DestAddr.getElementType());
-      Args.push_back(DestAddr.getPointer());
+      Args.push_back(DestAddr.emitRawPointer(*this));
       Constraints += "=*";
       Constraints += OutputConstraint;
       ReadOnly = ReadNone = false;
@@ -3076,8 +3076,8 @@ CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) {
   CapturedStmtInfo->setContextValue(Builder.CreateLoad(DeclPtr));
 
   // Initialize variable-length arrays.
-  LValue Base = MakeNaturalAlignAddrLValue(CapturedStmtInfo->getContextValue(),
-                                           Ctx.getTagDeclType(RD));
+  LValue Base = MakeNaturalAlignRawAddrLValue(
+      CapturedStmtInfo->getContextValue(), Ctx.getTagDeclType(RD));
   for (auto *FD : RD->fields()) {
     if (FD->hasCapturedVLAType()) {
       auto *ExprArg =
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index f37ac549d10a..e6d504bcdeca 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -350,7 +350,8 @@ void CodeGenFunction::GenerateOpenMPCapturedVars(
         LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
 
         llvm::Value *SrcAddrVal = EmitScalarConversion(
-            DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
+            DstAddr.emitRawPointer(*this),
+            Ctx.getPointerType(Ctx.getUIntPtrType()),
             Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
         LValue SrcLV =
             MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
@@ -364,7 +365,8 @@ void CodeGenFunction::GenerateOpenMPCapturedVars(
       CapturedVars.push_back(CV);
     } else {
       assert(CurCap->capturesVariable() && "Expected capture by reference.");
-      CapturedVars.push_back(EmitLValue(*I).getAddress(*this).getPointer());
+      CapturedVars.push_back(
+          EmitLValue(*I).getAddress(*this).emitRawPointer(*this));
     }
   }
 }
@@ -375,8 +377,9 @@ static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc,
   ASTContext &Ctx = CGF.getContext();
 
   llvm::Value *CastedPtr = CGF.EmitScalarConversion(
-      AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(),
+      AddrLV.getAddress(CGF).emitRawPointer(CGF), Ctx.getUIntPtrType(),
       Ctx.getPointerType(DstType), Loc);
+  // FIXME: should the pointee type (DstType) be passed?
   Address TmpAddr =
       CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress(CGF);
   return TmpAddr;
@@ -702,8 +705,8 @@ void CodeGenFunction::EmitOMPAggregateAssign(
   llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
   SrcAddr = SrcAddr.withElementType(DestAddr.getElementType());
 
-  llvm::Value *SrcBegin = SrcAddr.getPointer();
-  llvm::Value *DestBegin = DestAddr.getPointer();
+  llvm::Value *SrcBegin = SrcAddr.emitRawPointer(*this);
+  llvm::Value *DestBegin = DestAddr.emitRawPointer(*this);
   // Cast from pointer to array type to pointer to single element.
   llvm::Value *DestEnd = Builder.CreateInBoundsGEP(DestAddr.getElementType(),
                                                    DestBegin, NumElements);
@@ -1007,10 +1010,10 @@ bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) {
           CopyBegin = createBasicBlock("copyin.not.master");
           CopyEnd = createBasicBlock("copyin.not.master.end");
           // TODO: Avoid ptrtoint conversion.
-          auto *MasterAddrInt =
-              Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy);
-          auto *PrivateAddrInt =
-              Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy);
+          auto *MasterAddrInt = Builder.CreatePtrToInt(
+              MasterAddr.emitRawPointer(*this), CGM.IntPtrTy);
+          auto *PrivateAddrInt = Builder.CreatePtrToInt(
+              PrivateAddr.emitRawPointer(*this), CGM.IntPtrTy);
           Builder.CreateCondBr(
               Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,
               CopyEnd);
@@ -1666,7 +1669,7 @@ Address CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
 
   llvm::Type *VarTy = VDAddr.getElementType();
   llvm::Value *Data =
-      CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy);
+      CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy);
   llvm::ConstantInt *Size = CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy));
   std::string Suffix = getNameWithSeparators({"cache", ""});
   llvm::Twine CacheName = Twine(CGM.getMangledName(VD)).concat(Suffix);
@@ -2045,7 +2048,7 @@ void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) {
                            ->getParam(0)
                            ->getType()
                            .getNonReferenceType();
-  Address CountAddr = CreateMemTemp(LogicalTy, ".count.addr");
+  RawAddress CountAddr = CreateMemTemp(LogicalTy, ".count.addr");
   emitCapturedStmtCall(*this, DistanceClosure, {CountAddr.getPointer()});
   llvm::Value *DistVal = Builder.CreateLoad(CountAddr, ".count");
 
@@ -2061,7 +2064,7 @@ void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) {
     LValue LCVal = EmitLValue(LoopVarRef);
     Address LoopVarAddress = LCVal.getAddress(*this);
     emitCapturedStmtCall(*this, LoopVarClosure,
-                         {LoopVarAddress.getPointer(), IndVar});
+                         {LoopVarAddress.emitRawPointer(*this), IndVar});
 
     RunCleanupsScope BodyScope(*this);
     EmitStmt(BodyStmt);
@@ -4795,7 +4798,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
       ParamTypes.push_back(PrivatesPtr->getType());
       for (const Expr *E : Data.PrivateVars) {
         const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
-        Address PrivatePtr = CGF.CreateMemTemp(
+        RawAddress PrivatePtr = CGF.CreateMemTemp(
             CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
         PrivatePtrs.emplace_back(VD, PrivatePtr);
         CallArgs.push_back(PrivatePtr.getPointer());
@@ -4803,7 +4806,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
       }
       for (const Expr *E : Data.FirstprivateVars) {
         const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
-        Address PrivatePtr =
+        RawAddress PrivatePtr =
             CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
                               ".firstpriv.ptr.addr");
         PrivatePtrs.emplace_back(VD, PrivatePtr);
@@ -4813,7 +4816,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
       }
       for (const Expr *E : Data.LastprivateVars) {
         const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
-        Address PrivatePtr =
+        RawAddress PrivatePtr =
             CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
                               ".lastpriv.ptr.addr");
         PrivatePtrs.emplace_back(VD, PrivatePtr);
@@ -4826,7 +4829,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
           Ty = CGF.getContext().getPointerType(Ty);
         if (isAllocatableDecl(VD))
           Ty = CGF.getContext().getPointerType(Ty);
-        Address PrivatePtr = CGF.CreateMemTemp(
+        RawAddress PrivatePtr = CGF.CreateMemTemp(
             CGF.getContext().getPointerType(Ty), ".local.ptr.addr");
         auto Result = UntiedLocalVars.insert(
             std::make_pair(VD, std::make_pair(PrivatePtr, Address::invalid())));
@@ -4859,7 +4862,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
         if (auto *DI = CGF.getDebugInfo())
           if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
             (void)DI->EmitDeclareOfAutoVariable(
-                Pair.first, Pair.second.getPointer(), CGF.Builder,
+                Pair.first, Pair.second.getBasePointer(), CGF.Builder,
                 /*UsePointerValue*/ true);
       }
       // Adjust mapping for internal locals by mapping actual memory instead of
@@ -4912,14 +4915,14 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
                                                            RedCG, Cnt);
         Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
             CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
-        Replacement =
-            Address(CGF.EmitScalarConversion(
-                        Replacement.getPointer(), CGF.getContext().VoidPtrTy,
-                        CGF.getContext().getPointerType(
-                            Data.ReductionCopies[Cnt]->getType()),
-                        Data.ReductionCopies[Cnt]->getExprLoc()),
-                    CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
-                    Replacement.getAlignment());
+        Replacement = Address(
+            CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),
+                                     CGF.getContext().VoidPtrTy,
+                                     CGF.getContext().getPointerType(
+                                         Data.ReductionCopies[Cnt]->getType()),
+                                     Data.ReductionCopies[Cnt]->getExprLoc()),
+            CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
+            Replacement.getAlignment());
         Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
         Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
       }
@@ -4970,7 +4973,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
             CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
         Replacement = Address(
             CGF.EmitScalarConversion(
-                Replacement.getPointer(), CGF.getContext().VoidPtrTy,
+                Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,
                 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
                 InRedPrivs[Cnt]->getExprLoc()),
             CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
@@ -5089,7 +5092,7 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
     // If there is no user-defined mapper, the mapper array will be nullptr. In
     // this case, we don't need to privatize it.
     if (!isa_and_nonnull<llvm::ConstantPointerNull>(
-            InputInfo.MappersArray.getPointer())) {
+            InputInfo.MappersArray.emitRawPointer(*this))) {
       MVD = createImplicitFirstprivateForType(
           getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
       TargetScope.addPrivate(MVD, InputInfo.MappersArray);
@@ -5115,7 +5118,7 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
       ParamTypes.push_back(PrivatesPtr->getType());
       for (const Expr *E : Data.FirstprivateVars) {
         const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
-        Address PrivatePtr =
+        RawAddress PrivatePtr =
             CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
                               ".firstpriv.ptr.addr");
         PrivatePtrs.emplace_back(VD, PrivatePtr);
@@ -5194,14 +5197,14 @@ void CodeGenFunction::processInReduction(const OMPExecutableDirective &S,
                                                          RedCG, Cnt);
       Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
           CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
-      Replacement =
-          Address(CGF.EmitScalarConversion(
-                      Replacement.getPointer(), CGF.getContext().VoidPtrTy,
-                      CGF.getContext().getPointerType(
-                          Data.ReductionCopies[Cnt]->getType()),
-                      Data.ReductionCopies[Cnt]->getExprLoc()),
-                  CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
-                  Replacement.getAlignment());
+      Replacement = Address(
+          CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),
+                                   CGF.getContext().VoidPtrTy,
+                                   CGF.getContext().getPointerType(
+                                       Data.ReductionCopies[Cnt]->getType()),
+                                   Data.ReductionCopies[Cnt]->getExprLoc()),
+          CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
+          Replacement.getAlignment());
       Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
       Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
     }
@@ -5247,7 +5250,7 @@ void CodeGenFunction::processInReduction(const OMPExecutableDirective &S,
           CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
       Replacement = Address(
           CGF.EmitScalarConversion(
-              Replacement.getPointer(), CGF.getContext().VoidPtrTy,
+              Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,
               CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
               InRedPrivs[Cnt]->getExprLoc()),
           CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
@@ -5394,7 +5397,7 @@ void CodeGenFunction::EmitOMPDepobjDirective(const OMPDepobjDirective &S) {
     Dependencies.DepExprs.append(DC->varlist_begin(), DC->varlist_end());
     Address DepAddr = CGM.getOpenMPRuntime().emitDepobjDependClause(
         *this, Dependencies, DC->getBeginLoc());
-    EmitStoreOfScalar(DepAddr.getPointer(), DOLVal);
+    EmitStoreOfScalar(DepAddr.emitRawPointer(*this), DOLVal);
     return;
   }
   if (const auto *DC = S.getSingleClause<OMPDestroyClause>()) {
@@ -6471,21 +6474,21 @@ static void emitOMPAtomicCompareExpr(
           D->getType()->hasSignedIntegerRepresentation());
 
   llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{
-      XAddr.getPointer(), XAddr.getElementType(),
+      XAddr.emitRawPointer(CGF), XAddr.getElementType(),
       X->getType()->hasSignedIntegerRepresentation(),
       X->getType().isVolatileQualified()};
   llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal;
   if (V) {
     LValue LV = CGF.EmitLValue(V);
     Address Addr = LV.getAddress(CGF);
-    VOpVal = {Addr.getPointer(), Addr.getElementType(),
+    VOpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(),
               V->getType()->hasSignedIntegerRepresentation(),
               V->getType().isVolatileQualified()};
   }
   if (R) {
     LValue LV = CGF.EmitLValue(R);
     Address Addr = LV.getAddress(CGF);
-    ROpVal = {Addr.getPointer(), Addr.getElementType(),
+    ROpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(),
               R->getType()->hasSignedIntegerRepresentation(),
               R->getType().isVolatileQualified()};
   }
@@ -7029,7 +7032,7 @@ void CodeGenFunction::EmitOMPInteropDirective(const OMPInteropDirective &S) {
     std::tie(NumDependences, DependenciesArray) =
         CGM.getOpenMPRuntime().emitDependClause(*this, Data.Dependences,
                                                 S.getBeginLoc());
-    DependenceList = DependenciesArray.getPointer();
+    DependenceList = DependenciesArray.emitRawPointer(*this);
   }
   Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();
 
diff --git a/clang/lib/CodeGen/CGVTables.cpp b/clang/lib/CodeGen/CGVTables.cpp
index 8dee3f74b44b..862369ae009f 100644
--- a/clang/lib/CodeGen/CGVTables.cpp
+++ b/clang/lib/CodeGen/CGVTables.cpp
@@ -201,14 +201,13 @@ CodeGenFunction::GenerateVarArgsThunk(llvm::Function *Fn,
 
   // Find the first store of "this", which will be to the alloca associated
   // with "this".
-  Address ThisPtr =
-      Address(&*AI, ConvertTypeForMem(MD->getFunctionObjectParameterType()),
-              CGM.getClassPointerAlignment(MD->getParent()));
+  Address ThisPtr = makeNaturalAddressForPointer(
+      &*AI, MD->getFunctionObjectParameterType(),
+      CGM.getClassPointerAlignment(MD->getParent()));
   llvm::BasicBlock *EntryBB = &Fn->front();
   llvm::BasicBlock::iterator ThisStore =
       llvm::find_if(*EntryBB, [&](llvm::Instruction &I) {
-        return isa<llvm::StoreInst>(I) &&
-               I.getOperand(0) == ThisPtr.getPointer();
+        return isa<llvm::StoreInst>(I) && I.getOperand(0) == &*AI;
       });
   assert(ThisStore != EntryBB->end() &&
          "Store of this should be in entry block?");
diff --git a/clang/lib/CodeGen/CGValue.h b/clang/lib/CodeGen/CGValue.h
index 1e6f67250583..9265bcbb438e 100644
--- a/clang/lib/CodeGen/CGValue.h
+++ b/clang/lib/CodeGen/CGValue.h
@@ -14,12 +14,13 @@
 #ifndef LLVM_CLANG_LIB_CODEGEN_CGVALUE_H
 #define LLVM_CLANG_LIB_CODEGEN_CGVALUE_H
 
+#include "Address.h"
+#include "CodeGenTBAA.h"
+#include "EHScopeStack.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Type.h"
-#include "llvm/IR/Value.h"
 #include "llvm/IR/Type.h"
-#include "Address.h"
-#include "CodeGenTBAA.h"
+#include "llvm/IR/Value.h"
 
 namespace llvm {
   class Constant;
@@ -28,57 +29,64 @@ namespace llvm {
 
 namespace clang {
 namespace CodeGen {
-  class AggValueSlot;
-  class CodeGenFunction;
-  struct CGBitFieldInfo;
+class AggValueSlot;
+class CGBuilderTy;
+class CodeGenFunction;
+struct CGBitFieldInfo;
 
 /// RValue - This trivial value class is used to represent the result of an
 /// expression that is evaluated.  It can be one of three things: either a
 /// simple LLVM SSA value, a pair of SSA values for complex numbers, or the
 /// address of an aggregate value in memory.
 class RValue {
-  enum Flavor { Scalar, Complex, Aggregate };
+  friend struct DominatingValue<RValue>;
 
-  // The shift to make to an aggregate's alignment to make it look
-  // like a pointer.
-  enum { AggAlignShift = 4 };
+  enum FlavorEnum { Scalar, Complex, Aggregate };
 
-  // Stores first value and flavor.
-  llvm::PointerIntPair<llvm::Value *, 2, Flavor> V1;
-  // Stores second value and volatility.
-  llvm::PointerIntPair<llvm::Value *, 1, bool> V2;
-  // Stores element type for aggregate values.
-  llvm::Type *ElementType;
+  union {
+    // Stores first and second value.
+    struct {
+      llvm::Value *first;
+      llvm::Value *second;
+    } Vals;
+
+    // Stores aggregate address.
+    Address AggregateAddr;
+  };
+
+  unsigned IsVolatile : 1;
+  unsigned Flavor : 2;
 
 public:
-  bool isScalar() const { return V1.getInt() == Scalar; }
-  bool isComplex() const { return V1.getInt() == Complex; }
-  bool isAggregate() const { return V1.getInt() == Aggregate; }
+  RValue() : Vals{nullptr, nullptr}, Flavor(Scalar) {}
+
+  bool isScalar() const { return Flavor == Scalar; }
+  bool isComplex() const { return Flavor == Complex; }
+  bool isAggregate() const { return Flavor == Aggregate; }
 
-  bool isVolatileQualified() const { return V2.getInt(); }
+  bool isVolatileQualified() const { return IsVolatile; }
 
   /// getScalarVal() - Return the Value* of this scalar value.
   llvm::Value *getScalarVal() const {
     assert(isScalar() && "Not a scalar!");
-    return V1.getPointer();
+    return Vals.first;
   }
 
   /// getComplexVal - Return the real/imag components of this complex value.
   ///
   std::pair<llvm::Value *, llvm::Value *> getComplexVal() const {
-    return std::make_pair(V1.getPointer(), V2.getPointer());
+    return std::make_pair(Vals.first, Vals.second);
   }
 
   /// getAggregateAddr() - Return the Value* of the address of the aggregate.
   Address getAggregateAddress() const {
     assert(isAggregate() && "Not an aggregate!");
-    auto align = reinterpret_cast<uintptr_t>(V2.getPointer()) >> AggAlignShift;
-    return Address(
-        V1.getPointer(), ElementType, CharUnits::fromQuantity(align));
+    return AggregateAddr;
   }
-  llvm::Value *getAggregatePointer() const {
-    assert(isAggregate() && "Not an aggregate!");
-    return V1.getPointer();
+
+  llvm::Value *getAggregatePointer(QualType PointeeType,
+                                   CodeGenFunction &CGF) const {
+    return getAggregateAddress().getBasePointer();
   }
 
   static RValue getIgnored() {
@@ -88,17 +96,19 @@ public:
 
   static RValue get(llvm::Value *V) {
     RValue ER;
-    ER.V1.setPointer(V);
-    ER.V1.setInt(Scalar);
-    ER.V2.setInt(false);
+    ER.Vals.first = V;
+    ER.Flavor = Scalar;
+    ER.IsVolatile = false;
     return ER;
   }
+  static RValue get(Address Addr, CodeGenFunction &CGF) {
+    return RValue::get(Addr.emitRawPointer(CGF));
+  }
   static RValue getComplex(llvm::Value *V1, llvm::Value *V2) {
     RValue ER;
-    ER.V1.setPointer(V1);
-    ER.V2.setPointer(V2);
-    ER.V1.setInt(Complex);
-    ER.V2.setInt(false);
+    ER.Vals = {V1, V2};
+    ER.Flavor = Complex;
+    ER.IsVolatile = false;
     return ER;
   }
   static RValue getComplex(const std::pair<llvm::Value *, llvm::Value *> &C) {
@@ -107,15 +117,15 @@ public:
   // FIXME: Aggregate rvalues need to retain information about whether they are
   // volatile or not.  Remove default to find all places that probably get this
   // wrong.
+
+  /// Convert an Address to an RValue. If the Address is not
+  /// signed, create an RValue using the unsigned address. Otherwise, resign the
+  /// address using the provided type.
   static RValue getAggregate(Address addr, bool isVolatile = false) {
     RValue ER;
-    ER.V1.setPointer(addr.getPointer());
-    ER.V1.setInt(Aggregate);
-    ER.ElementType = addr.getElementType();
-
-    auto align = static_cast<uintptr_t>(addr.getAlignment().getQuantity());
-    ER.V2.setPointer(reinterpret_cast<llvm::Value*>(align << AggAlignShift));
-    ER.V2.setInt(isVolatile);
+    ER.AggregateAddr = addr;
+    ER.Flavor = Aggregate;
+    ER.IsVolatile = isVolatile;
     return ER;
   }
 };
@@ -178,8 +188,10 @@ class LValue {
     MatrixElt     // This is a matrix element, use getVector*
   } LVType;
 
-  llvm::Value *V;
-  llvm::Type *ElementType;
+  union {
+    Address Addr = Address::invalid();
+    llvm::Value *V;
+  };
 
   union {
     // Index into a vector subscript: V[i]
@@ -197,10 +209,6 @@ class LValue {
   // 'const' is unused here
   Qualifiers Quals;
 
-  // The alignment to use when accessing this lvalue.  (For vector elements,
-  // this is the alignment of the whole vector.)
-  unsigned Alignment;
-
   // objective-c's ivar
   bool Ivar:1;
 
@@ -234,23 +242,19 @@ class LValue {
   Expr *BaseIvarExp;
 
 private:
-  void Initialize(QualType Type, Qualifiers Quals, CharUnits Alignment,
+  void Initialize(QualType Type, Qualifiers Quals, Address Addr,
                   LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) {
-    assert((!Alignment.isZero() || Type->isIncompleteType()) &&
-           "initializing l-value with zero alignment!");
-    if (isGlobalReg())
-      assert(ElementType == nullptr && "Global reg does not store elem type");
-    else
-      assert(ElementType != nullptr && "Must have elem type");
-
     this->Type = Type;
     this->Quals = Quals;
     const unsigned MaxAlign = 1U << 31;
-    this->Alignment = Alignment.getQuantity() <= MaxAlign
-                          ? Alignment.getQuantity()
-                          : MaxAlign;
-    assert(this->Alignment == Alignment.getQuantity() &&
-           "Alignment exceeds allowed max!");
+    CharUnits Alignment = Addr.getAlignment();
+    assert((isGlobalReg() || !Alignment.isZero() || Type->isIncompleteType()) &&
+           "initializing l-value with zero alignment!");
+    if (Alignment.getQuantity() > MaxAlign) {
+      assert(false && "Alignment exceeds allowed max!");
+      Alignment = CharUnits::fromQuantity(MaxAlign);
+    }
+    this->Addr = Addr;
     this->BaseInfo = BaseInfo;
     this->TBAAInfo = TBAAInfo;
 
@@ -262,6 +266,16 @@ private:
     this->BaseIvarExp = nullptr;
   }
 
+  void initializeSimpleLValue(Address Addr, QualType Type,
+                              LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo,
+                              ASTContext &Context) {
+    Qualifiers QS = Type.getQualifiers();
+    QS.setObjCGCAttr(Context.getObjCGCAttrKind(Type));
+    LVType = Simple;
+    Initialize(Type, QS, Addr, BaseInfo, TBAAInfo);
+    assert(Addr.getBasePointer()->getType()->isPointerTy());
+  }
+
 public:
   bool isSimple() const { return LVType == Simple; }
   bool isVectorElt() const { return LVType == VectorElt; }
@@ -328,8 +342,8 @@ public:
 
   LangAS getAddressSpace() const { return Quals.getAddressSpace(); }
 
-  CharUnits getAlignment() const { return CharUnits::fromQuantity(Alignment); }
-  void setAlignment(CharUnits A) { Alignment = A.getQuantity(); }
+  CharUnits getAlignment() const { return Addr.getAlignment(); }
+  void setAlignment(CharUnits A) { Addr.setAlignment(A); }
 
   LValueBaseInfo getBaseInfo() const { return BaseInfo; }
   void setBaseInfo(LValueBaseInfo Info) { BaseInfo = Info; }
@@ -345,28 +359,32 @@ public:
   // simple lvalue
   llvm::Value *getPointer(CodeGenFunction &CGF) const {
     assert(isSimple());
-    return V;
+    return Addr.getBasePointer();
   }
-  Address getAddress(CodeGenFunction &CGF) const {
-    return Address(getPointer(CGF), ElementType, getAlignment(),
-                   isKnownNonNull());
-  }
-  void setAddress(Address address) {
+  llvm::Value *emitRawPointer(CodeGenFunction &CGF) const {
     assert(isSimple());
-    V = address.getPointer();
-    ElementType = address.getElementType();
-    Alignment = address.getAlignment().getQuantity();
-    IsKnownNonNull = address.isKnownNonNull();
+    return Addr.isValid() ? Addr.emitRawPointer(CGF) : nullptr;
   }
 
+  Address getAddress(CodeGenFunction &CGF) const {
+    // FIXME: remove parameter.
+    return Addr;
+  }
+
+  void setAddress(Address address) { Addr = address; }
+
   // vector elt lvalue
   Address getVectorAddress() const {
-    return Address(getVectorPointer(), ElementType, getAlignment(),
-                   (KnownNonNull_t)isKnownNonNull());
+    assert(isVectorElt());
+    return Addr;
+  }
+  llvm::Value *getRawVectorPointer(CodeGenFunction &CGF) const {
+    assert(isVectorElt());
+    return Addr.emitRawPointer(CGF);
   }
   llvm::Value *getVectorPointer() const {
     assert(isVectorElt());
-    return V;
+    return Addr.getBasePointer();
   }
   llvm::Value *getVectorIdx() const {
     assert(isVectorElt());
@@ -374,12 +392,12 @@ public:
   }
 
   Address getMatrixAddress() const {
-    return Address(getMatrixPointer(), ElementType, getAlignment(),
-                   (KnownNonNull_t)isKnownNonNull());
+    assert(isMatrixElt());
+    return Addr;
   }
   llvm::Value *getMatrixPointer() const {
     assert(isMatrixElt());
-    return V;
+    return Addr.getBasePointer();
   }
   llvm::Value *getMatrixIdx() const {
     assert(isMatrixElt());
@@ -388,12 +406,12 @@ public:
 
   // extended vector elements.
   Address getExtVectorAddress() const {
-    return Address(getExtVectorPointer(), ElementType, getAlignment(),
-                   (KnownNonNull_t)isKnownNonNull());
+    assert(isExtVectorElt());
+    return Addr;
   }
-  llvm::Value *getExtVectorPointer() const {
+  llvm::Value *getRawExtVectorPointer(CodeGenFunction &CGF) const {
     assert(isExtVectorElt());
-    return V;
+    return Addr.emitRawPointer(CGF);
   }
   llvm::Constant *getExtVectorElts() const {
     assert(isExtVectorElt());
@@ -402,10 +420,14 @@ public:
 
   // bitfield lvalue
   Address getBitFieldAddress() const {
-    return Address(getBitFieldPointer(), ElementType, getAlignment(),
-                   (KnownNonNull_t)isKnownNonNull());
+    assert(isBitField());
+    return Addr;
+  }
+  llvm::Value *getRawBitFieldPointer(CodeGenFunction &CGF) const {
+    assert(isBitField());
+    return Addr.emitRawPointer(CGF);
   }
-  llvm::Value *getBitFieldPointer() const { assert(isBitField()); return V; }
+
   const CGBitFieldInfo &getBitFieldInfo() const {
     assert(isBitField());
     return *BitFieldInfo;
@@ -414,18 +436,13 @@ public:
   // global register lvalue
   llvm::Value *getGlobalReg() const { assert(isGlobalReg()); return V; }
 
-  static LValue MakeAddr(Address address, QualType type, ASTContext &Context,
+  static LValue MakeAddr(Address Addr, QualType type, ASTContext &Context,
                          LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) {
-    Qualifiers qs = type.getQualifiers();
-    qs.setObjCGCAttr(Context.getObjCGCAttrKind(type));
-
     LValue R;
     R.LVType = Simple;
-    assert(address.getPointer()->getType()->isPointerTy());
-    R.V = address.getPointer();
-    R.ElementType = address.getElementType();
-    R.IsKnownNonNull = address.isKnownNonNull();
-    R.Initialize(type, qs, address.getAlignment(), BaseInfo, TBAAInfo);
+    R.initializeSimpleLValue(Addr, type, BaseInfo, TBAAInfo, Context);
+    R.Addr = Addr;
+    assert(Addr.getType()->isPointerTy());
     return R;
   }
 
@@ -434,26 +451,18 @@ public:
                               TBAAAccessInfo TBAAInfo) {
     LValue R;
     R.LVType = VectorElt;
-    R.V = vecAddress.getPointer();
-    R.ElementType = vecAddress.getElementType();
     R.VectorIdx = Idx;
-    R.IsKnownNonNull = vecAddress.isKnownNonNull();
-    R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(),
-                 BaseInfo, TBAAInfo);
+    R.Initialize(type, type.getQualifiers(), vecAddress, BaseInfo, TBAAInfo);
     return R;
   }
 
-  static LValue MakeExtVectorElt(Address vecAddress, llvm::Constant *Elts,
+  static LValue MakeExtVectorElt(Address Addr, llvm::Constant *Elts,
                                  QualType type, LValueBaseInfo BaseInfo,
                                  TBAAAccessInfo TBAAInfo) {
     LValue R;
     R.LVType = ExtVectorElt;
-    R.V = vecAddress.getPointer();
-    R.ElementType = vecAddress.getElementType();
     R.VectorElts = Elts;
-    R.IsKnownNonNull = vecAddress.isKnownNonNull();
-    R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(),
-                 BaseInfo, TBAAInfo);
+    R.Initialize(type, type.getQualifiers(), Addr, BaseInfo, TBAAInfo);
     return R;
   }
 
@@ -468,12 +477,8 @@ public:
                              TBAAAccessInfo TBAAInfo) {
     LValue R;
     R.LVType = BitField;
-    R.V = Addr.getPointer();
-    R.ElementType = Addr.getElementType();
     R.BitFieldInfo = &Info;
-    R.IsKnownNonNull = Addr.isKnownNonNull();
-    R.Initialize(type, type.getQualifiers(), Addr.getAlignment(), BaseInfo,
-                 TBAAInfo);
+    R.Initialize(type, type.getQualifiers(), Addr, BaseInfo, TBAAInfo);
     return R;
   }
 
@@ -481,11 +486,9 @@ public:
                               QualType type) {
     LValue R;
     R.LVType = GlobalReg;
-    R.V = V;
-    R.ElementType = nullptr;
-    R.IsKnownNonNull = true;
-    R.Initialize(type, type.getQualifiers(), alignment,
+    R.Initialize(type, type.getQualifiers(), Address::invalid(),
                  LValueBaseInfo(AlignmentSource::Decl), TBAAAccessInfo());
+    R.V = V;
     return R;
   }
 
@@ -494,12 +497,8 @@ public:
                               TBAAAccessInfo TBAAInfo) {
     LValue R;
     R.LVType = MatrixElt;
-    R.V = matAddress.getPointer();
-    R.ElementType = matAddress.getElementType();
     R.VectorIdx = Idx;
-    R.IsKnownNonNull = matAddress.isKnownNonNull();
-    R.Initialize(type, type.getQualifiers(), matAddress.getAlignment(),
-                 BaseInfo, TBAAInfo);
+    R.Initialize(type, type.getQualifiers(), matAddress, BaseInfo, TBAAInfo);
     return R;
   }
 
@@ -643,17 +642,17 @@ public:
     return NeedsGCBarriers_t(ObjCGCFlag);
   }
 
-  llvm::Value *getPointer() const {
-    return Addr.getPointer();
+  llvm::Value *getPointer(QualType PointeeTy, CodeGenFunction &CGF) const;
+
+  llvm::Value *emitRawPointer(CodeGenFunction &CGF) const {
+    return Addr.isValid() ? Addr.emitRawPointer(CGF) : nullptr;
   }
 
   Address getAddress() const {
     return Addr;
   }
 
-  bool isIgnored() const {
-    return !Addr.isValid();
-  }
+  bool isIgnored() const { return !Addr.isValid(); }
 
   CharUnits getAlignment() const {
     return Addr.getAlignment();
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index fad26c43da3d..0ceac7db89df 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -193,26 +193,35 @@ CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() {
   CGF.Builder.setDefaultConstrainedRounding(OldRounding);
 }
 
-LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) {
+static LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
+                                         bool ForPointeeType,
+                                         CodeGenFunction &CGF) {
   LValueBaseInfo BaseInfo;
   TBAAAccessInfo TBAAInfo;
-  CharUnits Alignment = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo);
-  Address Addr(V, ConvertTypeForMem(T), Alignment);
-  return LValue::MakeAddr(Addr, T, getContext(), BaseInfo, TBAAInfo);
+  CharUnits Alignment =
+      CGF.CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo, ForPointeeType);
+  Address Addr = Address(V, CGF.ConvertTypeForMem(T), Alignment);
+  return CGF.MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
+}
+
+LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) {
+  return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false, *this);
 }
 
-/// Given a value of type T* that may not be to a complete object,
-/// construct an l-value with the natural pointee alignment of T.
 LValue
 CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {
-  LValueBaseInfo BaseInfo;
-  TBAAAccessInfo TBAAInfo;
-  CharUnits Align = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo,
-                                                /* forPointeeType= */ true);
-  Address Addr(V, ConvertTypeForMem(T), Align);
-  return MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
+  return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true, *this);
+}
+
+LValue CodeGenFunction::MakeNaturalAlignRawAddrLValue(llvm::Value *V,
+                                                      QualType T) {
+  return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false, *this);
 }
 
+LValue CodeGenFunction::MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V,
+                                                             QualType T) {
+  return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true, *this);
+}
 
 llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {
   return CGM.getTypes().ConvertTypeForMem(T);
@@ -525,7 +534,8 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
     ReturnBlock.getBlock()->eraseFromParent();
   }
   if (ReturnValue.isValid()) {
-    auto *RetAlloca = dyn_cast<llvm::AllocaInst>(ReturnValue.getPointer());
+    auto *RetAlloca =
+        dyn_cast<llvm::AllocaInst>(ReturnValue.emitRawPointer(*this));
     if (RetAlloca && RetAlloca->use_empty()) {
       RetAlloca->eraseFromParent();
       ReturnValue = Address::invalid();
@@ -1122,13 +1132,14 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
     auto AI = CurFn->arg_begin();
     if (CurFnInfo->getReturnInfo().isSRetAfterThis())
       ++AI;
-    ReturnValue =
-        Address(&*AI, ConvertType(RetTy),
-                CurFnInfo->getReturnInfo().getIndirectAlign(), KnownNonNull);
+    ReturnValue = makeNaturalAddressForPointer(
+        &*AI, RetTy, CurFnInfo->getReturnInfo().getIndirectAlign(), false,
+        nullptr, nullptr, KnownNonNull);
     if (!CurFnInfo->getReturnInfo().getIndirectByVal()) {
-      ReturnValuePointer = CreateDefaultAlignTempAlloca(
-          ReturnValue.getPointer()->getType(), "result.ptr");
-      Builder.CreateStore(ReturnValue.getPointer(), ReturnValuePointer);
+      ReturnValuePointer =
+          CreateDefaultAlignTempAlloca(ReturnValue.getType(), "result.ptr");
+      Builder.CreateStore(ReturnValue.emitRawPointer(*this),
+                          ReturnValuePointer);
     }
   } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca &&
              !hasScalarEvaluationKind(CurFnInfo->getReturnType())) {
@@ -1189,8 +1200,9 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
         // or contains the address of the enclosing object).
         LValue ThisFieldLValue = EmitLValueForLambdaField(LambdaThisCaptureField);
         if (!LambdaThisCaptureField->getType()->isPointerType()) {
-          // If the enclosing object was captured by value, just use its address.
-          CXXThisValue = ThisFieldLValue.getAddress(*this).getPointer();
+          // If the enclosing object was captured by value, just use its
+          // address. Sign this pointer.
+          CXXThisValue = ThisFieldLValue.getPointer(*this);
         } else {
           // Load the lvalue pointed to by the field, since '*this' was captured
           // by reference.
@@ -2012,8 +2024,9 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
     = llvm::ConstantInt::get(CGF.IntPtrTy, baseSize.getQuantity());
 
   Address begin = dest.withElementType(CGF.Int8Ty);
-  llvm::Value *end = Builder.CreateInBoundsGEP(
-      begin.getElementType(), begin.getPointer(), sizeInChars, "vla.end");
+  llvm::Value *end = Builder.CreateInBoundsGEP(begin.getElementType(),
+                                               begin.emitRawPointer(CGF),
+                                               sizeInChars, "vla.end");
 
   llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock();
   llvm::BasicBlock *loopBB = CGF.createBasicBlock("vla-init.loop");
@@ -2024,7 +2037,7 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
   CGF.EmitBlock(loopBB);
 
   llvm::PHINode *cur = Builder.CreatePHI(begin.getType(), 2, "vla.cur");
-  cur->addIncoming(begin.getPointer(), originBB);
+  cur->addIncoming(begin.emitRawPointer(CGF), originBB);
 
   CharUnits curAlign =
     dest.getAlignment().alignmentOfArrayElement(baseSize);
@@ -2218,10 +2231,10 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
     addr = addr.withElementType(baseType);
   } else {
     // Create the actual GEP.
-    addr = Address(Builder.CreateInBoundsGEP(
-        addr.getElementType(), addr.getPointer(), gepIndices, "array.begin"),
-        ConvertTypeForMem(eltType),
-        addr.getAlignment());
+    addr = Address(Builder.CreateInBoundsGEP(addr.getElementType(),
+                                             addr.emitRawPointer(*this),
+                                             gepIndices, "array.begin"),
+                   ConvertTypeForMem(eltType), addr.getAlignment());
   }
 
   baseType = eltType;
@@ -2562,7 +2575,7 @@ void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) {
 Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
                                               Address Addr) {
   assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
-  llvm::Value *V = Addr.getPointer();
+  llvm::Value *V = Addr.emitRawPointer(*this);
   llvm::Type *VTy = V->getType();
   auto *PTy = dyn_cast<llvm::PointerType>(VTy);
   unsigned AS = PTy ? PTy->getAddressSpace() : 0;
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index e8f8aa601ed0..8dd6da5f85f1 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -151,6 +151,9 @@ struct DominatingLLVMValue {
 
   /// Answer whether the given value needs extra work to be saved.
   static bool needsSaving(llvm::Value *value) {
+    if (!value)
+      return false;
+
     // If it's not an instruction, we don't need to save.
     if (!isa<llvm::Instruction>(value)) return false;
 
@@ -177,21 +180,28 @@ template <> struct DominatingValue<Address> {
   typedef Address type;
 
   struct saved_type {
-    DominatingLLVMValue::saved_type SavedValue;
+    DominatingLLVMValue::saved_type BasePtr;
     llvm::Type *ElementType;
     CharUnits Alignment;
+    DominatingLLVMValue::saved_type Offset;
+    llvm::PointerType *EffectiveType;
   };
 
   static bool needsSaving(type value) {
-    return DominatingLLVMValue::needsSaving(value.getPointer());
+    if (DominatingLLVMValue::needsSaving(value.getBasePointer()) ||
+        DominatingLLVMValue::needsSaving(value.getOffset()))
+      return true;
+    return false;
   }
   static saved_type save(CodeGenFunction &CGF, type value) {
-    return { DominatingLLVMValue::save(CGF, value.getPointer()),
-             value.getElementType(), value.getAlignment() };
+    return {DominatingLLVMValue::save(CGF, value.getBasePointer()),
+            value.getElementType(), value.getAlignment(),
+            DominatingLLVMValue::save(CGF, value.getOffset()), value.getType()};
   }
   static type restore(CodeGenFunction &CGF, saved_type value) {
-    return Address(DominatingLLVMValue::restore(CGF, value.SavedValue),
-                   value.ElementType, value.Alignment);
+    return Address(DominatingLLVMValue::restore(CGF, value.BasePtr),
+                   value.ElementType, value.Alignment,
+                   DominatingLLVMValue::restore(CGF, value.Offset));
   }
 };
 
@@ -201,14 +211,26 @@ template <> struct DominatingValue<RValue> {
   class saved_type {
     enum Kind { ScalarLiteral, ScalarAddress, AggregateLiteral,
                 AggregateAddress, ComplexAddress };
-
-    llvm::Value *Value;
-    llvm::Type *ElementType;
+    union {
+      struct {
+        DominatingLLVMValue::saved_type first, second;
+      } Vals;
+      DominatingValue<Address>::saved_type AggregateAddr;
+    };
     LLVM_PREFERRED_TYPE(Kind)
     unsigned K : 3;
-    unsigned Align : 29;
-    saved_type(llvm::Value *v, llvm::Type *e, Kind k, unsigned a = 0)
-      : Value(v), ElementType(e), K(k), Align(a) {}
+    unsigned IsVolatile : 1;
+
+    saved_type(DominatingLLVMValue::saved_type Val1, unsigned K)
+        : Vals{Val1, DominatingLLVMValue::saved_type()}, K(K) {}
+
+    saved_type(DominatingLLVMValue::saved_type Val1,
+               DominatingLLVMValue::saved_type Val2)
+        : Vals{Val1, Val2}, K(ComplexAddress) {}
+
+    saved_type(DominatingValue<Address>::saved_type AggregateAddr,
+               bool IsVolatile, unsigned K)
+        : AggregateAddr(AggregateAddr), K(K) {}
 
   public:
     static bool needsSaving(RValue value);
@@ -659,7 +681,7 @@ public:
     llvm::Value *Size;
 
   public:
-    CallLifetimeEnd(Address addr, llvm::Value *size)
+    CallLifetimeEnd(RawAddress addr, llvm::Value *size)
         : Addr(addr.getPointer()), Size(size) {}
 
     void Emit(CodeGenFunction &CGF, Flags flags) override {
@@ -684,7 +706,7 @@ public:
   };
 
   /// i32s containing the indexes of the cleanup destinations.
-  Address NormalCleanupDest = Address::invalid();
+  RawAddress NormalCleanupDest = RawAddress::invalid();
 
   unsigned NextCleanupDestIndex = 1;
 
@@ -819,10 +841,10 @@ public:
   template <class T, class... As>
   void pushCleanupAfterFullExpr(CleanupKind Kind, As... A) {
     if (!isInConditionalBranch())
-      return pushCleanupAfterFullExprWithActiveFlag<T>(Kind, Address::invalid(),
-                                                       A...);
+      return pushCleanupAfterFullExprWithActiveFlag<T>(
+          Kind, RawAddress::invalid(), A...);
 
-    Address ActiveFlag = createCleanupActiveFlag();
+    RawAddress ActiveFlag = createCleanupActiveFlag();
     assert(!DominatingValue<Address>::needsSaving(ActiveFlag) &&
            "cleanup active flag should never need saving");
 
@@ -835,7 +857,7 @@ public:
 
   template <class T, class... As>
   void pushCleanupAfterFullExprWithActiveFlag(CleanupKind Kind,
-                                              Address ActiveFlag, As... A) {
+                                              RawAddress ActiveFlag, As... A) {
     LifetimeExtendedCleanupHeader Header = {sizeof(T), Kind,
                                             ActiveFlag.isValid()};
 
@@ -850,7 +872,7 @@ public:
     new (Buffer) LifetimeExtendedCleanupHeader(Header);
     new (Buffer + sizeof(Header)) T(A...);
     if (Header.IsConditional)
-      new (Buffer + sizeof(Header) + sizeof(T)) Address(ActiveFlag);
+      new (Buffer + sizeof(Header) + sizeof(T)) RawAddress(ActiveFlag);
   }
 
   /// Set up the last cleanup that was pushed as a conditional
@@ -859,8 +881,8 @@ public:
     initFullExprCleanupWithFlag(createCleanupActiveFlag());
   }
 
-  void initFullExprCleanupWithFlag(Address ActiveFlag);
-  Address createCleanupActiveFlag();
+  void initFullExprCleanupWithFlag(RawAddress ActiveFlag);
+  RawAddress createCleanupActiveFlag();
 
   /// PushDestructorCleanup - Push a cleanup to call the
   /// complete-object destructor of an object of the given type at the
@@ -1048,7 +1070,7 @@ public:
       QualType VarTy = LocalVD->getType();
       if (VarTy->isReferenceType()) {
         Address Temp = CGF.CreateMemTemp(VarTy);
-        CGF.Builder.CreateStore(TempAddr.getPointer(), Temp);
+        CGF.Builder.CreateStore(TempAddr.emitRawPointer(CGF), Temp);
         TempAddr = Temp;
       }
       SavedTempAddresses.try_emplace(LocalVD, TempAddr);
@@ -1243,10 +1265,12 @@ public:
   /// one branch or the other of a conditional expression.
   bool isInConditionalBranch() const { return OutermostConditional != nullptr; }
 
-  void setBeforeOutermostConditional(llvm::Value *value, Address addr) {
+  void setBeforeOutermostConditional(llvm::Value *value, Address addr,
+                                     CodeGenFunction &CGF) {
     assert(isInConditionalBranch());
     llvm::BasicBlock *block = OutermostConditional->getStartingBlock();
-    auto store = new llvm::StoreInst(value, addr.getPointer(), &block->back());
+    auto store =
+        new llvm::StoreInst(value, addr.emitRawPointer(CGF), &block->back());
     store->setAlignment(addr.getAlignment().getAsAlign());
   }
 
@@ -1601,7 +1625,7 @@ public:
   /// If \p StepV is null, the default increment is 1.
   void maybeUpdateMCDCTestVectorBitmap(const Expr *E) {
     if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) {
-      PGO.emitMCDCTestVectorBitmapUpdate(Builder, E, MCDCCondBitmapAddr);
+      PGO.emitMCDCTestVectorBitmapUpdate(Builder, E, MCDCCondBitmapAddr, *this);
       PGO.setCurrentStmt(E);
     }
   }
@@ -1609,7 +1633,7 @@ public:
   /// Update the MCDC temp value with the condition's evaluated result.
   void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val) {
     if (isMCDCCoverageEnabled()) {
-      PGO.emitMCDCCondBitmapUpdate(Builder, E, MCDCCondBitmapAddr, Val);
+      PGO.emitMCDCCondBitmapUpdate(Builder, E, MCDCCondBitmapAddr, Val, *this);
       PGO.setCurrentStmt(E);
     }
   }
@@ -1704,7 +1728,7 @@ public:
         : CGF(CGF), OldCXXThisValue(CGF.CXXThisValue),
           OldCXXThisAlignment(CGF.CXXThisAlignment),
           SourceLocScope(E, CGF.CurSourceLocExprScope) {
-      CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getPointer();
+      CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getBasePointer();
       CGF.CXXThisAlignment = CGF.CXXDefaultInitExprThis.getAlignment();
     }
     ~CXXDefaultInitExprScope() {
@@ -2090,7 +2114,7 @@ public:
   llvm::Value *getExceptionFromSlot();
   llvm::Value *getSelectorFromSlot();
 
-  Address getNormalCleanupDestSlot();
+  RawAddress getNormalCleanupDestSlot();
 
   llvm::BasicBlock *getUnreachableBlock() {
     if (!UnreachableBlock) {
@@ -2579,10 +2603,40 @@ public:
   //                                  Helpers
   //===--------------------------------------------------------------------===//
 
+  Address mergeAddressesInConditionalExpr(Address LHS, Address RHS,
+                                          llvm::BasicBlock *LHSBlock,
+                                          llvm::BasicBlock *RHSBlock,
+                                          llvm::BasicBlock *MergeBlock,
+                                          QualType MergedType) {
+    Builder.SetInsertPoint(MergeBlock);
+    llvm::PHINode *PtrPhi = Builder.CreatePHI(LHS.getType(), 2, "cond");
+    PtrPhi->addIncoming(LHS.getBasePointer(), LHSBlock);
+    PtrPhi->addIncoming(RHS.getBasePointer(), RHSBlock);
+    LHS.replaceBasePointer(PtrPhi);
+    LHS.setAlignment(std::min(LHS.getAlignment(), RHS.getAlignment()));
+    return LHS;
+  }
+
+  /// Construct an address with the natural alignment of T. If a pointer to T
+  /// is expected to be signed, the pointer passed to this function must have
+  /// been signed, and the returned Address will have the pointer authentication
+  /// information needed to authenticate the signed pointer.
+  Address makeNaturalAddressForPointer(
+      llvm::Value *Ptr, QualType T, CharUnits Alignment = CharUnits::Zero(),
+      bool ForPointeeType = false, LValueBaseInfo *BaseInfo = nullptr,
+      TBAAAccessInfo *TBAAInfo = nullptr,
+      KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {
+    if (Alignment.isZero())
+      Alignment =
+          CGM.getNaturalTypeAlignment(T, BaseInfo, TBAAInfo, ForPointeeType);
+    return Address(Ptr, ConvertTypeForMem(T), Alignment, nullptr,
+                   IsKnownNonNull);
+  }
+
   LValue MakeAddrLValue(Address Addr, QualType T,
                         AlignmentSource Source = AlignmentSource::Type) {
-    return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source),
-                            CGM.getTBAAAccessInfo(T));
+    return MakeAddrLValue(Addr, T, LValueBaseInfo(Source),
+                          CGM.getTBAAAccessInfo(T));
   }
 
   LValue MakeAddrLValue(Address Addr, QualType T, LValueBaseInfo BaseInfo,
@@ -2592,6 +2646,14 @@ public:
 
   LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
                         AlignmentSource Source = AlignmentSource::Type) {
+    return MakeAddrLValue(makeNaturalAddressForPointer(V, T, Alignment), T,
+                          LValueBaseInfo(Source), CGM.getTBAAAccessInfo(T));
+  }
+
+  /// Same as MakeAddrLValue above except that the pointer is known to be
+  /// unsigned.
+  LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
+                           AlignmentSource Source = AlignmentSource::Type) {
     Address Addr(V, ConvertTypeForMem(T), Alignment);
     return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source),
                             CGM.getTBAAAccessInfo(T));
@@ -2604,9 +2666,18 @@ public:
                             TBAAAccessInfo());
   }
 
+  /// Given a value of type T* that may not be to a complete object, construct
+  /// an l-value with the natural pointee alignment of T.
   LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T);
+
   LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T);
 
+  /// Same as MakeNaturalAlignPointeeAddrLValue except that the pointer is known
+  /// to be unsigned.
+  LValue MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, QualType T);
+
+  LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T);
+
   Address EmitLoadOfReference(LValue RefLVal,
                               LValueBaseInfo *PointeeBaseInfo = nullptr,
                               TBAAAccessInfo *PointeeTBAAInfo = nullptr);
@@ -2655,13 +2726,13 @@ public:
   /// more efficient if the caller knows that the address will not be exposed.
   llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty, const Twine &Name = "tmp",
                                      llvm::Value *ArraySize = nullptr);
-  Address CreateTempAlloca(llvm::Type *Ty, CharUnits align,
-                           const Twine &Name = "tmp",
-                           llvm::Value *ArraySize = nullptr,
-                           Address *Alloca = nullptr);
-  Address CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align,
-                                      const Twine &Name = "tmp",
-                                      llvm::Value *ArraySize = nullptr);
+  RawAddress CreateTempAlloca(llvm::Type *Ty, CharUnits align,
+                              const Twine &Name = "tmp",
+                              llvm::Value *ArraySize = nullptr,
+                              RawAddress *Alloca = nullptr);
+  RawAddress CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align,
+                                         const Twine &Name = "tmp",
+                                         llvm::Value *ArraySize = nullptr);
 
   /// CreateDefaultAlignedTempAlloca - This creates an alloca with the
   /// default ABI alignment of the given LLVM type.
@@ -2673,8 +2744,8 @@ public:
   /// not hand this address off to arbitrary IRGen routines, and especially
   /// do not pass it as an argument to a function that might expect a
   /// properly ABI-aligned value.
-  Address CreateDefaultAlignTempAlloca(llvm::Type *Ty,
-                                       const Twine &Name = "tmp");
+  RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty,
+                                          const Twine &Name = "tmp");
 
   /// CreateIRTemp - Create a temporary IR object of the given type, with
   /// appropriate alignment. This routine should only be used when an temporary
@@ -2684,32 +2755,31 @@ public:
   ///
   /// That is, this is exactly equivalent to CreateMemTemp, but calling
   /// ConvertType instead of ConvertTypeForMem.
-  Address CreateIRTemp(QualType T, const Twine &Name = "tmp");
+  RawAddress CreateIRTemp(QualType T, const Twine &Name = "tmp");
 
   /// CreateMemTemp - Create a temporary memory object of the given type, with
   /// appropriate alignmen and cast it to the default address space. Returns
   /// the original alloca instruction by \p Alloca if it is not nullptr.
-  Address CreateMemTemp(QualType T, const Twine &Name = "tmp",
-                        Address *Alloca = nullptr);
-  Address CreateMemTemp(QualType T, CharUnits Align, const Twine &Name = "tmp",
-                        Address *Alloca = nullptr);
+  RawAddress CreateMemTemp(QualType T, const Twine &Name = "tmp",
+                           RawAddress *Alloca = nullptr);
+  RawAddress CreateMemTemp(QualType T, CharUnits Align,
+                           const Twine &Name = "tmp",
+                           RawAddress *Alloca = nullptr);
 
   /// CreateMemTemp - Create a temporary memory object of the given type, with
   /// appropriate alignmen without casting it to the default address space.
-  Address CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp");
-  Address CreateMemTempWithoutCast(QualType T, CharUnits Align,
-                                   const Twine &Name = "tmp");
+  RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp");
+  RawAddress CreateMemTempWithoutCast(QualType T, CharUnits Align,
+                                      const Twine &Name = "tmp");
 
   /// CreateAggTemp - Create a temporary memory object for the given
   /// aggregate type.
   AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp",
-                             Address *Alloca = nullptr) {
-    return AggValueSlot::forAddr(CreateMemTemp(T, Name, Alloca),
-                                 T.getQualifiers(),
-                                 AggValueSlot::IsNotDestructed,
-                                 AggValueSlot::DoesNotNeedGCBarriers,
-                                 AggValueSlot::IsNotAliased,
-                                 AggValueSlot::DoesNotOverlap);
+                             RawAddress *Alloca = nullptr) {
+    return AggValueSlot::forAddr(
+        CreateMemTemp(T, Name, Alloca), T.getQualifiers(),
+        AggValueSlot::IsNotDestructed, AggValueSlot::DoesNotNeedGCBarriers,
+        AggValueSlot::IsNotAliased, AggValueSlot::DoesNotOverlap);
   }
 
   /// EvaluateExprAsBool - Perform the usual unary conversions on the specified
@@ -3083,6 +3153,25 @@ public:
   /// calls to EmitTypeCheck can be skipped.
   bool sanitizePerformTypeCheck() const;
 
+  void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV,
+                     QualType Type, SanitizerSet SkippedChecks = SanitizerSet(),
+                     llvm::Value *ArraySize = nullptr) {
+    if (!sanitizePerformTypeCheck())
+      return;
+    EmitTypeCheck(TCK, Loc, LV.emitRawPointer(*this), Type, LV.getAlignment(),
+                  SkippedChecks, ArraySize);
+  }
+
+  void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, Address Addr,
+                     QualType Type, CharUnits Alignment = CharUnits::Zero(),
+                     SanitizerSet SkippedChecks = SanitizerSet(),
+                     llvm::Value *ArraySize = nullptr) {
+    if (!sanitizePerformTypeCheck())
+      return;
+    EmitTypeCheck(TCK, Loc, Addr.emitRawPointer(*this), Type, Alignment,
+                  SkippedChecks, ArraySize);
+  }
+
   /// Emit a check that \p V is the address of storage of the
   /// appropriate size and alignment for an object of type \p Type
   /// (or if ArraySize is provided, for an array of that bound).
@@ -3183,17 +3272,17 @@ public:
 
     /// Address with original alloca instruction. Invalid if the variable was
     /// emitted as a global constant.
-    Address AllocaAddr;
+    RawAddress AllocaAddr;
 
     struct Invalid {};
     AutoVarEmission(Invalid)
         : Variable(nullptr), Addr(Address::invalid()),
-          AllocaAddr(Address::invalid()) {}
+          AllocaAddr(RawAddress::invalid()) {}
 
     AutoVarEmission(const VarDecl &variable)
         : Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr),
           IsEscapingByRef(false), IsConstantAggregate(false),
-          SizeForLifetimeMarkers(nullptr), AllocaAddr(Address::invalid()) {}
+          SizeForLifetimeMarkers(nullptr), AllocaAddr(RawAddress::invalid()) {}
 
     bool wasEmittedAsGlobal() const { return !Addr.isValid(); }
 
@@ -3216,7 +3305,7 @@ public:
     }
 
     /// Returns the address for the original alloca instruction.
-    Address getOriginalAllocatedAddress() const { return AllocaAddr; }
+    RawAddress getOriginalAllocatedAddress() const { return AllocaAddr; }
 
     /// Returns the address of the object within this declaration.
     /// Note that this does not chase the forwarding pointer for
@@ -3246,23 +3335,32 @@ public:
                          llvm::GlobalValue::LinkageTypes Linkage);
 
   class ParamValue {
-    llvm::Value *Value;
-    llvm::Type *ElementType;
-    unsigned Alignment;
-    ParamValue(llvm::Value *V, llvm::Type *T, unsigned A)
-        : Value(V), ElementType(T), Alignment(A) {}
+    union {
+      Address Addr;
+      llvm::Value *Value;
+    };
+
+    bool IsIndirect;
+
+    ParamValue(llvm::Value *V) : Value(V), IsIndirect(false) {}
+    ParamValue(Address A) : Addr(A), IsIndirect(true) {}
+
   public:
     static ParamValue forDirect(llvm::Value *value) {
-      return ParamValue(value, nullptr, 0);
+      return ParamValue(value);
     }
     static ParamValue forIndirect(Address addr) {
       assert(!addr.getAlignment().isZero());
-      return ParamValue(addr.getPointer(), addr.getElementType(),
-                        addr.getAlignment().getQuantity());
+      return ParamValue(addr);
     }
 
-    bool isIndirect() const { return Alignment != 0; }
-    llvm::Value *getAnyValue() const { return Value; }
+    bool isIndirect() const { return IsIndirect; }
+    llvm::Value *getAnyValue() const {
+      if (!isIndirect())
+        return Value;
+      assert(!Addr.hasOffset() && "unexpected offset");
+      return Addr.getBasePointer();
+    }
 
     llvm::Value *getDirectValue() const {
       assert(!isIndirect());
@@ -3271,8 +3369,7 @@ public:
 
     Address getIndirectAddress() const {
       assert(isIndirect());
-      return Address(Value, ElementType, CharUnits::fromQuantity(Alignment),
-                     KnownNonNull);
+      return Addr;
     }
   };
 
@@ -4182,6 +4279,9 @@ public:
                                   const Twine &name = "");
   llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
                                           const Twine &name = "");
+  llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
+                                          ArrayRef<Address> args,
+                                          const Twine &name = "");
   llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
                                           ArrayRef<llvm::Value *> args,
                                           const Twine &name = "");
@@ -4208,6 +4308,12 @@ public:
                                                CXXDtorType Type,
                                                const CXXRecordDecl *RD);
 
+  llvm::Value *getAsNaturalPointerTo(Address Addr, QualType PointeeType) {
+    return Addr.getBasePointer();
+  }
+
+  bool isPointerKnownNonNull(const Expr *E);
+
   // Return the copy constructor name with the prefix "__copy_constructor_"
   // removed.
   static std::string getNonTrivialCopyConstructorStr(QualType QT,
@@ -4780,6 +4886,11 @@ public:
                                       SourceLocation Loc,
                                       const Twine &Name = "");
 
+  Address EmitCheckedInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList,
+                                 llvm::Type *elementType, bool SignedIndices,
+                                 bool IsSubtraction, SourceLocation Loc,
+                                 CharUnits Align, const Twine &Name = "");
+
   /// Specifies which type of sanitizer check to apply when handling a
   /// particular builtin.
   enum BuiltinCheckKind {
@@ -4842,6 +4953,10 @@ public:
   void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc,
                            AbstractCallee AC, unsigned ParmNum);
 
+  void EmitNonNullArgCheck(Address Addr, QualType ArgType,
+                           SourceLocation ArgLoc, AbstractCallee AC,
+                           unsigned ParmNum);
+
   /// EmitCallArg - Emit a single call argument.
   void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType);
 
@@ -5050,7 +5165,7 @@ DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) {
       CGF.CreateTempAlloca(value->getType(), align, "cond-cleanup.save");
   CGF.Builder.CreateStore(value, alloca);
 
-  return saved_type(alloca.getPointer(), true);
+  return saved_type(alloca.emitRawPointer(CGF), true);
 }
 
 inline llvm::Value *DominatingLLVMValue::restore(CodeGenFunction &CGF,
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index ac81df8cf7ad..da411a9fa579 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -7272,7 +7272,7 @@ void CodeGenFunction::EmitDeclMetadata() {
 
   for (auto &I : LocalDeclMap) {
     const Decl *D = I.first;
-    llvm::Value *Addr = I.second.getPointer();
+    llvm::Value *Addr = I.second.emitRawPointer(*this);
     if (auto *Alloca = dyn_cast<llvm::AllocaInst>(Addr)) {
       llvm::Value *DAddr = GetPointerConstant(getLLVMContext(), D);
       Alloca->setMetadata(
diff --git a/clang/lib/CodeGen/CodeGenPGO.cpp b/clang/lib/CodeGen/CodeGenPGO.cpp
index 2619edfeb7dc..76704c4d7be4 100644
--- a/clang/lib/CodeGen/CodeGenPGO.cpp
+++ b/clang/lib/CodeGen/CodeGenPGO.cpp
@@ -1239,7 +1239,8 @@ void CodeGenPGO::emitMCDCParameters(CGBuilderTy &Builder) {
 
 void CodeGenPGO::emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder,
                                                 const Expr *S,
-                                                Address MCDCCondBitmapAddr) {
+                                                Address MCDCCondBitmapAddr,
+                                                CodeGenFunction &CGF) {
   if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState)
     return;
 
@@ -1262,7 +1263,7 @@ void CodeGenPGO::emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder,
                           Builder.getInt64(FunctionHash),
                           Builder.getInt32(RegionMCDCState->BitmapBytes),
                           Builder.getInt32(MCDCTestVectorBitmapOffset),
-                          MCDCCondBitmapAddr.getPointer()};
+                          MCDCCondBitmapAddr.emitRawPointer(CGF)};
   Builder.CreateCall(
       CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_tvbitmap_update), Args);
 }
@@ -1283,7 +1284,8 @@ void CodeGenPGO::emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S,
 
 void CodeGenPGO::emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S,
                                           Address MCDCCondBitmapAddr,
-                                          llvm::Value *Val) {
+                                          llvm::Value *Val,
+                                          CodeGenFunction &CGF) {
   if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState)
     return;
 
@@ -1312,7 +1314,7 @@ void CodeGenPGO::emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S,
   llvm::Value *Args[5] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
                           Builder.getInt64(FunctionHash),
                           Builder.getInt32(Branch.ID),
-                          MCDCCondBitmapAddr.getPointer(), Val};
+                          MCDCCondBitmapAddr.emitRawPointer(CGF), Val};
   Builder.CreateCall(
       CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_condbitmap_update),
       Args);
diff --git a/clang/lib/CodeGen/CodeGenPGO.h b/clang/lib/CodeGen/CodeGenPGO.h
index 036fbf6815a4..9d66ffad6f43 100644
--- a/clang/lib/CodeGen/CodeGenPGO.h
+++ b/clang/lib/CodeGen/CodeGenPGO.h
@@ -113,12 +113,14 @@ public:
   void emitCounterSetOrIncrement(CGBuilderTy &Builder, const Stmt *S,
                                  llvm::Value *StepV);
   void emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, const Expr *S,
-                                      Address MCDCCondBitmapAddr);
+                                      Address MCDCCondBitmapAddr,
+                                      CodeGenFunction &CGF);
   void emitMCDCParameters(CGBuilderTy &Builder);
   void emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S,
                                Address MCDCCondBitmapAddr);
   void emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S,
-                                Address MCDCCondBitmapAddr, llvm::Value *Val);
+                                Address MCDCCondBitmapAddr, llvm::Value *Val,
+                                CodeGenFunction &CGF);
 
   /// Return the region count for the counter at the given index.
   uint64_t getRegionCount(const Stmt *S) {
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index bdd53a192f82..fd71317572f0 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -307,10 +307,6 @@ public:
       CodeGenFunction &CGF, const CXXRecordDecl *VTableClass,
       BaseSubobject Base, const CXXRecordDecl *NearestVBase);
 
-  llvm::Constant *
-  getVTableAddressPointForConstExpr(BaseSubobject Base,
-                                    const CXXRecordDecl *VTableClass) override;
-
   llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD,
                                         CharUnits VPtrOffset) override;
 
@@ -646,7 +642,7 @@ CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
 
   // Apply the adjustment and cast back to the original struct type
   // for consistency.
-  llvm::Value *This = ThisAddr.getPointer();
+  llvm::Value *This = ThisAddr.emitRawPointer(CGF);
   This = Builder.CreateInBoundsGEP(Builder.getInt8Ty(), This, Adj);
   ThisPtrForCall = This;
 
@@ -850,7 +846,7 @@ llvm::Value *ItaniumCXXABI::EmitMemberDataPointerAddress(
   CGBuilderTy &Builder = CGF.Builder;
 
   // Apply the offset, which we assume is non-null.
-  return Builder.CreateInBoundsGEP(CGF.Int8Ty, Base.getPointer(), MemPtr,
+  return Builder.CreateInBoundsGEP(CGF.Int8Ty, Base.emitRawPointer(CGF), MemPtr,
                                    "memptr.offset");
 }
 
@@ -1245,7 +1241,7 @@ void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF,
                                                         CGF.getPointerAlign());
 
     // Apply the offset.
-    llvm::Value *CompletePtr = Ptr.getPointer();
+    llvm::Value *CompletePtr = Ptr.emitRawPointer(CGF);
     CompletePtr =
         CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, CompletePtr, Offset);
 
@@ -1482,7 +1478,8 @@ llvm::Value *ItaniumCXXABI::emitDynamicCastCall(
       computeOffsetHint(CGF.getContext(), SrcDecl, DestDecl).getQuantity());
 
   // Emit the call to __dynamic_cast.
-  llvm::Value *Args[] = {ThisAddr.getPointer(), SrcRTTI, DestRTTI, OffsetHint};
+  llvm::Value *Args[] = {ThisAddr.emitRawPointer(CGF), SrcRTTI, DestRTTI,
+                         OffsetHint};
   llvm::Value *Value =
       CGF.EmitNounwindRuntimeCall(getItaniumDynamicCastFn(CGF), Args);
 
@@ -1571,7 +1568,7 @@ llvm::Value *ItaniumCXXABI::emitExactDynamicCast(
       VPtr, CGM.getTBAAVTablePtrAccessInfo(CGF.VoidPtrPtrTy));
   llvm::Value *Success = CGF.Builder.CreateICmpEQ(
       VPtr, getVTableAddressPoint(BaseSubobject(SrcDecl, *Offset), DestDecl));
-  llvm::Value *Result = ThisAddr.getPointer();
+  llvm::Value *Result = ThisAddr.emitRawPointer(CGF);
   if (!Offset->isZero())
     Result = CGF.Builder.CreateInBoundsGEP(
         CGF.CharTy, Result,
@@ -1611,7 +1608,7 @@ llvm::Value *ItaniumCXXABI::emitDynamicCastToVoid(CodeGenFunction &CGF,
         PtrDiffLTy, OffsetToTop, CGF.getPointerAlign(), "offset.to.top");
   }
   // Finally, add the offset to the pointer.
-  return CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ThisAddr.getPointer(),
+  return CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ThisAddr.emitRawPointer(CGF),
                                        OffsetToTop);
 }
 
@@ -1792,8 +1789,8 @@ void ItaniumCXXABI::EmitDestructorCall(CodeGenFunction &CGF,
   else
     Callee = CGCallee::forDirect(CGM.getAddrOfCXXStructor(GD), GD);
 
-  CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, VTT, VTTTy,
-                            nullptr);
+  CGF.EmitCXXDestructorCall(GD, Callee, CGF.getAsNaturalPointerTo(This, ThisTy),
+                            ThisTy, VTT, VTTTy, nullptr);
 }
 
 void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT,
@@ -1952,11 +1949,6 @@ llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructorWithVTT(
                                        CGF.getPointerAlign());
 }
 
-llvm::Constant *ItaniumCXXABI::getVTableAddressPointForConstExpr(
-    BaseSubobject Base, const CXXRecordDecl *VTableClass) {
-  return getVTableAddressPoint(Base, VTableClass);
-}
-
 llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD,
                                                      CharUnits VPtrOffset) {
   assert(VPtrOffset.isZero() && "Itanium ABI only supports zero vptr offsets");
@@ -2088,8 +2080,8 @@ llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall(
     ThisTy = D->getDestroyedType();
   }
 
-  CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, nullptr,
-                            QualType(), nullptr);
+  CGF.EmitCXXDestructorCall(GD, Callee, This.emitRawPointer(CGF), ThisTy,
+                            nullptr, QualType(), nullptr);
   return nullptr;
 }
 
@@ -2162,7 +2154,7 @@ static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF,
                                           int64_t VirtualAdjustment,
                                           bool IsReturnAdjustment) {
   if (!NonVirtualAdjustment && !VirtualAdjustment)
-    return InitialPtr.getPointer();
+    return InitialPtr.emitRawPointer(CGF);
 
   Address V = InitialPtr.withElementType(CGF.Int8Ty);
 
@@ -2195,10 +2187,10 @@ static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF,
                                              CGF.getPointerAlign());
     }
     // Adjust our pointer.
-    ResultPtr = CGF.Builder.CreateInBoundsGEP(
-        V.getElementType(), V.getPointer(), Offset);
+    ResultPtr = CGF.Builder.CreateInBoundsGEP(V.getElementType(),
+                                              V.emitRawPointer(CGF), Offset);
   } else {
-    ResultPtr = V.getPointer();
+    ResultPtr = V.emitRawPointer(CGF);
   }
 
   // In a derived-to-base conversion, the non-virtual adjustment is
@@ -2284,7 +2276,7 @@ Address ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
         llvm::FunctionType::get(CGM.VoidTy, NumElementsPtr.getType(), false);
     llvm::FunctionCallee F =
         CGM.CreateRuntimeFunction(FTy, "__asan_poison_cxx_array_cookie");
-    CGF.Builder.CreateCall(F, NumElementsPtr.getPointer());
+    CGF.Builder.CreateCall(F, NumElementsPtr.emitRawPointer(CGF));
   }
 
   // Finally, compute a pointer to the actual data buffer by skipping
@@ -2315,7 +2307,7 @@ llvm::Value *ItaniumCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
       llvm::FunctionType::get(CGF.SizeTy, CGF.UnqualPtrTy, false);
   llvm::FunctionCallee F =
       CGM.CreateRuntimeFunction(FTy, "__asan_load_cxx_array_cookie");
-  return CGF.Builder.CreateCall(F, numElementsPtr.getPointer());
+  return CGF.Builder.CreateCall(F, numElementsPtr.emitRawPointer(CGF));
 }
 
 CharUnits ARMCXXABI::getArrayCookieSizeImpl(QualType elementType) {
@@ -2627,7 +2619,7 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF,
 
     // Call __cxa_guard_release.  This cannot throw.
     CGF.EmitNounwindRuntimeCall(getGuardReleaseFn(CGM, guardPtrTy),
-                                guardAddr.getPointer());
+                                guardAddr.emitRawPointer(CGF));
   } else if (D.isLocalVarDecl()) {
     // For local variables, store 1 into the first byte of the guard variable
     // after the object initialization completes so that initialization is
@@ -3120,10 +3112,10 @@ LValue ItaniumCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF,
 
   LValue LV;
   if (VD->getType()->isReferenceType())
-    LV = CGF.MakeNaturalAlignAddrLValue(CallVal, LValType);
+    LV = CGF.MakeNaturalAlignRawAddrLValue(CallVal, LValType);
   else
-    LV = CGF.MakeAddrLValue(CallVal, LValType,
-                            CGF.getContext().getDeclAlign(VD));
+    LV = CGF.MakeRawAddrLValue(CallVal, LValType,
+                               CGF.getContext().getDeclAlign(VD));
   // FIXME: need setObjCGCLValueClass?
   return LV;
 }
@@ -4604,7 +4596,7 @@ static void InitCatchParam(CodeGenFunction &CGF,
         CGF.Builder.CreateStore(Casted, ExnPtrTmp);
 
         // Bind the reference to the temporary.
-        AdjustedExn = ExnPtrTmp.getPointer();
+        AdjustedExn = ExnPtrTmp.emitRawPointer(CGF);
       }
     }
 
diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
index 172c4c937b97..d38a26940a3c 100644
--- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -327,10 +327,6 @@ public:
       CodeGenFunction &CGF, const CXXRecordDecl *VTableClass,
       BaseSubobject Base, const CXXRecordDecl *NearestVBase) override;
 
-  llvm::Constant *
-  getVTableAddressPointForConstExpr(BaseSubobject Base,
-                                    const CXXRecordDecl *VTableClass) override;
-
   llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD,
                                         CharUnits VPtrOffset) override;
 
@@ -937,7 +933,7 @@ void MicrosoftCXXABI::emitBeginCatch(CodeGenFunction &CGF,
   }
 
   CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(*CatchParam);
-  CPI->setArgOperand(2, var.getObjectAddress(CGF).getPointer());
+  CPI->setArgOperand(2, var.getObjectAddress(CGF).emitRawPointer(CGF));
   CGF.EHStack.pushCleanup<CatchRetScope>(NormalCleanup, CPI);
   CGF.EmitAutoVarCleanups(var);
 }
@@ -974,7 +970,7 @@ MicrosoftCXXABI::performBaseAdjustment(CodeGenFunction &CGF, Address Value,
   llvm::Value *Offset =
     GetVirtualBaseClassOffset(CGF, Value, SrcDecl, PolymorphicBase);
   llvm::Value *Ptr = CGF.Builder.CreateInBoundsGEP(
-      Value.getElementType(), Value.getPointer(), Offset);
+      Value.getElementType(), Value.emitRawPointer(CGF), Offset);
   CharUnits VBaseAlign =
     CGF.CGM.getVBaseAlignment(Value.getAlignment(), SrcDecl, PolymorphicBase);
   return std::make_tuple(Address(Ptr, CGF.Int8Ty, VBaseAlign), Offset,
@@ -1011,7 +1007,7 @@ llvm::Value *MicrosoftCXXABI::EmitTypeid(CodeGenFunction &CGF,
                                          llvm::Type *StdTypeInfoPtrTy) {
   std::tie(ThisPtr, std::ignore, std::ignore) =
       performBaseAdjustment(CGF, ThisPtr, SrcRecordTy);
-  llvm::CallBase *Typeid = emitRTtypeidCall(CGF, ThisPtr.getPointer());
+  llvm::CallBase *Typeid = emitRTtypeidCall(CGF, ThisPtr.emitRawPointer(CGF));
   return CGF.Builder.CreateBitCast(Typeid, StdTypeInfoPtrTy);
 }
 
@@ -1033,7 +1029,7 @@ llvm::Value *MicrosoftCXXABI::emitDynamicCastCall(
   llvm::Value *Offset;
   std::tie(This, Offset, std::ignore) =
       performBaseAdjustment(CGF, This, SrcRecordTy);
-  llvm::Value *ThisPtr = This.getPointer();
+  llvm::Value *ThisPtr = This.emitRawPointer(CGF);
   Offset = CGF.Builder.CreateTrunc(Offset, CGF.Int32Ty);
 
   // PVOID __RTDynamicCast(
@@ -1065,7 +1061,7 @@ llvm::Value *MicrosoftCXXABI::emitDynamicCastToVoid(CodeGenFunction &CGF,
   llvm::FunctionCallee Function = CGF.CGM.CreateRuntimeFunction(
       llvm::FunctionType::get(CGF.Int8PtrTy, ArgTypes, false),
       "__RTCastToVoid");
-  llvm::Value *Args[] = {Value.getPointer()};
+  llvm::Value *Args[] = {Value.emitRawPointer(CGF)};
   return CGF.EmitRuntimeCall(Function, Args);
 }
 
@@ -1493,7 +1489,7 @@ Address MicrosoftCXXABI::adjustThisArgumentForVirtualFunctionCall(
     llvm::Value *VBaseOffset =
       GetVirtualBaseClassOffset(CGF, Result, Derived, VBase);
     llvm::Value *VBasePtr = CGF.Builder.CreateInBoundsGEP(
-        Result.getElementType(), Result.getPointer(), VBaseOffset);
+        Result.getElementType(), Result.emitRawPointer(CGF), VBaseOffset);
     CharUnits VBaseAlign =
       CGF.CGM.getVBaseAlignment(Result.getAlignment(), Derived, VBase);
     Result = Address(VBasePtr, CGF.Int8Ty, VBaseAlign);
@@ -1660,7 +1656,8 @@ void MicrosoftCXXABI::EmitDestructorCall(CodeGenFunction &CGF,
   llvm::Value *Implicit =
       getCXXDestructorImplicitParam(CGF, DD, Type, ForVirtualBase,
                                     Delegating); // = nullptr
-  CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy,
+  CGF.EmitCXXDestructorCall(GD, Callee, CGF.getAsNaturalPointerTo(This, ThisTy),
+                            ThisTy,
                             /*ImplicitParam=*/Implicit,
                             /*ImplicitParamTy=*/QualType(), nullptr);
   if (BaseDtorEndBB) {
@@ -1791,13 +1788,6 @@ MicrosoftCXXABI::getVTableAddressPoint(BaseSubobject Base,
   return VFTablesMap[ID];
 }
 
-llvm::Constant *MicrosoftCXXABI::getVTableAddressPointForConstExpr(
-    BaseSubobject Base, const CXXRecordDecl *VTableClass) {
-  llvm::Constant *VFTable = getVTableAddressPoint(Base, VTableClass);
-  assert(VFTable && "Couldn't find a vftable for the given base?");
-  return VFTable;
-}
-
 llvm::GlobalVariable *MicrosoftCXXABI::getAddrOfVTable(const CXXRecordDecl *RD,
                                                        CharUnits VPtrOffset) {
   // getAddrOfVTable may return 0 if asked to get an address of a vtable which
@@ -2013,8 +2003,9 @@ llvm::Value *MicrosoftCXXABI::EmitVirtualDestructorCall(
   }
 
   This = adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, true);
-  RValue RV = CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy,
-                                        ImplicitParam, Context.IntTy, CE);
+  RValue RV =
+      CGF.EmitCXXDestructorCall(GD, Callee, This.emitRawPointer(CGF), ThisTy,
+                                ImplicitParam, Context.IntTy, CE);
   return RV.getScalarVal();
 }
 
@@ -2212,13 +2203,13 @@ llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF,
                                                     Address This,
                                                     const ThisAdjustment &TA) {
   if (TA.isEmpty())
-    return This.getPointer();
+    return This.emitRawPointer(CGF);
 
   This = This.withElementType(CGF.Int8Ty);
 
   llvm::Value *V;
   if (TA.Virtual.isEmpty()) {
-    V = This.getPointer();
+    V = This.emitRawPointer(CGF);
   } else {
     assert(TA.Virtual.Microsoft.VtordispOffset < 0);
     // Adjust the this argument based on the vtordisp value.
@@ -2227,7 +2218,7 @@ llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF,
                  CharUnits::fromQuantity(TA.Virtual.Microsoft.VtordispOffset));
     VtorDispPtr = VtorDispPtr.withElementType(CGF.Int32Ty);
     llvm::Value *VtorDisp = CGF.Builder.CreateLoad(VtorDispPtr, "vtordisp");
-    V = CGF.Builder.CreateGEP(This.getElementType(), This.getPointer(),
+    V = CGF.Builder.CreateGEP(This.getElementType(), This.emitRawPointer(CGF),
                               CGF.Builder.CreateNeg(VtorDisp));
 
     // Unfortunately, having applied the vtordisp means that we no
@@ -2264,11 +2255,11 @@ llvm::Value *
 MicrosoftCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret,
                                          const ReturnAdjustment &RA) {
   if (RA.isEmpty())
-    return Ret.getPointer();
+    return Ret.emitRawPointer(CGF);
 
   Ret = Ret.withElementType(CGF.Int8Ty);
 
-  llvm::Value *V = Ret.getPointer();
+  llvm::Value *V = Ret.emitRawPointer(CGF);
   if (RA.Virtual.Microsoft.VBIndex) {
     assert(RA.Virtual.Microsoft.VBIndex > 0);
     int32_t IntSize = CGF.getIntSize().getQuantity();
@@ -2583,7 +2574,7 @@ struct ResetGuardBit final : EHScopeStack::Cleanup {
 
 struct CallInitThreadAbort final : EHScopeStack::Cleanup {
   llvm::Value *Guard;
-  CallInitThreadAbort(Address Guard) : Guard(Guard.getPointer()) {}
+  CallInitThreadAbort(RawAddress Guard) : Guard(Guard.getPointer()) {}
 
   void Emit(CodeGenFunction &CGF, Flags flags) override {
     // Calling _Init_thread_abort will reset the guard's state.
@@ -3123,8 +3114,8 @@ MicrosoftCXXABI::GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF,
                                          llvm::Value **VBPtrOut) {
   CGBuilderTy &Builder = CGF.Builder;
   // Load the vbtable pointer from the vbptr in the instance.
-  llvm::Value *VBPtr = Builder.CreateInBoundsGEP(CGM.Int8Ty, This.getPointer(),
-                                                 VBPtrOffset, "vbptr");
+  llvm::Value *VBPtr = Builder.CreateInBoundsGEP(
+      CGM.Int8Ty, This.emitRawPointer(CGF), VBPtrOffset, "vbptr");
   if (VBPtrOut)
     *VBPtrOut = VBPtr;
 
@@ -3203,7 +3194,7 @@ llvm::Value *MicrosoftCXXABI::AdjustVirtualBase(
     Builder.CreateBr(SkipAdjustBB);
     CGF.EmitBlock(SkipAdjustBB);
     llvm::PHINode *Phi = Builder.CreatePHI(CGM.Int8PtrTy, 2, "memptr.base");
-    Phi->addIncoming(Base.getPointer(), OriginalBB);
+    Phi->addIncoming(Base.emitRawPointer(CGF), OriginalBB);
     Phi->addIncoming(AdjustedBase, VBaseAdjustBB);
     return Phi;
   }
@@ -3238,7 +3229,7 @@ llvm::Value *MicrosoftCXXABI::EmitMemberDataPointerAddress(
     Addr = AdjustVirtualBase(CGF, E, RD, Base, VirtualBaseAdjustmentOffset,
                              VBPtrOffset);
   } else {
-    Addr = Base.getPointer();
+    Addr = Base.emitRawPointer(CGF);
   }
 
   // Apply the offset, which we assume is non-null.
@@ -3526,7 +3517,7 @@ CGCallee MicrosoftCXXABI::EmitLoadOfMemberFunctionPointer(
     ThisPtrForCall = AdjustVirtualBase(CGF, E, RD, This,
                                    VirtualBaseAdjustmentOffset, VBPtrOffset);
   } else {
-    ThisPtrForCall = This.getPointer();
+    ThisPtrForCall = This.emitRawPointer(CGF);
   }
 
   if (NonVirtualBaseAdjustment)
@@ -4445,10 +4436,7 @@ void MicrosoftCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) {
   llvm::GlobalVariable *TI = getThrowInfo(ThrowType);
 
   // Call into the runtime to throw the exception.
-  llvm::Value *Args[] = {
-    AI.getPointer(),
-    TI
-  };
+  llvm::Value *Args[] = {AI.emitRawPointer(CGF), TI};
   CGF.EmitNoreturnRuntimeCallOrInvoke(getThrowFn(), Args);
 }
 
diff --git a/clang/lib/CodeGen/TargetInfo.h b/clang/lib/CodeGen/TargetInfo.h
index 6893b50a3cfe..b1dfe5bf8f27 100644
--- a/clang/lib/CodeGen/TargetInfo.h
+++ b/clang/lib/CodeGen/TargetInfo.h
@@ -295,6 +295,11 @@ public:
   /// Get the AST address space for alloca.
   virtual LangAS getASTAllocaAddressSpace() const { return LangAS::Default; }
 
+  Address performAddrSpaceCast(CodeGen::CodeGenFunction &CGF, Address Addr,
+                               LangAS SrcAddr, LangAS DestAddr,
+                               llvm::Type *DestTy,
+                               bool IsNonNull = false) const;
+
   /// Perform address space cast of an expression of pointer type.
   /// \param V is the LLVM value to be casted to another address space.
   /// \param SrcAddr is the language address space of \p V.
diff --git a/clang/lib/CodeGen/Targets/NVPTX.cpp b/clang/lib/CodeGen/Targets/NVPTX.cpp
index 8718f1ecf3a7..7dce5042c3dc 100644
--- a/clang/lib/CodeGen/Targets/NVPTX.cpp
+++ b/clang/lib/CodeGen/Targets/NVPTX.cpp
@@ -85,7 +85,7 @@ private:
                                            LValue Src) {
     llvm::Value *Handle = nullptr;
     llvm::Constant *C =
-        llvm::dyn_cast<llvm::Constant>(Src.getAddress(CGF).getPointer());
+        llvm::dyn_cast<llvm::Constant>(Src.getAddress(CGF).emitRawPointer(CGF));
     // Lookup `addrspacecast` through the constant pointer if any.
     if (auto *ASC = llvm::dyn_cast_or_null<llvm::AddrSpaceCastOperator>(C))
       C = llvm::cast<llvm::Constant>(ASC->getPointerOperand());
diff --git a/clang/lib/CodeGen/Targets/PPC.cpp b/clang/lib/CodeGen/Targets/PPC.cpp
index 00b04723f17d..362add30b435 100644
--- a/clang/lib/CodeGen/Targets/PPC.cpp
+++ b/clang/lib/CodeGen/Targets/PPC.cpp
@@ -513,9 +513,10 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
     CharUnits RegSize = CharUnits::fromQuantity((isInt || IsSoftFloatABI) ? 4 : 8);
     llvm::Value *RegOffset =
         Builder.CreateMul(NumRegs, Builder.getInt8(RegSize.getQuantity()));
-    RegAddr = Address(
-        Builder.CreateInBoundsGEP(CGF.Int8Ty, RegAddr.getPointer(), RegOffset),
-        DirectTy, RegAddr.getAlignment().alignmentOfArrayElement(RegSize));
+    RegAddr = Address(Builder.CreateInBoundsGEP(
+                          CGF.Int8Ty, RegAddr.emitRawPointer(CGF), RegOffset),
+                      DirectTy,
+                      RegAddr.getAlignment().alignmentOfArrayElement(RegSize));
 
     // Increase the used-register count.
     NumRegs =
@@ -551,7 +552,7 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
     // Round up address of argument to alignment
     CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
     if (Align > OverflowAreaAlign) {
-      llvm::Value *Ptr = OverflowArea.getPointer();
+      llvm::Value *Ptr = OverflowArea.emitRawPointer(CGF);
       OverflowArea = Address(emitRoundPointerUpToAlignment(CGF, Ptr, Align),
                              OverflowArea.getElementType(), Align);
     }
@@ -560,7 +561,7 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
 
     // Increase the overflow area.
     OverflowArea = Builder.CreateConstInBoundsByteGEP(OverflowArea, Size);
-    Builder.CreateStore(OverflowArea.getPointer(), OverflowAreaAddr);
+    Builder.CreateStore(OverflowArea.emitRawPointer(CGF), OverflowAreaAddr);
     CGF.EmitBranch(Cont);
   }
 
diff --git a/clang/lib/CodeGen/Targets/Sparc.cpp b/clang/lib/CodeGen/Targets/Sparc.cpp
index a337a52a94ec..9025a633f328 100644
--- a/clang/lib/CodeGen/Targets/Sparc.cpp
+++ b/clang/lib/CodeGen/Targets/Sparc.cpp
@@ -326,7 +326,7 @@ Address SparcV9ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
 
   // Update VAList.
   Address NextPtr = Builder.CreateConstInBoundsByteGEP(Addr, Stride, "ap.next");
-  Builder.CreateStore(NextPtr.getPointer(), VAListAddr);
+  Builder.CreateStore(NextPtr.emitRawPointer(CGF), VAListAddr);
 
   return ArgAddr.withElementType(ArgTy);
 }
diff --git a/clang/lib/CodeGen/Targets/SystemZ.cpp b/clang/lib/CodeGen/Targets/SystemZ.cpp
index 6eb0c6ef2f7d..deaafc85a315 100644
--- a/clang/lib/CodeGen/Targets/SystemZ.cpp
+++ b/clang/lib/CodeGen/Targets/SystemZ.cpp
@@ -306,7 +306,7 @@ Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
 
     // Update overflow_arg_area_ptr pointer
     llvm::Value *NewOverflowArgArea = CGF.Builder.CreateGEP(
-        OverflowArgArea.getElementType(), OverflowArgArea.getPointer(),
+        OverflowArgArea.getElementType(), OverflowArgArea.emitRawPointer(CGF),
         PaddedSizeV, "overflow_arg_area");
     CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr);
 
@@ -382,10 +382,9 @@ Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
   Address MemAddr = RawMemAddr.withElementType(DirectTy);
 
   // Update overflow_arg_area_ptr pointer
-  llvm::Value *NewOverflowArgArea =
-    CGF.Builder.CreateGEP(OverflowArgArea.getElementType(),
-                          OverflowArgArea.getPointer(), PaddedSizeV,
-                          "overflow_arg_area");
+  llvm::Value *NewOverflowArgArea = CGF.Builder.CreateGEP(
+      OverflowArgArea.getElementType(), OverflowArgArea.emitRawPointer(CGF),
+      PaddedSizeV, "overflow_arg_area");
   CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr);
   CGF.EmitBranch(ContBlock);
 
diff --git a/clang/lib/CodeGen/Targets/XCore.cpp b/clang/lib/CodeGen/Targets/XCore.cpp
index aeb48f851e16..88edb781a947 100644
--- a/clang/lib/CodeGen/Targets/XCore.cpp
+++ b/clang/lib/CodeGen/Targets/XCore.cpp
@@ -180,7 +180,7 @@ Address XCoreABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
   // Increment the VAList.
   if (!ArgSize.isZero()) {
     Address APN = Builder.CreateConstInBoundsByteGEP(AP, ArgSize);
-    Builder.CreateStore(APN.getPointer(), VAListAddr);
+    Builder.CreateStore(APN.emitRawPointer(CGF), VAListAddr);
   }
 
   return Val;
diff --git a/clang/utils/TableGen/MveEmitter.cpp b/clang/utils/TableGen/MveEmitter.cpp
index 3a90eee5f1c9..aa20c758d84a 100644
--- a/clang/utils/TableGen/MveEmitter.cpp
+++ b/clang/utils/TableGen/MveEmitter.cpp
@@ -575,7 +575,7 @@ public:
   // Emit code to generate this result as a Value *.
   std::string asValue() override {
     if (AddressType)
-      return "(" + varname() + ".getPointer())";
+      return "(" + varname() + ".emitRawPointer(*this))";
     return Result::asValue();
   }
   bool hasIntegerValue() const override { return Immediate; }
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index c07ffea71151..bee1a1f207fc 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -2707,6 +2707,7 @@ public:
   IRBuilder(const IRBuilder &) = delete;
 
   InserterTy &getInserter() { return Inserter; }
+  const InserterTy &getInserter() const { return Inserter; }
 };
 
 template <typename FolderTy, typename InserterTy>
-- 
GitLab


From 6420f379268e9178f9f938cef223194ad3daae4e Mon Sep 17 00:00:00 2001
From: YAMAMOTO Takashi <yamamoto@midokura.com>
Date: Tue, 26 Mar 2024 10:11:56 +0900
Subject: [PATCH 230/404] [WebAssembly] Implement an alternative translation
 for -wasm-enable-sjlj (#84137)

Instead of maintaining per-function-invocation malloc()'ed tables to track which functions each label belongs to, store the equivalent info in jump buffers (jmp_buf) themselves.

Also, use a less emscripten-looking ABI symbols:
```
    saveSetjmp     -> __wasm_setjmp
    testSetjmp      -> __wasm_setjmp_test
    getTempRet0    -> (removed)
    __wasm_longjmp  -> (no change)
```

While I want to use this for WASI, it should work for emscripten as well.

An example runtime and a few tests:
https://github.com/yamt/garbage/tree/wasm-sjlj-alt2/wasm/longjmp

wasi-libc version of the runtime:
https://github.com/WebAssembly/wasi-libc/pull/483

emscripten version of the runtime:
https://github.com/emscripten-core/emscripten/pull/21502

Discussion:
https://docs.google.com/document/d/1ZvTPT36K5jjiedF8MCXbEmYjULJjI723aOAks1IdLLg/edit
---
 .../WebAssemblyLowerEmscriptenEHSjLj.cpp      | 314 ++++++------------
 .../WebAssembly/lower-em-ehsjlj-options.ll    |   4 +-
 .../CodeGen/WebAssembly/lower-em-ehsjlj.ll    |   5 +-
 .../WebAssembly/lower-em-sjlj-alias.ll        |   2 +-
 .../WebAssembly/lower-em-sjlj-debuginfo.ll    |  35 +-
 .../test/CodeGen/WebAssembly/lower-em-sjlj.ll |  37 +--
 .../CodeGen/WebAssembly/lower-wasm-ehsjlj.ll  |   6 +-
 .../CodeGen/WebAssembly/lower-wasm-sjlj.ll    |  22 +-
 8 files changed, 127 insertions(+), 298 deletions(-)

diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
index 0427fe473f8e..027ee1086bf4 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
@@ -129,27 +129,23 @@
 ///
 /// If there are calls to setjmp()
 ///
-/// 2) In the function entry that calls setjmp, initialize setjmpTable and
-///    sejmpTableSize as follows:
-///      setjmpTableSize = 4;
-///      setjmpTable = (int *) malloc(40);
-///      setjmpTable[0] = 0;
-///    setjmpTable and setjmpTableSize are used to call saveSetjmp() function in
-///    Emscripten compiler-rt.
+/// 2) In the function entry that calls setjmp, initialize
+///    functionInvocationId as follows:
+///
+///    functionInvocationId = alloca(4)
+///
+///    Note: the alloca size is not important as this pointer is
+///    merely used for pointer comparisions.
 ///
 /// 3) Lower
 ///      setjmp(env)
 ///    into
-///      setjmpTable = saveSetjmp(env, label, setjmpTable, setjmpTableSize);
-///      setjmpTableSize = getTempRet0();
-///    For each dynamic setjmp call, setjmpTable stores its ID (a number which
-///    is incrementally assigned from 0) and its label (a unique number that
-///    represents each callsite of setjmp). When we need more entries in
-///    setjmpTable, it is reallocated in saveSetjmp() in Emscripten's
-///    compiler-rt and it will return the new table address, and assign the new
-///    table size in setTempRet0(). saveSetjmp also stores the setjmp's ID into
-///    the buffer 'env'. A BB with setjmp is split into two after setjmp call in
-///    order to make the post-setjmp BB the possible destination of longjmp BB.
+///      __wasm_setjmp(env, label, functionInvocationId)
+///
+///    __wasm_setjmp records the necessary info (the label and
+///    functionInvocationId) to the "env".
+///    A BB with setjmp is split into two after setjmp call in order to
+///    make the post-setjmp BB the possible destination of longjmp BB.
 ///
 /// 4) Lower every call that might longjmp into
 ///      __THREW__ = 0;
@@ -158,8 +154,7 @@
 ///      __THREW__ = 0;
 ///      %__threwValue.val = __threwValue;
 ///      if (%__THREW__.val != 0 & %__threwValue.val != 0) {
-///        %label = testSetjmp(mem[%__THREW__.val], setjmpTable,
-///                            setjmpTableSize);
+///        %label = __wasm_setjmp_test(%__THREW__.val, functionInvocationId);
 ///        if (%label == 0)
 ///          emscripten_longjmp(%__THREW__.val, %__threwValue.val);
 ///        setTempRet0(%__threwValue.val);
@@ -173,16 +168,16 @@
 ///        ...
 ///        default: goto splitted next BB
 ///      }
-///    testSetjmp examines setjmpTable to see if there is a matching setjmp
-///    call. After calling an invoke wrapper, if a longjmp occurred, __THREW__
-///    will be the address of matching jmp_buf buffer and __threwValue be the
-///    second argument to longjmp. mem[%__THREW__.val] is a setjmp ID that is
-///    stored in saveSetjmp. testSetjmp returns a setjmp label, a unique ID to
-///    each setjmp callsite. Label 0 means this longjmp buffer does not
-///    correspond to one of the setjmp callsites in this function, so in this
-///    case we just chain the longjmp to the caller. Label -1 means no longjmp
-///    occurred. Otherwise we jump to the right post-setjmp BB based on the
-///    label.
+///
+///    __wasm_setjmp_test examines the jmp buf to see if it was for a matching
+///    setjmp call. After calling an invoke wrapper, if a longjmp occurred,
+///    __THREW__ will be the address of matching jmp_buf buffer and
+///    __threwValue be the second argument to longjmp.
+///    __wasm_setjmp_test returns a setjmp label, a unique ID to each setjmp
+///    callsite. Label 0 means this longjmp buffer does not correspond to one
+///    of the setjmp callsites in this function, so in this case we just chain
+///    the longjmp to the caller. Label -1 means no longjmp occurred.
+///    Otherwise we jump to the right post-setjmp BB based on the label.
 ///
 /// * Wasm setjmp / longjmp handling
 /// This mode still uses some Emscripten library functions but not JavaScript's
@@ -199,45 +194,44 @@
 /// If there are calls to setjmp()
 ///
 /// 2) and 3): The same as 2) and 3) in Emscripten SjLj.
-/// (setjmpTable/setjmpTableSize initialization + setjmp callsite
-/// transformation)
+/// (functionInvocationId initialization + setjmp callsite transformation)
 ///
 /// 4) Create a catchpad with a wasm.catch() intrinsic, which returns the value
-/// thrown by __wasm_longjmp function. In Emscripten library, we have this
-/// struct:
+/// thrown by __wasm_longjmp function. In the runtime library, we have an
+/// equivalent of the following struct:
 ///
 /// struct __WasmLongjmpArgs {
 ///   void *env;
 ///   int val;
 /// };
-/// struct __WasmLongjmpArgs __wasm_longjmp_args;
 ///
-/// The thrown value here is a pointer to __wasm_longjmp_args struct object. We
-/// use this struct to transfer two values by throwing a single value. Wasm
-/// throw and catch instructions are capable of throwing and catching multiple
-/// values, but it also requires multivalue support that is currently not very
-/// reliable.
+/// The thrown value here is a pointer to the struct. We use this struct to
+/// transfer two values by throwing a single value. Wasm throw and catch
+/// instructions are capable of throwing and catching multiple values, but
+/// it also requires multivalue support that is currently not very reliable.
 /// TODO Switch to throwing and catching two values without using the struct
 ///
 /// All longjmpable function calls will be converted to an invoke that will
 /// unwind to this catchpad in case a longjmp occurs. Within the catchpad, we
-/// test the thrown values using testSetjmp function as we do for Emscripten
-/// SjLj. The main difference is, in Emscripten SjLj, we need to transform every
-/// longjmpable callsite into a sequence of code including testSetjmp() call; in
-/// Wasm SjLj we do the testing in only one place, in this catchpad.
+/// test the thrown values using __wasm_setjmp_test function as we do for
+/// Emscripten SjLj. The main difference is, in Emscripten SjLj, we need to
+/// transform every longjmpable callsite into a sequence of code including
+/// __wasm_setjmp_test() call; in Wasm SjLj we do the testing in only one
+/// place, in this catchpad.
 ///
-/// After testing calling testSetjmp(), if the longjmp does not correspond to
-/// one of the setjmps within the current function, it rethrows the longjmp
-/// by calling __wasm_longjmp(). If it corresponds to one of setjmps in the
-/// function, we jump to the beginning of the function, which contains a switch
-/// to each post-setjmp BB. Again, in Emscripten SjLj, this switch is added for
-/// every longjmpable callsite; in Wasm SjLj we do this only once at the top of
-/// the function. (after setjmpTable/setjmpTableSize initialization)
+/// After testing calling __wasm_setjmp_test(), if the longjmp does not
+/// correspond to one of the setjmps within the current function, it rethrows
+/// the longjmp by calling __wasm_longjmp(). If it corresponds to one of
+/// setjmps in the function, we jump to the beginning of the function, which
+/// contains a switch to each post-setjmp BB. Again, in Emscripten SjLj, this
+/// switch is added for every longjmpable callsite; in Wasm SjLj we do this
+/// only once at the top of the function. (after functionInvocationId
+/// initialization)
 ///
 /// The below is the pseudocode for what we have described
 ///
 /// entry:
-///   Initialize setjmpTable and setjmpTableSize
+///   Initialize functionInvocationId
 ///
 /// setjmp.dispatch:
 ///    switch %label {
@@ -260,7 +254,7 @@
 ///   %longjmp.args = wasm.catch() ;; struct __WasmLongjmpArgs
 ///   %env = load 'env' field from __WasmLongjmpArgs
 ///   %val = load 'val' field from __WasmLongjmpArgs
-///   %label = testSetjmp(mem[%env], setjmpTable, setjmpTableSize);
+///   %label = __wasm_setjmp_test(%env, functionInvocationId);
 ///   if (%label == 0)
 ///     __wasm_longjmp(%env, %val)
 ///   catchret to %setjmp.dispatch
@@ -309,8 +303,8 @@ class WebAssemblyLowerEmscriptenEHSjLj final : public ModulePass {
   Function *ResumeF = nullptr;            // __resumeException() (Emscripten)
   Function *EHTypeIDF = nullptr;          // llvm.eh.typeid.for() (intrinsic)
   Function *EmLongjmpF = nullptr;         // emscripten_longjmp() (Emscripten)
-  Function *SaveSetjmpF = nullptr;        // saveSetjmp() (Emscripten)
-  Function *TestSetjmpF = nullptr;        // testSetjmp() (Emscripten)
+  Function *WasmSetjmpF = nullptr;        // __wasm_setjmp() (Emscripten)
+  Function *WasmSetjmpTestF = nullptr;    // __wasm_setjmp_test() (Emscripten)
   Function *WasmLongjmpF = nullptr;       // __wasm_longjmp() (Emscripten)
   Function *CatchF = nullptr;             // wasm.catch() (intrinsic)
 
@@ -335,18 +329,17 @@ class WebAssemblyLowerEmscriptenEHSjLj final : public ModulePass {
   bool runEHOnFunction(Function &F);
   bool runSjLjOnFunction(Function &F);
   void handleLongjmpableCallsForEmscriptenSjLj(
-      Function &F, InstVector &SetjmpTableInsts,
-      InstVector &SetjmpTableSizeInsts,
+      Function &F, Instruction *FunctionInvocationId,
       SmallVectorImpl<PHINode *> &SetjmpRetPHIs);
   void
-  handleLongjmpableCallsForWasmSjLj(Function &F, InstVector &SetjmpTableInsts,
-                                    InstVector &SetjmpTableSizeInsts,
+  handleLongjmpableCallsForWasmSjLj(Function &F,
+                                    Instruction *FunctionInvocationId,
                                     SmallVectorImpl<PHINode *> &SetjmpRetPHIs);
   Function *getFindMatchingCatch(Module &M, unsigned NumClauses);
 
   Value *wrapInvoke(CallBase *CI);
   void wrapTestSetjmp(BasicBlock *BB, DebugLoc DL, Value *Threw,
-                      Value *SetjmpTable, Value *SetjmpTableSize, Value *&Label,
+                      Value *FunctionInvocationId, Value *&Label,
                       Value *&LongjmpResult, BasicBlock *&CallEmLongjmpBB,
                       PHINode *&CallEmLongjmpBBThrewPHI,
                       PHINode *&CallEmLongjmpBBThrewValuePHI,
@@ -618,7 +611,7 @@ static bool canLongjmp(const Value *Callee) {
 
   // There are functions in Emscripten's JS glue code or compiler-rt
   if (CalleeName == "__resumeException" || CalleeName == "llvm_eh_typeid_for" ||
-      CalleeName == "saveSetjmp" || CalleeName == "testSetjmp" ||
+      CalleeName == "__wasm_setjmp" || CalleeName == "__wasm_setjmp_test" ||
       CalleeName == "getTempRet0" || CalleeName == "setTempRet0")
     return false;
 
@@ -687,11 +680,12 @@ static bool isEmAsmCall(const Value *Callee) {
          CalleeName == "emscripten_asm_const_async_on_main_thread";
 }
 
-// Generate testSetjmp function call seqence with preamble and postamble.
-// The code this generates is equivalent to the following JavaScript code:
+// Generate __wasm_setjmp_test function call seqence with preamble and
+// postamble. The code this generates is equivalent to the following
+// JavaScript code:
 // %__threwValue.val = __threwValue;
 // if (%__THREW__.val != 0 & %__threwValue.val != 0) {
-//   %label = testSetjmp(mem[%__THREW__.val], setjmpTable, setjmpTableSize);
+//   %label = __wasm_setjmp_test(%__THREW__.val, functionInvocationId);
 //   if (%label == 0)
 //     emscripten_longjmp(%__THREW__.val, %__threwValue.val);
 //   setTempRet0(%__threwValue.val);
@@ -703,10 +697,10 @@ static bool isEmAsmCall(const Value *Callee) {
 // As output parameters. returns %label, %longjmp_result, and the BB the last
 // instruction (%longjmp_result = ...) is in.
 void WebAssemblyLowerEmscriptenEHSjLj::wrapTestSetjmp(
-    BasicBlock *BB, DebugLoc DL, Value *Threw, Value *SetjmpTable,
-    Value *SetjmpTableSize, Value *&Label, Value *&LongjmpResult,
-    BasicBlock *&CallEmLongjmpBB, PHINode *&CallEmLongjmpBBThrewPHI,
-    PHINode *&CallEmLongjmpBBThrewValuePHI, BasicBlock *&EndBB) {
+    BasicBlock *BB, DebugLoc DL, Value *Threw, Value *FunctionInvocationId,
+    Value *&Label, Value *&LongjmpResult, BasicBlock *&CallEmLongjmpBB,
+    PHINode *&CallEmLongjmpBBThrewPHI, PHINode *&CallEmLongjmpBBThrewValuePHI,
+    BasicBlock *&EndBB) {
   Function *F = BB->getParent();
   Module *M = F->getParent();
   LLVMContext &C = M->getContext();
@@ -743,16 +737,14 @@ void WebAssemblyLowerEmscriptenEHSjLj::wrapTestSetjmp(
     CallEmLongjmpBBThrewValuePHI->addIncoming(ThrewValue, ThenBB1);
   }
 
-  // %label = testSetjmp(mem[%__THREW__.val], setjmpTable, setjmpTableSize);
+  // %label = __wasm_setjmp_test(%__THREW__.val, functionInvocationId);
   // if (%label == 0)
   IRB.SetInsertPoint(ThenBB1);
   BasicBlock *EndBB2 = BasicBlock::Create(C, "if.end2", F);
   Value *ThrewPtr =
       IRB.CreateIntToPtr(Threw, getAddrPtrType(M), Threw->getName() + ".p");
-  Value *LoadedThrew = IRB.CreateLoad(getAddrIntType(M), ThrewPtr,
-                                      ThrewPtr->getName() + ".loaded");
-  Value *ThenLabel = IRB.CreateCall(
-      TestSetjmpF, {LoadedThrew, SetjmpTable, SetjmpTableSize}, "label");
+  Value *ThenLabel = IRB.CreateCall(WasmSetjmpTestF,
+                                    {ThrewPtr, FunctionInvocationId}, "label");
   Value *Cmp2 = IRB.CreateICmpEQ(ThenLabel, IRB.getInt32(0));
   IRB.CreateCondBr(Cmp2, CallEmLongjmpBB, EndBB2);
 
@@ -1007,17 +999,17 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runOnModule(Module &M) {
       Type *Int8PtrTy = IRB.getPtrTy();
       Type *Int32PtrTy = IRB.getPtrTy();
       Type *Int32Ty = IRB.getInt32Ty();
-      // Register saveSetjmp function
+
+      // Register __wasm_setjmp function
       FunctionType *SetjmpFTy = SetjmpF->getFunctionType();
       FunctionType *FTy = FunctionType::get(
-          Int32PtrTy,
-          {SetjmpFTy->getParamType(0), Int32Ty, Int32PtrTy, Int32Ty}, false);
-      SaveSetjmpF = getEmscriptenFunction(FTy, "saveSetjmp", &M);
+          IRB.getVoidTy(), {SetjmpFTy->getParamType(0), Int32Ty, Int32PtrTy},
+          false);
+      WasmSetjmpF = getEmscriptenFunction(FTy, "__wasm_setjmp", &M);
 
-      // Register testSetjmp function
-      FTy = FunctionType::get(Int32Ty,
-                              {getAddrIntType(&M), Int32PtrTy, Int32Ty}, false);
-      TestSetjmpF = getEmscriptenFunction(FTy, "testSetjmp", &M);
+      // Register __wasm_setjmp_test function
+      FTy = FunctionType::get(Int32Ty, {Int32PtrTy, Int32PtrTy}, false);
+      WasmSetjmpTestF = getEmscriptenFunction(FTy, "__wasm_setjmp_test", &M);
 
       // wasm.catch() will be lowered down to wasm 'catch' instruction in
       // instruction selection.
@@ -1063,7 +1055,7 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runOnModule(Module &M) {
     if (V && V->use_empty())
       V->eraseFromParent();
   for (auto *V : {GetTempRet0F, SetTempRet0F, ResumeF, EHTypeIDF, EmLongjmpF,
-                  SaveSetjmpF, TestSetjmpF, WasmLongjmpF, CatchF})
+                  WasmSetjmpF, WasmSetjmpTestF, WasmLongjmpF, CatchF})
     if (V && V->use_empty())
       V->eraseFromParent();
 
@@ -1268,42 +1260,20 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runSjLjOnFunction(Function &F) {
   LLVMContext &C = F.getContext();
   IRBuilder<> IRB(C);
   SmallVector<Instruction *, 64> ToErase;
-  // Vector of %setjmpTable values
-  SmallVector<Instruction *, 4> SetjmpTableInsts;
-  // Vector of %setjmpTableSize values
-  SmallVector<Instruction *, 4> SetjmpTableSizeInsts;
 
   // Setjmp preparation
 
-  // This instruction effectively means %setjmpTableSize = 4.
-  // We create this as an instruction intentionally, and we don't want to fold
-  // this instruction to a constant 4, because this value will be used in
-  // SSAUpdater.AddAvailableValue(...) later.
   BasicBlock *Entry = &F.getEntryBlock();
   DebugLoc FirstDL = getOrCreateDebugLoc(&*Entry->begin(), F.getSubprogram());
   SplitBlock(Entry, &*Entry->getFirstInsertionPt());
 
-  BinaryOperator *SetjmpTableSize = BinaryOperator::Create(
-      Instruction::Add, IRB.getInt32(4), IRB.getInt32(0), "setjmpTableSize",
-      Entry->getTerminator()->getIterator());
-  SetjmpTableSize->setDebugLoc(FirstDL);
-  // setjmpTable = (int *) malloc(40);
-  Type *IntPtrTy = getAddrIntType(&M);
-  Constant *size = ConstantInt::get(IntPtrTy, 40);
-  IRB.SetInsertPoint(SetjmpTableSize);
-  auto *SetjmpTable = IRB.CreateMalloc(IntPtrTy, IRB.getInt32Ty(), size,
-                                       nullptr, nullptr, "setjmpTable");
-  SetjmpTable->setDebugLoc(FirstDL);
-  // CallInst::CreateMalloc may return a bitcast instruction if the result types
-  // mismatch. We need to set the debug loc for the original call too.
-  auto *MallocCall = SetjmpTable->stripPointerCasts();
-  if (auto *MallocCallI = dyn_cast<Instruction>(MallocCall)) {
-    MallocCallI->setDebugLoc(FirstDL);
-  }
-  // setjmpTable[0] = 0;
-  IRB.CreateStore(IRB.getInt32(0), SetjmpTable);
-  SetjmpTableInsts.push_back(SetjmpTable);
-  SetjmpTableSizeInsts.push_back(SetjmpTableSize);
+  IRB.SetInsertPoint(Entry->getTerminator()->getIterator());
+  // This alloca'ed pointer is used by the runtime to identify function
+  // invocations. It's just for pointer comparisons. It will never be
+  // dereferenced.
+  Instruction *FunctionInvocationId =
+      IRB.CreateAlloca(IRB.getInt32Ty(), nullptr, "functionInvocationId");
+  FunctionInvocationId->setDebugLoc(FirstDL);
 
   // Setjmp transformation
   SmallVector<PHINode *, 4> SetjmpRetPHIs;
@@ -1350,92 +1320,22 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runSjLjOnFunction(Function &F) {
     // 0, because index 0 means the longjmp is not ours to handle.
     IRB.SetInsertPoint(CI);
     Value *Args[] = {CI->getArgOperand(0), IRB.getInt32(SetjmpRetPHIs.size()),
-                     SetjmpTable, SetjmpTableSize};
-    Instruction *NewSetjmpTable =
-        IRB.CreateCall(SaveSetjmpF, Args, "setjmpTable");
-    Instruction *NewSetjmpTableSize =
-        IRB.CreateCall(GetTempRet0F, std::nullopt, "setjmpTableSize");
-    SetjmpTableInsts.push_back(NewSetjmpTable);
-    SetjmpTableSizeInsts.push_back(NewSetjmpTableSize);
+                     FunctionInvocationId};
+    IRB.CreateCall(WasmSetjmpF, Args);
     ToErase.push_back(CI);
   }
 
   // Handle longjmpable calls.
   if (EnableEmSjLj)
-    handleLongjmpableCallsForEmscriptenSjLj(
-        F, SetjmpTableInsts, SetjmpTableSizeInsts, SetjmpRetPHIs);
+    handleLongjmpableCallsForEmscriptenSjLj(F, FunctionInvocationId,
+                                            SetjmpRetPHIs);
   else // EnableWasmSjLj
-    handleLongjmpableCallsForWasmSjLj(F, SetjmpTableInsts, SetjmpTableSizeInsts,
-                                      SetjmpRetPHIs);
+    handleLongjmpableCallsForWasmSjLj(F, FunctionInvocationId, SetjmpRetPHIs);
 
   // Erase everything we no longer need in this function
   for (Instruction *I : ToErase)
     I->eraseFromParent();
 
-  // Free setjmpTable buffer before each return instruction + function-exiting
-  // call
-  SmallVector<Instruction *, 16> ExitingInsts;
-  for (BasicBlock &BB : F) {
-    Instruction *TI = BB.getTerminator();
-    if (isa<ReturnInst>(TI))
-      ExitingInsts.push_back(TI);
-    // Any 'call' instruction with 'noreturn' attribute exits the function at
-    // this point. If this throws but unwinds to another EH pad within this
-    // function instead of exiting, this would have been an 'invoke', which
-    // happens if we use Wasm EH or Wasm SjLJ.
-    for (auto &I : BB) {
-      if (auto *CI = dyn_cast<CallInst>(&I)) {
-        bool IsNoReturn = CI->hasFnAttr(Attribute::NoReturn);
-        if (Function *CalleeF = CI->getCalledFunction())
-          IsNoReturn |= CalleeF->hasFnAttribute(Attribute::NoReturn);
-        if (IsNoReturn)
-          ExitingInsts.push_back(&I);
-      }
-    }
-  }
-  for (auto *I : ExitingInsts) {
-    DebugLoc DL = getOrCreateDebugLoc(I, F.getSubprogram());
-    // If this existing instruction is a call within a catchpad, we should add
-    // it as "funclet" to the operand bundle of 'free' call
-    SmallVector<OperandBundleDef, 1> Bundles;
-    if (auto *CB = dyn_cast<CallBase>(I))
-      if (auto Bundle = CB->getOperandBundle(LLVMContext::OB_funclet))
-        Bundles.push_back(OperandBundleDef(*Bundle));
-    IRB.SetInsertPoint(I);
-    auto *Free = IRB.CreateFree(SetjmpTable, Bundles);
-    Free->setDebugLoc(DL);
-  }
-
-  // Every call to saveSetjmp can change setjmpTable and setjmpTableSize
-  // (when buffer reallocation occurs)
-  // entry:
-  //   setjmpTableSize = 4;
-  //   setjmpTable = (int *) malloc(40);
-  //   setjmpTable[0] = 0;
-  // ...
-  // somebb:
-  //   setjmpTable = saveSetjmp(env, label, setjmpTable, setjmpTableSize);
-  //   setjmpTableSize = getTempRet0();
-  // So we need to make sure the SSA for these variables is valid so that every
-  // saveSetjmp and testSetjmp calls have the correct arguments.
-  SSAUpdater SetjmpTableSSA;
-  SSAUpdater SetjmpTableSizeSSA;
-  SetjmpTableSSA.Initialize(PointerType::get(C, 0), "setjmpTable");
-  SetjmpTableSizeSSA.Initialize(Type::getInt32Ty(C), "setjmpTableSize");
-  for (Instruction *I : SetjmpTableInsts)
-    SetjmpTableSSA.AddAvailableValue(I->getParent(), I);
-  for (Instruction *I : SetjmpTableSizeInsts)
-    SetjmpTableSizeSSA.AddAvailableValue(I->getParent(), I);
-
-  for (auto &U : make_early_inc_range(SetjmpTable->uses()))
-    if (auto *I = dyn_cast<Instruction>(U.getUser()))
-      if (I->getParent() != Entry)
-        SetjmpTableSSA.RewriteUse(U);
-  for (auto &U : make_early_inc_range(SetjmpTableSize->uses()))
-    if (auto *I = dyn_cast<Instruction>(U.getUser()))
-      if (I->getParent() != Entry)
-        SetjmpTableSizeSSA.RewriteUse(U);
-
   // Finally, our modifications to the cfg can break dominance of SSA variables.
   // For example, in this code,
   // if (x()) { .. setjmp() .. }
@@ -1454,21 +1354,13 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runSjLjOnFunction(Function &F) {
 // setjmp. Refer to 4) of "Emscripten setjmp/longjmp handling" section in the
 // comments at top of the file for details.
 void WebAssemblyLowerEmscriptenEHSjLj::handleLongjmpableCallsForEmscriptenSjLj(
-    Function &F, InstVector &SetjmpTableInsts, InstVector &SetjmpTableSizeInsts,
+    Function &F, Instruction *FunctionInvocationId,
     SmallVectorImpl<PHINode *> &SetjmpRetPHIs) {
   Module &M = *F.getParent();
   LLVMContext &C = F.getContext();
   IRBuilder<> IRB(C);
   SmallVector<Instruction *, 64> ToErase;
 
-  // We need to pass setjmpTable and setjmpTableSize to testSetjmp function.
-  // These values are defined in the beginning of the function and also in each
-  // setjmp callsite, but we don't know which values we should use at this
-  // point. So here we arbitraily use the ones defined in the beginning of the
-  // function, and SSAUpdater will later update them to the correct values.
-  Instruction *SetjmpTable = *SetjmpTableInsts.begin();
-  Instruction *SetjmpTableSize = *SetjmpTableSizeInsts.begin();
-
   // call.em.longjmp BB that will be shared within the function.
   BasicBlock *CallEmLongjmpBB = nullptr;
   // PHI node for the loaded value of __THREW__ global variable in
@@ -1601,7 +1493,7 @@ void WebAssemblyLowerEmscriptenEHSjLj::handleLongjmpableCallsForEmscriptenSjLj(
 
           IRB.SetInsertPoint(NormalBB);
           IRB.CreateBr(Tail);
-          BB = NormalBB; // New insertion point to insert testSetjmp()
+          BB = NormalBB; // New insertion point to insert __wasm_setjmp_test()
         }
       }
 
@@ -1610,16 +1502,15 @@ void WebAssemblyLowerEmscriptenEHSjLj::handleLongjmpableCallsForEmscriptenSjLj(
       // right setjmp-tail if so
       ToErase.push_back(BB->getTerminator());
 
-      // Generate a function call to testSetjmp function and preamble/postamble
-      // code to figure out (1) whether longjmp occurred (2) if longjmp
-      // occurred, which setjmp it corresponds to
+      // Generate a function call to __wasm_setjmp_test function and
+      // preamble/postamble code to figure out (1) whether longjmp
+      // occurred (2) if longjmp occurred, which setjmp it corresponds to
       Value *Label = nullptr;
       Value *LongjmpResult = nullptr;
       BasicBlock *EndBB = nullptr;
-      wrapTestSetjmp(BB, CI->getDebugLoc(), Threw, SetjmpTable, SetjmpTableSize,
-                     Label, LongjmpResult, CallEmLongjmpBB,
-                     CallEmLongjmpBBThrewPHI, CallEmLongjmpBBThrewValuePHI,
-                     EndBB);
+      wrapTestSetjmp(BB, CI->getDebugLoc(), Threw, FunctionInvocationId, Label,
+                     LongjmpResult, CallEmLongjmpBB, CallEmLongjmpBBThrewPHI,
+                     CallEmLongjmpBBThrewValuePHI, EndBB);
       assert(Label && LongjmpResult && EndBB);
 
       // Create switch instruction
@@ -1658,7 +1549,7 @@ static BasicBlock *getCleanupRetUnwindDest(const CleanupPadInst *CPI) {
 // BBs. Refer to 4) of "Wasm setjmp/longjmp handling" section in the comments at
 // top of the file for details.
 void WebAssemblyLowerEmscriptenEHSjLj::handleLongjmpableCallsForWasmSjLj(
-    Function &F, InstVector &SetjmpTableInsts, InstVector &SetjmpTableSizeInsts,
+    Function &F, Instruction *FunctionInvocationId,
     SmallVectorImpl<PHINode *> &SetjmpRetPHIs) {
   Module &M = *F.getParent();
   LLVMContext &C = F.getContext();
@@ -1682,18 +1573,13 @@ void WebAssemblyLowerEmscriptenEHSjLj::handleLongjmpableCallsForWasmSjLj(
   DebugLoc FirstDL = getOrCreateDebugLoc(&*Entry->begin(), F.getSubprogram());
   IRB.SetCurrentDebugLocation(FirstDL);
 
-  // Arbitrarily use the ones defined in the beginning of the function.
-  // SSAUpdater will later update them to the correct values.
-  Instruction *SetjmpTable = *SetjmpTableInsts.begin();
-  Instruction *SetjmpTableSize = *SetjmpTableSizeInsts.begin();
-
   // Add setjmp.dispatch BB right after the entry block. Because we have
-  // initialized setjmpTable/setjmpTableSize in the entry block and split the
+  // initialized functionInvocationId in the entry block and split the
   // rest into another BB, here 'OrigEntry' is the function's original entry
   // block before the transformation.
   //
   // entry:
-  //   setjmpTable / setjmpTableSize initialization
+  //   functionInvocationId initialization
   // setjmp.dispatch:
   //   switch will be inserted here later
   // entry.split: (OrigEntry)
@@ -1731,17 +1617,15 @@ void WebAssemblyLowerEmscriptenEHSjLj::handleLongjmpableCallsForWasmSjLj(
   // int val = __wasm_longjmp_args.val;
   Instruction *Val = IRB.CreateLoad(IRB.getInt32Ty(), ValField, "val");
 
-  // %label = testSetjmp(mem[%env], setjmpTable, setjmpTableSize);
+  // %label = __wasm_setjmp_test(%env, functionInvocatinoId);
   // if (%label == 0)
   //   __wasm_longjmp(%env, %val)
   // catchret to %setjmp.dispatch
   BasicBlock *ThenBB = BasicBlock::Create(C, "if.then", &F);
   BasicBlock *EndBB = BasicBlock::Create(C, "if.end", &F);
   Value *EnvP = IRB.CreateBitCast(Env, getAddrPtrType(&M), "env.p");
-  Value *SetjmpID = IRB.CreateLoad(getAddrIntType(&M), EnvP, "setjmp.id");
-  Value *Label =
-      IRB.CreateCall(TestSetjmpF, {SetjmpID, SetjmpTable, SetjmpTableSize},
-                     OperandBundleDef("funclet", CatchPad), "label");
+  Value *Label = IRB.CreateCall(WasmSetjmpTestF, {EnvP, FunctionInvocationId},
+                                OperandBundleDef("funclet", CatchPad), "label");
   Value *Cmp = IRB.CreateICmpEQ(Label, IRB.getInt32(0));
   IRB.CreateCondBr(Cmp, ThenBB, EndBB);
 
diff --git a/llvm/test/CodeGen/WebAssembly/lower-em-ehsjlj-options.ll b/llvm/test/CodeGen/WebAssembly/lower-em-ehsjlj-options.ll
index aa4d87756c87..4a63c812d6ae 100644
--- a/llvm/test/CodeGen/WebAssembly/lower-em-ehsjlj-options.ll
+++ b/llvm/test/CodeGen/WebAssembly/lower-em-ehsjlj-options.ll
@@ -59,12 +59,12 @@ entry:
   %call = call i32 @setjmp(ptr %buf) #0
   call void @longjmp(ptr %buf, i32 1) #1
   unreachable
-; SJLJ: call saveSetjmp
+; SJLJ: call __wasm_setjmp
 ; SJLJ: i32.const emscripten_longjmp
 ; SJLJ-NOT: i32.const emscripten_longjmp_jmpbuf
 ; SJLJ: call invoke_vii
 ; SJLJ-NOT: call "__invoke_void_ptr_i32"
-; SJLJ: call testSetjmp
+; SJLJ: call __wasm_setjmp_test
 
 ; NONE: call setjmp
 ; NONE: call longjmp
diff --git a/llvm/test/CodeGen/WebAssembly/lower-em-ehsjlj.ll b/llvm/test/CodeGen/WebAssembly/lower-em-ehsjlj.ll
index 7cf05cc922cd..32942cd92e68 100644
--- a/llvm/test/CodeGen/WebAssembly/lower-em-ehsjlj.ll
+++ b/llvm/test/CodeGen/WebAssembly/lower-em-ehsjlj.ll
@@ -49,7 +49,7 @@ try.cont:                                         ; preds = %lpad, %entry
 
 ; longjmp checking part
 ; CHECK:    if.then1:
-; CHECK:      call i32 @testSetjmp
+; CHECK:      call i32 @__wasm_setjmp_test
 }
 
 ; @foo can either throw an exception or longjmp. Because this function doesn't
@@ -117,7 +117,6 @@ if.end:                                           ; preds = %entry
 
 ; CHECK:    rethrow.exn:
 ; CHECK-NEXT: %exn = call ptr @__cxa_find_matching_catch_2()
-; CHECK-NEXT: call void @free(ptr %setjmpTable{{.*}})
 ; CHECK-NEXT: call void @__resumeException(ptr %exn)
 ; CHECK-NEXT: unreachable
 
@@ -147,7 +146,6 @@ throw:                                            ; preds = %if.end, %entry
   unreachable
 
 ; CHECK:    throw:
-; CHECK-NEXT: call void @free(ptr %setjmpTable{{.*}})
 ; CHECK-NEXT: call void @__cxa_throw(ptr null, ptr null, ptr null)
 ; CHECK-NEXT: unreachable
 }
@@ -208,7 +206,6 @@ return:                                           ; preds = %entry, %if.end
 
 ; CHECK:    rethrow.exn:
 ; CHECK-NEXT: %exn = call ptr @__cxa_find_matching_catch_2()
-; CHECK-NEXT: tail call void @free(ptr %setjmpTable{{.*}})
 ; CHECK-NEXT: call void @__resumeException(ptr %exn)
 ; CHECK-NEXT: unreachable
 }
diff --git a/llvm/test/CodeGen/WebAssembly/lower-em-sjlj-alias.ll b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj-alias.ll
index 1a85a63e44ad..79ae16191d6b 100644
--- a/llvm/test/CodeGen/WebAssembly/lower-em-sjlj-alias.ll
+++ b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj-alias.ll
@@ -12,7 +12,7 @@ target triple = "wasm32-unknown-emscripten"
 ; CHECK-LABEL: @malloc_test
 define void @malloc_test() {
 entry:
-  ; CHECK: call ptr @malloc
+  ; CHECK: alloca i32
   %retval = alloca i32, align 4
   %jmp = alloca [1 x %struct.__jmp_buf_tag], align 16
   store i32 0, ptr %retval, align 4
diff --git a/llvm/test/CodeGen/WebAssembly/lower-em-sjlj-debuginfo.ll b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj-debuginfo.ll
index 4f694151c761..fec9836a1607 100644
--- a/llvm/test/CodeGen/WebAssembly/lower-em-sjlj-debuginfo.ll
+++ b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj-debuginfo.ll
@@ -16,25 +16,22 @@ entry:
   call void @foo(), !dbg !7
   ret void, !dbg !8
 ; CHECK: entry:
-  ; CHECK-NEXT: call ptr @malloc(i32 40), !dbg ![[DL0:.*]]
+  ; CHECK-NEXT: %functionInvocationId = alloca i32, align 4, !dbg ![[DL0:.*]]
 
 ; CHECK: entry.split:
   ; CHECK: alloca {{.*}}, !dbg ![[DL0]]
-  ; CHECK: call ptr @saveSetjmp{{.*}}, !dbg ![[DL1:.*]]
-  ; CHECK-NEXT: call i32 @getTempRet0{{.*}}, !dbg ![[DL1]]
+  ; CHECK: call void @__wasm_setjmp{{.*}}, !dbg ![[DL1:.*]]
   ; CHECK-NEXT: br {{.*}}, !dbg ![[DL2:.*]]
 
 ; CHECK: entry.split.split:
   ; CHECK: call {{.*}} void @__invoke_void{{.*}}, !dbg ![[DL2]]
 
 ; CHECK: entry.split.split.split:
-  ; CHECK-NEXT: call void @free{{.*}}, !dbg ![[DL3:.*]]
 
 ; CHECK: if.then1:
-  ; CHECK: call i32 @testSetjmp{{.*}}, !dbg ![[DL2]]
+  ; CHECK: call i32 @__wasm_setjmp_test{{.*}}, !dbg ![[DL2]]
 
 ; CHECK: if.end:
-  ; CHECK: call i32 @getTempRet0{{.*}}, !dbg ![[DL2]]
 
 ; CHECK: call.em.longjmp:
   ; CHECK: call void @emscripten_longjmp{{.*}}, !dbg ![[DL2]]
@@ -43,26 +40,6 @@ entry:
   ; CHECK: call void @setTempRet0{{.*}}, !dbg ![[DL2]]
 }
 
-; No instruction has debug info but the current function (setjmp_debug_info2)
-; and the called function (malloc / free) have DISubprograms, so the newly
-; generated calls should have debug info attached. We don't have an instruction
-; to take debug info from, so we create dummy debug info.
-define void @setjmp_debug_info1() !dbg !9 {
-; CHECK-LABEL: @setjmp_debug_info1
-entry:
-  %buf = alloca [1 x %struct.__jmp_buf_tag], align 16
-  %arraydecay = getelementptr inbounds [1 x %struct.__jmp_buf_tag], ptr %buf, i32 0, i32 0
-  %call = call i32 @setjmp(ptr %arraydecay) #0
-  call void @foo()
-  ret void
-  ; CHECK: call ptr @malloc(i32 40), !dbg ![[DL_DUMMY:.*]]
-  ; CHECK: call void @free{{.*}}, !dbg ![[DL_DUMMY]]
-}
-
-; Note that these functions have DISubprograms.
-declare !dbg !10 ptr @malloc(i32)
-declare !dbg !11 void @free(ptr)
-
 declare void @foo()
 ; Function Attrs: returns_twice
 declare i32 @setjmp(ptr) #0
@@ -79,9 +56,3 @@ declare i32 @setjmp(ptr) #0
 !6 = !DILocation(line:4, scope: !3)
 !7 = !DILocation(line:5, scope: !3)
 !8 = !DILocation(line:6, scope: !3)
-!9 = distinct !DISubprogram(name: "setjmp_debug_info1", unit:!2, file: !1, line: 50)
-!10 = !DISubprogram(name: "malloc", file: !1, line: 10, isDefinition: false)
-!11 = !DISubprogram(name: "free", file: !1, line: 20, isDefinition: false)
-
-; Dummy debug info generated
-; CHECK: ![[DL_DUMMY]] = !DILocation(line: 50, column: 1, scope: !9)
diff --git a/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll
index 7115b01ed161..27ec95a2c462 100644
--- a/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll
+++ b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj.ll
@@ -22,15 +22,12 @@ entry:
   call void @longjmp(ptr %buf, i32 1) #1
   unreachable
 ; CHECK: entry:
-; CHECK-NEXT: %[[MALLOCCALL:.*]] = tail call ptr @malloc([[PTR]] 40)
-; CHECK-NEXT: store i32 0, ptr %[[MALLOCCALL]]
-; CHECK-NEXT: %[[SETJMP_TABLE_SIZE:.*]] = add i32 4, 0
+; CHECK-NEXT: %functionInvocationId = alloca i32, align 4
 ; CHECK-NEXT: br label %entry.split
 
 ; CHECK: entry.split
 ; CHECK-NEXT: %[[BUF:.*]] = alloca [1 x %struct.__jmp_buf_tag]
-; CHECK-NEXT: %[[SETJMP_TABLE1:.*]] = call ptr @saveSetjmp(ptr %[[BUF]], i32 1, ptr %[[MALLOCCALL]], i32 %[[SETJMP_TABLE_SIZE]])
-; CHECK-NEXT: %[[SETJMP_TABLE_SIZE1:.*]] = call i32 @getTempRet0()
+; CHECK-NEXT: call void @__wasm_setjmp(ptr %[[BUF]], i32 1, ptr %functionInvocationId)
 ; CHECK-NEXT: br label %entry.split.split
 
 ; CHECK: entry.split.split:
@@ -51,8 +48,7 @@ entry:
 
 ; CHECK: if.then1:
 ; CHECK-NEXT: %[[__THREW__VAL_P:.*]] = inttoptr [[PTR]] %[[__THREW__VAL]] to ptr
-; CHECK-NEXT: %[[__THREW__VAL_P_LOADED:.*]] = load [[PTR]], ptr %[[__THREW__VAL_P]]
-; CHECK-NEXT: %[[LABEL:.*]] = call i32 @testSetjmp([[PTR]] %[[__THREW__VAL_P_LOADED]], ptr %[[SETJMP_TABLE1]], i32 %[[SETJMP_TABLE_SIZE1]])
+; CHECK-NEXT: %[[LABEL:.*]] = call i32 @__wasm_setjmp_test(ptr %[[__THREW__VAL_P]], ptr %functionInvocationId)
 ; CHECK-NEXT: %[[CMP:.*]] = icmp eq i32 %[[LABEL]], 0
 ; CHECK-NEXT: br i1 %[[CMP]], label %call.em.longjmp, label %if.end2
 
@@ -69,7 +65,6 @@ entry:
 ; CHECK: call.em.longjmp:
 ; CHECK-NEXT: %threw.phi = phi [[PTR]] [ %[[__THREW__VAL]], %if.then1 ]
 ; CHECK-NEXT: %threwvalue.phi = phi i32 [ %[[THREWVALUE_VAL]], %if.then1 ]
-; CHECK-NEXT: tail call void @free(ptr %[[SETJMP_TABLE1]])
 ; CHECK-NEXT: call void @emscripten_longjmp([[PTR]] %threw.phi, i32 %threwvalue.phi)
 ; CHECK-NEXT: unreachable
 
@@ -87,13 +82,12 @@ entry:
   call void @foo()
   ret void
 ; CHECK: entry:
-; CHECK: %[[SETJMP_TABLE:.*]] = call ptr @saveSetjmp(
+; CHECK: call void @__wasm_setjmp(
 
 ; CHECK: entry.split.split:
 ; CHECK: @__invoke_void(ptr @foo)
 
 ; CHECK: entry.split.split.split:
-; CHECK-NEXT: tail call void @free(ptr %[[SETJMP_TABLE]])
 ; CHECK-NEXT: ret void
 }
 
@@ -110,9 +104,8 @@ entry:
   call void @foo()
   ret void
 ; CHECK: call.em.longjmp:
-; CHECK-NEXT:  %threw.phi = phi [[PTR]] [ %__THREW__.val, %if.then1 ], [ %__THREW__.val4, %if.then15 ]
-; CHECK-NEXT:  %threwvalue.phi = phi i32 [ %__threwValue.val, %if.then1 ], [ %__threwValue.val8, %if.then15 ]
-; CHECK-NEXT: tail call void @free(ptr %[[SETJMP_TABLE1]])
+; CHECK-NEXT:  %threw.phi = phi [[PTR]] [ %__THREW__.val, %if.then1 ], [ %__THREW__.val2, %if.then13 ]
+; CHECK-NEXT:  %threwvalue.phi = phi i32 [ %__threwValue.val, %if.then1 ], [ %__threwValue.val6, %if.then13 ]
 ; CHECK-NEXT: call void @emscripten_longjmp([[PTR]] %threw.phi, i32 %threwvalue.phi)
 ; CHECK-NEXT: unreachable
 }
@@ -145,7 +138,6 @@ entry:
   %cmp = icmp sgt i32 %n, 5
   br i1 %cmp, label %if.then, label %if.end
 ; CHECK: entry:
-; CHECK: %[[SETJMP_TABLE_SIZE0:.*]] = add i32 4, 0
 
 if.then:                                          ; preds = %entry
   %0 = load i32, ptr @global_var, align 4
@@ -154,13 +146,10 @@ if.then:                                          ; preds = %entry
   br label %if.end
 ; CHECK: if.then:
 ; CHECK: %[[VAR0:.*]] = load i32, ptr @global_var, align 4
-; CHECK: %[[SETJMP_TABLE1:.*]] = call ptr @saveSetjmp(
-; CHECK-NEXT: %[[SETJMP_TABLE_SIZE1:.*]] = call i32 @getTempRet0()
+; CHECK: call void @__wasm_setjmp(
 
 ; CHECK: if.then.split:
-; CHECK: %[[VAR1:.*]] = phi i32 [ %[[VAR2:.*]], %if.end3 ], [ %[[VAR0]], %if.then ]
-; CHECK: %[[SETJMP_TABLE_SIZE2:.*]] = phi i32 [ %[[SETJMP_TABLE_SIZE1]], %if.then ], [ %[[SETJMP_TABLE_SIZE3:.*]], %if.end3 ]
-; CHECK: %[[SETJMP_TABLE2:.*]] = phi ptr [ %[[SETJMP_TABLE1]], %if.then ], [ %[[SETJMP_TABLE3:.*]], %if.end3 ]
+; CHECK: %[[VAR1:.*]] = phi i32 [ %[[VAR2:.*]], %if.end1 ], [ %[[VAR0]], %if.then ]
 ; CHECK: store i32 %[[VAR1]], ptr @global_var, align 4
 
 if.end:                                           ; preds = %if.then, %entry
@@ -168,8 +157,6 @@ if.end:                                           ; preds = %if.then, %entry
   unreachable
 ; CHECK: if.end:
 ; CHECK: %[[VAR2]] = phi i32 [ %[[VAR1]], %if.then.split ], [ undef, %entry.split ]
-; CHECK: %[[SETJMP_TABLE_SIZE3]] = phi i32 [ %[[SETJMP_TABLE_SIZE2]], %if.then.split ], [ %[[SETJMP_TABLE_SIZE0]], %entry.split ]
-; CHECK: %[[SETJMP_TABLE3]] = phi ptr [ %[[SETJMP_TABLE2]], %if.then.split ], [ %setjmpTable, %entry.split ]
 }
 
 ; Test a case when a function only calls other functions that are neither setjmp nor longjmp
@@ -296,8 +283,8 @@ declare void @free(ptr)
 ; JS glue functions and invoke wrappers declaration
 ; CHECK-DAG: declare i32 @getTempRet0()
 ; CHECK-DAG: declare void @setTempRet0(i32)
-; CHECK-DAG: declare ptr @saveSetjmp(ptr, i32, ptr, i32)
-; CHECK-DAG: declare i32 @testSetjmp([[PTR]], ptr, i32)
+; CHECK-DAG: declare void @__wasm_setjmp(ptr, i32, ptr)
+; CHECK-DAG: declare i32 @__wasm_setjmp_test(ptr, ptr)
 ; CHECK-DAG: declare void @emscripten_longjmp([[PTR]], i32)
 ; CHECK-DAG: declare void @__invoke_void(ptr)
 
@@ -308,8 +295,8 @@ attributes #3 = { allocsize(0) }
 ; CHECK-DAG: attributes #{{[0-9]+}} = { nounwind "wasm-import-module"="env" "wasm-import-name"="getTempRet0" }
 ; CHECK-DAG: attributes #{{[0-9]+}} = { nounwind "wasm-import-module"="env" "wasm-import-name"="setTempRet0" }
 ; CHECK-DAG: attributes #{{[0-9]+}} = { "wasm-import-module"="env" "wasm-import-name"="__invoke_void" }
-; CHECK-DAG: attributes #{{[0-9]+}} = { "wasm-import-module"="env" "wasm-import-name"="saveSetjmp" }
-; CHECK-DAG: attributes #{{[0-9]+}} = { "wasm-import-module"="env" "wasm-import-name"="testSetjmp" }
+; CHECK-DAG: attributes #{{[0-9]+}} = { "wasm-import-module"="env" "wasm-import-name"="__wasm_setjmp" }
+; CHECK-DAG: attributes #{{[0-9]+}} = { "wasm-import-module"="env" "wasm-import-name"="__wasm_setjmp_test" }
 ; CHECK-DAG: attributes #{{[0-9]+}} = { noreturn "wasm-import-module"="env" "wasm-import-name"="emscripten_longjmp" }
 ; CHECK-DAG: attributes #{{[0-9]+}} = { "wasm-import-module"="env" "wasm-import-name"="__invoke_ptr_i32_ptr" }
 ; CHECK-DAG: attributes #[[ALLOCSIZE_ATTR]] = { allocsize(1) }
diff --git a/llvm/test/CodeGen/WebAssembly/lower-wasm-ehsjlj.ll b/llvm/test/CodeGen/WebAssembly/lower-wasm-ehsjlj.ll
index 25471eb50081..bd8db83a0e57 100644
--- a/llvm/test/CodeGen/WebAssembly/lower-wasm-ehsjlj.ll
+++ b/llvm/test/CodeGen/WebAssembly/lower-wasm-ehsjlj.ll
@@ -108,8 +108,8 @@ catch:                                            ; preds = %catch.start
   call void @__cxa_end_catch() [ "funclet"(token %2) ]
   catchret from %2 to label %catchret.dest
 ; CHECK: catch:                                            ; preds = %catch.start
-; CHECK-NEXT:   %exn = load ptr, ptr %exn.slot15, align 4
-; CHECK-NEXT:   %5 = call ptr @__cxa_begin_catch(ptr %exn) #7 [ "funclet"(token %2) ]
+; CHECK-NEXT:   %exn = load ptr, ptr %exn.slot6, align 4
+; CHECK-NEXT:   %5 = call ptr @__cxa_begin_catch(ptr %exn) #6 [ "funclet"(token %2) ]
 ; CHECK-NEXT:   invoke void @__cxa_end_catch() [ "funclet"(token %2) ]
 ; CHECK-NEXT:           to label %.noexc unwind label %catch.dispatch.longjmp
 
@@ -265,7 +265,7 @@ ehcleanup:                                        ; preds = %entry
   ; (cleanuppad), whose parent is 'none', so we should unwind directly to
   ; %catch.dispatch.longjmp.
   %call2 = call noundef ptr @_ZN4TempD2Ev(ptr noundef %t) #2 [ "funclet"(token %0) ]
-; CHECK: %call13 = invoke {{.*}} ptr @_ZN4TempD2Ev(ptr
+; CHECK: %call11 = invoke {{.*}} ptr @_ZN4TempD2Ev(ptr
 ; CHECK-NEXT:    to label {{.*}} unwind label %catch.dispatch.longjmp
   cleanupret from %0 unwind to caller
 }
diff --git a/llvm/test/CodeGen/WebAssembly/lower-wasm-sjlj.ll b/llvm/test/CodeGen/WebAssembly/lower-wasm-sjlj.ll
index b8d2230fac9f..82c04e24b72f 100644
--- a/llvm/test/CodeGen/WebAssembly/lower-wasm-sjlj.ll
+++ b/llvm/test/CodeGen/WebAssembly/lower-wasm-sjlj.ll
@@ -25,16 +25,12 @@ entry:
   unreachable
 
 ; CHECK:    entry:
-; CHECK-NEXT: %setjmpTable = tail call ptr @malloc([[PTR]] 40)
-; CHECK-NEXT: store i32 0, ptr %setjmpTable, align 4
-; CHECK-NEXT: %setjmpTableSize = add i32 4, 0
+; CHECK-NEXT: %functionInvocationId = alloca i32, align 4
 ; CHECK-NEXT: br label %setjmp.dispatch
 
 ; CHECK:    setjmp.dispatch:
 ; CHECK-NEXT: %[[VAL2:.*]] = phi i32 [ %val, %if.end ], [ undef, %entry ]
 ; CHECK-NEXT: %[[BUF:.*]] = phi ptr [ %[[BUF2:.*]], %if.end ], [ undef, %entry ]
-; CHECK-NEXT: %[[SETJMPTABLESIZE2:.*]] = phi i32 [ %[[SETJMPTABLESIZE3:.*]], %if.end ], [ %setjmpTableSize, %entry ]
-; CHECK-NEXT: %[[SETJMPTABLE2:.*]] = phi ptr [ %[[SETJMPTABLE3:.*]], %if.end ], [ %setjmpTable, %entry ]
 ; CHECK-NEXT: %label.phi = phi i32 [ %label, %if.end ], [ -1, %entry ]
 ; CHECK-NEXT: switch i32 %label.phi, label %entry.split [
 ; CHECK-NEXT:   i32 1, label %entry.split.split
@@ -42,14 +38,11 @@ entry:
 
 ; CHECK:    entry.split:
 ; CHECK-NEXT: %buf = alloca [1 x %struct.__jmp_buf_tag], align 16
-; CHECK-NEXT: %[[SETJMPTABLE4:.*]] = call ptr @saveSetjmp(ptr %buf, i32 1, ptr %[[SETJMPTABLE2]], i32 %[[SETJMPTABLESIZE2]])
-; CHECK-NEXT: %[[SETJMPTABLESIZE4:.*]] = call i32 @getTempRet0()
+; CHECK-NEXT: call void @__wasm_setjmp(ptr %buf, i32 1, ptr %functionInvocationId)
 ; CHECK-NEXT: br label %entry.split.split
 
 ; CHECK:    entry.split.split:
 ; CHECK-NEXT: %[[BUF2]] = phi ptr [ %[[BUF]], %setjmp.dispatch ], [ %buf, %entry.split ]
-; CHECK-NEXT: %[[SETJMPTABLESIZE3]] = phi i32 [ %[[SETJMPTABLESIZE4]], %entry.split ], [ %[[SETJMPTABLESIZE2]], %setjmp.dispatch ]
-; CHECK-NEXT: %[[SETJMPTABLE3]] = phi ptr [ %[[SETJMPTABLE4]], %entry.split ], [ %[[SETJMPTABLE2]], %setjmp.dispatch ]
 ; CHECK-NEXT: %setjmp.ret = phi i32 [ 0, %entry.split ], [ %[[VAL2]], %setjmp.dispatch ]
 ; CHECK-NEXT: invoke void @__wasm_longjmp(ptr %[[BUF2]], i32 1)
 ; CHECK-NEXT:         to label %.noexc unwind label %catch.dispatch.longjmp
@@ -67,13 +60,11 @@ entry:
 ; CHECK-NEXT: %val_gep = getelementptr { ptr, i32 }, ptr %thrown, i32 0, i32 1
 ; CHECK-NEXT: %env = load ptr, ptr %env_gep, align {{.*}}
 ; CHECK-NEXT: %val = load i32, ptr %val_gep, align 4
-; CHECK-NEXT: %setjmp.id = load [[PTR]], ptr %env, align {{.*}}
-; CHECK-NEXT: %label = call i32 @testSetjmp([[PTR]] %setjmp.id, ptr %[[SETJMPTABLE3]], i32 %[[SETJMPTABLESIZE3]]) [ "funclet"(token %1) ]
+; CHECK-NEXT: %label = call i32 @__wasm_setjmp_test(ptr %env, ptr %functionInvocationId) [ "funclet"(token %1) ]
 ; CHECK-NEXT: %2 = icmp eq i32 %label, 0
 ; CHECK-NEXT: br i1 %2, label %if.then, label %if.end
 
 ; CHECK:    if.then:
-; CHECK-NEXT: tail call void @free(ptr %[[SETJMPTABLE3]]) [ "funclet"(token %1) ]
 ; CHECK-NEXT: call void @__wasm_longjmp(ptr %env, i32 %val) [ "funclet"(token %1) ]
 ; CHECK-NEXT: unreachable
 
@@ -142,10 +133,9 @@ declare ptr @__cxa_begin_catch(ptr)
 declare void @__cxa_end_catch()
 declare void @free(ptr)
 
-; JS glue function declarations
-; CHECK-DAG: declare i32 @getTempRet0()
-; CHECK-DAG: declare ptr @saveSetjmp(ptr, i32, ptr, i32)
-; CHECK-DAG: declare i32 @testSetjmp([[PTR]], ptr, i32)
+; Runtime glue function declarations
+; CHECK-DAG: declare void @__wasm_setjmp(ptr, i32, ptr)
+; CHECK-DAG: declare i32 @__wasm_setjmp_test(ptr, ptr)
 ; CHECK-DAG: declare void @__wasm_longjmp(ptr, i32)
 
 attributes #0 = { returns_twice }
-- 
GitLab


From 9c8dd5e6f6bd93deb95de9642632223f54a18a11 Mon Sep 17 00:00:00 2001
From: Longsheng Mou <moulongsheng@huawei.com>
Date: Tue, 26 Mar 2024 09:19:42 +0800
Subject: [PATCH 231/404] [X86_64] fix SSE type error in vaarg. (#86377)

tweak the position of the ++neededSSE when Lo is NoClass and Hi is SSE.
Fix #86371.
---
 clang/lib/CodeGen/Targets/X86.cpp      |  3 +-
 clang/test/CodeGen/X86/x86_64-vaarg.c  | 69 ++++++++++++++++++++++++++
 clang/test/CodeGenCXX/x86_64-vaarg.cpp | 50 +++++++++++++++++--
 3 files changed, 117 insertions(+), 5 deletions(-)
 create mode 100644 clang/test/CodeGen/X86/x86_64-vaarg.c

diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index 1ec0f159ebcb..6931768147a9 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -2788,12 +2788,11 @@ X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned freeIntRegs,
     // memory), except in situations involving unions.
   case X87Up:
   case SSE:
+    ++neededSSE;
     HighPart = GetSSETypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);
 
     if (Lo == NoClass)  // Pass HighPart at offset 8 in memory.
       return ABIArgInfo::getDirect(HighPart, 8);
-
-    ++neededSSE;
     break;
 
     // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
diff --git a/clang/test/CodeGen/X86/x86_64-vaarg.c b/clang/test/CodeGen/X86/x86_64-vaarg.c
new file mode 100644
index 000000000000..a18ba8364238
--- /dev/null
+++ b/clang/test/CodeGen/X86/x86_64-vaarg.c
@@ -0,0 +1,69 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -o - %s | FileCheck %s
+
+
+typedef struct { struct {} a; } empty;
+
+// CHECK-LABEL: define dso_local void @empty_record_test(
+// CHECK-SAME: i32 noundef [[Z:%.*]], ...) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_EMPTY:%.*]], align 1
+// CHECK-NEXT:    [[Z_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[LIST:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
+// CHECK-NEXT:    [[TMP:%.*]] = alloca [[STRUCT_EMPTY]], align 1
+// CHECK-NEXT:    store i32 [[Z]], ptr [[Z_ADDR]], align 4
+// CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[LIST]], i64 0, i64 0
+// CHECK-NEXT:    call void @llvm.va_start(ptr [[ARRAYDECAY]])
+// CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[LIST]], i64 0, i64 0
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[RETVAL]], ptr align 1 [[TMP]], i64 0, i1 false)
+// CHECK-NEXT:    ret void
+//
+empty empty_record_test(int z, ...) {
+  __builtin_va_list list;
+  __builtin_va_start(list, z);
+  return __builtin_va_arg(list, empty);
+}
+
+typedef struct {
+  struct{} a;
+  double b;
+} s1;
+
+// CHECK-LABEL: define dso_local double @f(
+// CHECK-SAME: i32 noundef [[Z:%.*]], ...) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_S1:%.*]], align 8
+// CHECK-NEXT:    [[Z_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[LIST:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
+// CHECK-NEXT:    store i32 [[Z]], ptr [[Z_ADDR]], align 4
+// CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[LIST]], i64 0, i64 0
+// CHECK-NEXT:    call void @llvm.va_start(ptr [[ARRAYDECAY]])
+// CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[LIST]], i64 0, i64 0
+// CHECK-NEXT:    [[FP_OFFSET_P:%.*]] = getelementptr inbounds [[STRUCT___VA_LIST_TAG:%.*]], ptr [[ARRAYDECAY1]], i32 0, i32 1
+// CHECK-NEXT:    [[FP_OFFSET:%.*]] = load i32, ptr [[FP_OFFSET_P]], align 4
+// CHECK-NEXT:    [[FITS_IN_FP:%.*]] = icmp ule i32 [[FP_OFFSET]], 160
+// CHECK-NEXT:    br i1 [[FITS_IN_FP]], label [[VAARG_IN_REG:%.*]], label [[VAARG_IN_MEM:%.*]]
+// CHECK:       vaarg.in_reg:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[STRUCT___VA_LIST_TAG]], ptr [[ARRAYDECAY1]], i32 0, i32 3
+// CHECK-NEXT:    [[REG_SAVE_AREA:%.*]] = load ptr, ptr [[TMP0]], align 16
+// CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[REG_SAVE_AREA]], i32 [[FP_OFFSET]]
+// CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[FP_OFFSET]], 16
+// CHECK-NEXT:    store i32 [[TMP2]], ptr [[FP_OFFSET_P]], align 4
+// CHECK-NEXT:    br label [[VAARG_END:%.*]]
+// CHECK:       vaarg.in_mem:
+// CHECK-NEXT:    [[OVERFLOW_ARG_AREA_P:%.*]] = getelementptr inbounds [[STRUCT___VA_LIST_TAG]], ptr [[ARRAYDECAY1]], i32 0, i32 2
+// CHECK-NEXT:    [[OVERFLOW_ARG_AREA:%.*]] = load ptr, ptr [[OVERFLOW_ARG_AREA_P]], align 8
+// CHECK-NEXT:    [[OVERFLOW_ARG_AREA_NEXT:%.*]] = getelementptr i8, ptr [[OVERFLOW_ARG_AREA]], i32 8
+// CHECK-NEXT:    store ptr [[OVERFLOW_ARG_AREA_NEXT]], ptr [[OVERFLOW_ARG_AREA_P]], align 8
+// CHECK-NEXT:    br label [[VAARG_END]]
+// CHECK:       vaarg.end:
+// CHECK-NEXT:    [[VAARG_ADDR:%.*]] = phi ptr [ [[TMP1]], [[VAARG_IN_REG]] ], [ [[OVERFLOW_ARG_AREA]], [[VAARG_IN_MEM]] ]
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[RETVAL]], ptr align 8 [[VAARG_ADDR]], i64 8, i1 false)
+// CHECK-NEXT:    [[TMP3:%.*]] = load double, ptr [[RETVAL]], align 8
+// CHECK-NEXT:    ret double [[TMP3]]
+//
+s1 f(int z, ...) {
+  __builtin_va_list list;
+  __builtin_va_start(list, z);
+  return __builtin_va_arg(list, s1);
+}
diff --git a/clang/test/CodeGenCXX/x86_64-vaarg.cpp b/clang/test/CodeGenCXX/x86_64-vaarg.cpp
index f0177906a09a..d221c1881d36 100644
--- a/clang/test/CodeGenCXX/x86_64-vaarg.cpp
+++ b/clang/test/CodeGenCXX/x86_64-vaarg.cpp
@@ -1,10 +1,9 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -x c -o - %s | FileCheck %s
 
 typedef struct { struct {} a; } empty;
 
-// CHECK-LABEL: @{{.*}}empty_record_test
+// CHECK-LABEL: @_Z17empty_record_testiz(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_EMPTY:%.*]], align 1
 // CHECK-NEXT:    [[Z_ADDR:%.*]] = alloca i32, align 4
@@ -14,10 +13,55 @@ typedef struct { struct {} a; } empty;
 // CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[LIST]], i64 0, i64 0
 // CHECK-NEXT:    call void @llvm.va_start(ptr [[ARRAYDECAY]])
 // CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[LIST]], i64 0, i64 0
-// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[RETVAL]], ptr align 1 [[TMP]], i64 {{.*}}, i1 false)
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[RETVAL]], ptr align 1 [[TMP]], i64 1, i1 false)
 // CHECK-NEXT:    ret void
+//
 empty empty_record_test(int z, ...) {
   __builtin_va_list list;
   __builtin_va_start(list, z);
   return __builtin_va_arg(list, empty);
 }
+
+typedef struct {
+  struct{} a;
+  double b;
+} s1;
+
+// CHECK-LABEL: @_Z1fiz(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca [[STRUCT_S1:%.*]], align 8
+// CHECK-NEXT:    [[Z_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[LIST:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
+// CHECK-NEXT:    store i32 [[Z:%.*]], ptr [[Z_ADDR]], align 4
+// CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[LIST]], i64 0, i64 0
+// CHECK-NEXT:    call void @llvm.va_start(ptr [[ARRAYDECAY]])
+// CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[LIST]], i64 0, i64 0
+// CHECK-NEXT:    [[FP_OFFSET_P:%.*]] = getelementptr inbounds [[STRUCT___VA_LIST_TAG:%.*]], ptr [[ARRAYDECAY1]], i32 0, i32 1
+// CHECK-NEXT:    [[FP_OFFSET:%.*]] = load i32, ptr [[FP_OFFSET_P]], align 4
+// CHECK-NEXT:    [[FITS_IN_FP:%.*]] = icmp ule i32 [[FP_OFFSET]], 160
+// CHECK-NEXT:    br i1 [[FITS_IN_FP]], label [[VAARG_IN_REG:%.*]], label [[VAARG_IN_MEM:%.*]]
+// CHECK:       vaarg.in_reg:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[STRUCT___VA_LIST_TAG]], ptr [[ARRAYDECAY1]], i32 0, i32 3
+// CHECK-NEXT:    [[REG_SAVE_AREA:%.*]] = load ptr, ptr [[TMP0]], align 16
+// CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[REG_SAVE_AREA]], i32 [[FP_OFFSET]]
+// CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[FP_OFFSET]], 16
+// CHECK-NEXT:    store i32 [[TMP2]], ptr [[FP_OFFSET_P]], align 4
+// CHECK-NEXT:    br label [[VAARG_END:%.*]]
+// CHECK:       vaarg.in_mem:
+// CHECK-NEXT:    [[OVERFLOW_ARG_AREA_P:%.*]] = getelementptr inbounds [[STRUCT___VA_LIST_TAG]], ptr [[ARRAYDECAY1]], i32 0, i32 2
+// CHECK-NEXT:    [[OVERFLOW_ARG_AREA:%.*]] = load ptr, ptr [[OVERFLOW_ARG_AREA_P]], align 8
+// CHECK-NEXT:    [[OVERFLOW_ARG_AREA_NEXT:%.*]] = getelementptr i8, ptr [[OVERFLOW_ARG_AREA]], i32 16
+// CHECK-NEXT:    store ptr [[OVERFLOW_ARG_AREA_NEXT]], ptr [[OVERFLOW_ARG_AREA_P]], align 8
+// CHECK-NEXT:    br label [[VAARG_END]]
+// CHECK:       vaarg.end:
+// CHECK-NEXT:    [[VAARG_ADDR:%.*]] = phi ptr [ [[TMP1]], [[VAARG_IN_REG]] ], [ [[OVERFLOW_ARG_AREA]], [[VAARG_IN_MEM]] ]
+// CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[RETVAL]], ptr align 8 [[VAARG_ADDR]], i64 16, i1 false)
+// CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[RETVAL]], i64 8
+// CHECK-NEXT:    [[TMP4:%.*]] = load double, ptr [[TMP3]], align 8
+// CHECK-NEXT:    ret double [[TMP4]]
+//
+s1 f(int z, ...) {
+  __builtin_va_list list;
+  __builtin_va_start(list, z);
+  return __builtin_va_arg(list, s1);
+}
-- 
GitLab


From ce73b1672a6053d5974dc2342881aac02efe2dbb Mon Sep 17 00:00:00 2001
From: dong jianqiang <dongjianqiang2@huawei.com>
Date: Tue, 26 Mar 2024 09:23:28 +0800
Subject: [PATCH 232/404] [CodeExtractor] Resolving the Inconsistency of
 Compiled Binary Files (#86497)

When the compiler enables ALSR by default, binary inconsistency occurs
when the extractCodeRegion function is called. The reason is that the
sequence of traversing ExitBlocks containers of the SmallPtrSet type is
changed due to randomization of BasicBlock pointers.
This fixes https://github.com/llvm/llvm-project/issues/86427
---
 llvm/include/llvm/Transforms/Utils/CodeExtractor.h | 2 +-
 llvm/lib/Transforms/Utils/CodeExtractor.cpp        | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Utils/CodeExtractor.h b/llvm/include/llvm/Transforms/Utils/CodeExtractor.h
index 27b34ef023db..333ed6774d6c 100644
--- a/llvm/include/llvm/Transforms/Utils/CodeExtractor.h
+++ b/llvm/include/llvm/Transforms/Utils/CodeExtractor.h
@@ -249,7 +249,7 @@ public:
                        Instruction *Addr, BasicBlock *ExitBlock) const;
 
     void severSplitPHINodesOfEntry(BasicBlock *&Header);
-    void severSplitPHINodesOfExits(const SmallPtrSetImpl<BasicBlock *> &Exits);
+    void severSplitPHINodesOfExits(const SetVector<BasicBlock *> &Exits);
     void splitReturnBlocks();
 
     Function *constructFunction(const ValueSet &inputs,
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 3191751d92e1..6988292ac715 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -745,7 +745,7 @@ void CodeExtractor::severSplitPHINodesOfEntry(BasicBlock *&Header) {
 /// and other with remaining incoming blocks; then first PHIs are placed in
 /// outlined region.
 void CodeExtractor::severSplitPHINodesOfExits(
-    const SmallPtrSetImpl<BasicBlock *> &Exits) {
+    const SetVector<BasicBlock *> &Exits) {
   for (BasicBlock *ExitBB : Exits) {
     BasicBlock *NewBB = nullptr;
 
@@ -1751,7 +1751,7 @@ CodeExtractor::extractCodeRegion(const CodeExtractorAnalysisCache &CEAC,
   // Calculate the exit blocks for the extracted region and the total exit
   // weights for each of those blocks.
   DenseMap<BasicBlock *, BlockFrequency> ExitWeights;
-  SmallPtrSet<BasicBlock *, 1> ExitBlocks;
+  SetVector<BasicBlock *> ExitBlocks;
   for (BasicBlock *Block : Blocks) {
     for (BasicBlock *Succ : successors(Block)) {
       if (!Blocks.count(Succ)) {
-- 
GitLab


From 3e664ac9cd83a2f466e40d1167c08c71a719500e Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 25 Mar 2024 18:21:30 -0700
Subject: [PATCH 233/404] [RISCV] Use inheritance instead of instantiating
 multiclasses inside another multiclass. NFC

---
 llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
index 4cf64e5b09de..8725d2d0e3bd 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
@@ -253,10 +253,8 @@ multiclass VPseudoVALU_S_NoMask_Zvk {
   }
 }
 
-multiclass VPseudoVALU_V_S_NoMask_Zvk {
-  defm "" : VPseudoVALU_V_NoMask_Zvk;
-  defm "" : VPseudoVALU_S_NoMask_Zvk;
-}
+multiclass VPseudoVALU_V_S_NoMask_Zvk
+  : VPseudoVALU_V_NoMask_Zvk, VPseudoVALU_S_NoMask_Zvk;
 
 multiclass VPseudoVALU_VV_NoMask_Zvk {
   foreach m = MxListVF4 in {
-- 
GitLab


From 93f9fb2c825dba48db64d5f726b54bcbd4766009 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 25 Mar 2024 18:37:30 -0700
Subject: [PATCH 234/404] [RISCV] Rename Binary->Ternary and Unary->Binary for
 some cases in RISCVInstrInfoZvk.td. NFC

For instructions that use vd as an input and not just a merge, we
weren't including it in the operand count for the naming.
---
 llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td | 28 +++++++++++-----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
index 8725d2d0e3bd..ff69b0665617 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
@@ -180,7 +180,7 @@ class ZvkMxSet<string vd_lmul> {
                                    !eq(vd_lmul, "MF8") : [V_MF8]);
 }
 
-class VPseudoUnaryNoMask_Zvk<DAGOperand RetClass, VReg OpClass> :
+class VPseudoBinaryNoMask_Zvk<DAGOperand RetClass, VReg OpClass> :
       Pseudo<(outs RetClass:$rd_wb),
         (ins RetClass:$rd, OpClass:$rs2, AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
         RISCVVPseudo {
@@ -194,9 +194,9 @@ class VPseudoUnaryNoMask_Zvk<DAGOperand RetClass, VReg OpClass> :
   let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
 }
 
-class VPseudoBinaryNoMask_Zvk<VReg RetClass,
-                              VReg Op1Class,
-                              DAGOperand Op2Class> :
+class VPseudoTernaryNoMask_Zvk<VReg RetClass,
+                               VReg Op1Class,
+                               DAGOperand Op2Class> :
         Pseudo<(outs RetClass:$rd_wb),
                (ins RetClass:$rd, Op1Class:$rs2, Op2Class:$rs1,
                     AVL:$vl, ixlenimm:$sew, ixlenimm:$policy), []>,
@@ -211,24 +211,24 @@ class VPseudoBinaryNoMask_Zvk<VReg RetClass,
   let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
 }
 
-multiclass VPseudoBinaryNoMask_Zvk<VReg RetClass,
+multiclass VPseudoTernaryNoMask_Zvk<VReg RetClass,
                                    VReg Op1Class,
                                    DAGOperand Op2Class,
                                    LMULInfo MInfo> {
   let VLMul = MInfo.value in
-    def "_" # MInfo.MX : VPseudoBinaryNoMask_Zvk<RetClass, Op1Class, Op2Class>;
+    def "_" # MInfo.MX : VPseudoTernaryNoMask_Zvk<RetClass, Op1Class, Op2Class>;
 }
 
-multiclass VPseudoUnaryV_V_NoMask_Zvk<LMULInfo m> {
+multiclass VPseudoBinaryV_V_NoMask_Zvk<LMULInfo m> {
   let VLMul = m.value in {
-    def "_VV_" # m.MX : VPseudoUnaryNoMask_Zvk<m.vrclass, m.vrclass>;
+    def "_VV_" # m.MX : VPseudoBinaryNoMask_Zvk<m.vrclass, m.vrclass>;
   }
 }
 
-multiclass VPseudoUnaryV_S_NoMask_Zvk<LMULInfo m> {
+multiclass VPseudoBinaryV_S_NoMask_Zvk<LMULInfo m> {
   let VLMul = m.value in
     foreach vs2_lmul = ZvkMxSet<m.MX>.vs2_lmuls in
-      def "_VS_" # m.MX # "_" # vs2_lmul.MX : VPseudoUnaryNoMask_Zvk<m.vrclass, vs2_lmul.vrclass>;
+      def "_VS_" # m.MX # "_" # vs2_lmul.MX : VPseudoBinaryNoMask_Zvk<m.vrclass, vs2_lmul.vrclass>;
 }
 
 multiclass VPseudoVALU_V_NoMask_Zvk {
@@ -237,7 +237,7 @@ multiclass VPseudoVALU_V_NoMask_Zvk {
     defvar WriteVIALUV_MX = !cast<SchedWrite>("WriteVIALUV_" # mx);
     defvar ReadVIALUV_MX = !cast<SchedRead>("ReadVIALUV_" # mx);
 
-    defm "" : VPseudoUnaryV_V_NoMask_Zvk<m>,
+    defm "" : VPseudoBinaryV_V_NoMask_Zvk<m>,
               Sched<[WriteVIALUV_MX, ReadVIALUV_MX, ReadVIALUV_MX, ReadVMask]>;
   }
 }
@@ -248,7 +248,7 @@ multiclass VPseudoVALU_S_NoMask_Zvk {
     defvar WriteVIALUV_MX = !cast<SchedWrite>("WriteVIALUV_" # mx);
     defvar ReadVIALUV_MX = !cast<SchedRead>("ReadVIALUV_" # mx);
 
-    defm "" : VPseudoUnaryV_S_NoMask_Zvk<m>,
+    defm "" : VPseudoBinaryV_S_NoMask_Zvk<m>,
               Sched<[WriteVIALUV_MX, ReadVIALUV_MX, ReadVIALUV_MX, ReadVMask]>;
   }
 }
@@ -262,7 +262,7 @@ multiclass VPseudoVALU_VV_NoMask_Zvk {
     defvar WriteVIALUV_MX = !cast<SchedWrite>("WriteVIALUV_" # mx);
     defvar ReadVIALUV_MX = !cast<SchedRead>("ReadVIALUV_" # mx);
 
-    defm _VV : VPseudoBinaryNoMask_Zvk<m.vrclass, m.vrclass, m.vrclass, m>,
+    defm _VV : VPseudoTernaryNoMask_Zvk<m.vrclass, m.vrclass, m.vrclass, m>,
                Sched<[WriteVIALUV_MX, ReadVIALUV_MX, ReadVIALUV_MX, ReadVMask]>;
   }
 }
@@ -273,7 +273,7 @@ multiclass VPseudoVALU_VI_NoMask_Zvk {
     defvar WriteVIALUV_MX = !cast<SchedWrite>("WriteVIALUV_" # mx);
     defvar ReadVIALUV_MX = !cast<SchedRead>("ReadVIALUV_" # mx);
 
-    defm _VI : VPseudoBinaryNoMask_Zvk<m.vrclass, m.vrclass, uimm5, m>,
+    defm _VI : VPseudoTernaryNoMask_Zvk<m.vrclass, m.vrclass, uimm5, m>,
                Sched<[WriteVIALUV_MX, ReadVIALUV_MX, ReadVIALUV_MX, ReadVMask]>;
   }
 }
-- 
GitLab


From cc3b6f94952e1184df44c2b039d21665116151ed Mon Sep 17 00:00:00 2001
From: Finn Plummer <50529406+inbelic@users.noreply.github.com>
Date: Mon, 25 Mar 2024 19:49:12 -0700
Subject: [PATCH 235/404] [mlir][spirv] Add folding for [S|U|GreaterThan[Equal]
 (#85434)

Add missing constant propogation folder for [S|U]GreaterThan[Equal].

Implement additional folding when the operands are equal for all ops.

Allows for constant folding in the IndexToSPIRV pass.

Part of work #70704
---
 .../mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td  |   8 +
 .../SPIRV/IR/SPIRVCanonicalization.cpp        |  84 +++++++++
 .../SPIRV/Transforms/canonicalize.mlir        | 176 ++++++++++++++++++
 3 files changed, 268 insertions(+)

diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td
index 14d639bc26f2..61c5a7a6394f 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVLogicalOps.td
@@ -659,6 +659,8 @@ def SPIRV_SGreaterThanOp : SPIRV_LogicalBinaryOp<"SGreaterThan",
 
     ```
   }];
+
+  let hasFolder = 1;
 }
 
 // -----
@@ -688,6 +690,8 @@ def SPIRV_SGreaterThanEqualOp : SPIRV_LogicalBinaryOp<"SGreaterThanEqual",
     %5 = spirv.SGreaterThanEqual %2, %3 : vector<4xi32>
     ```
   }];
+
+  let hasFolder = 1;
 }
 
 // -----
@@ -834,6 +838,8 @@ def SPIRV_UGreaterThanOp : SPIRV_LogicalBinaryOp<"UGreaterThan",
     %5 = spirv.UGreaterThan %2, %3 : vector<4xi32>
     ```
   }];
+
+  let hasFolder = 1;
 }
 
 // -----
@@ -863,6 +869,8 @@ def SPIRV_UGreaterThanEqualOp : SPIRV_LogicalBinaryOp<"UGreaterThanEqual",
     %5 = spirv.UGreaterThanEqual %2, %3 : vector<4xi32>
     ```
   }];
+
+  let hasFolder = 1;
 }
 
 // -----
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
index 3f1a7826f8b6..5f47cff71cba 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp
@@ -880,6 +880,90 @@ OpFoldResult spirv::INotEqualOp::fold(spirv::INotEqualOp::FoldAdaptor adaptor) {
       });
 }
 
+//===----------------------------------------------------------------------===//
+// spirv.SGreaterThan
+//===----------------------------------------------------------------------===//
+
+OpFoldResult
+spirv::SGreaterThanOp::fold(spirv::SGreaterThanOp::FoldAdaptor adaptor) {
+  // x == x -> false
+  if (getOperand1() == getOperand2()) {
+    auto falseAttr = BoolAttr::get(getContext(), false);
+    if (isa<IntegerType>(getType()))
+      return falseAttr;
+    if (auto vecTy = dyn_cast<VectorType>(getType()))
+      return SplatElementsAttr::get(vecTy, falseAttr);
+  }
+
+  return constFoldBinaryOp<IntegerAttr>(
+      adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) {
+        return a.sgt(b) ? APInt::getAllOnes(1) : APInt::getZero(1);
+      });
+}
+
+//===----------------------------------------------------------------------===//
+// spirv.SGreaterThanEqual
+//===----------------------------------------------------------------------===//
+
+OpFoldResult spirv::SGreaterThanEqualOp::fold(
+    spirv::SGreaterThanEqualOp::FoldAdaptor adaptor) {
+  // x == x -> true
+  if (getOperand1() == getOperand2()) {
+    auto trueAttr = BoolAttr::get(getContext(), true);
+    if (isa<IntegerType>(getType()))
+      return trueAttr;
+    if (auto vecTy = dyn_cast<VectorType>(getType()))
+      return SplatElementsAttr::get(vecTy, trueAttr);
+  }
+
+  return constFoldBinaryOp<IntegerAttr>(
+      adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) {
+        return a.sge(b) ? APInt::getAllOnes(1) : APInt::getZero(1);
+      });
+}
+
+//===----------------------------------------------------------------------===//
+// spirv.UGreaterThan
+//===----------------------------------------------------------------------===//
+
+OpFoldResult
+spirv::UGreaterThanOp::fold(spirv::UGreaterThanOp::FoldAdaptor adaptor) {
+  // x == x -> false
+  if (getOperand1() == getOperand2()) {
+    auto falseAttr = BoolAttr::get(getContext(), false);
+    if (isa<IntegerType>(getType()))
+      return falseAttr;
+    if (auto vecTy = dyn_cast<VectorType>(getType()))
+      return SplatElementsAttr::get(vecTy, falseAttr);
+  }
+
+  return constFoldBinaryOp<IntegerAttr>(
+      adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) {
+        return a.ugt(b) ? APInt::getAllOnes(1) : APInt::getZero(1);
+      });
+}
+
+//===----------------------------------------------------------------------===//
+// spirv.UGreaterThanEqual
+//===----------------------------------------------------------------------===//
+
+OpFoldResult spirv::UGreaterThanEqualOp::fold(
+    spirv::UGreaterThanEqualOp::FoldAdaptor adaptor) {
+  // x == x -> true
+  if (getOperand1() == getOperand2()) {
+    auto trueAttr = BoolAttr::get(getContext(), true);
+    if (isa<IntegerType>(getType()))
+      return trueAttr;
+    if (auto vecTy = dyn_cast<VectorType>(getType()))
+      return SplatElementsAttr::get(vecTy, trueAttr);
+  }
+
+  return constFoldBinaryOp<IntegerAttr>(
+      adaptor.getOperands(), getType(), [](const APInt &a, const APInt &b) {
+        return a.uge(b) ? APInt::getAllOnes(1) : APInt::getZero(1);
+      });
+}
+
 //===----------------------------------------------------------------------===//
 // spirv.SLessThan
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir
index ed0bd070c43c..977d31a6bfe5 100644
--- a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir
+++ b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir
@@ -1478,6 +1478,182 @@ func.func @const_fold_vector_inotequal() -> vector<3xi1> {
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// spirv.SGreaterThan
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @sgt_same
+func.func @sgt_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) {
+  // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false
+  // CHECK-DAG: %[[CVFALSE:.*]] = spirv.Constant dense<false>
+  %0 = spirv.SGreaterThan %arg0, %arg0 : i32
+  %1 = spirv.SGreaterThan %arg1, %arg1 : vector<3xi32>
+
+  // CHECK: return %[[CFALSE]], %[[CVFALSE]]
+  return %0, %1 : i1, vector<3xi1>
+}
+
+// CHECK-LABEL: @const_fold_scalar_sgt
+func.func @const_fold_scalar_sgt() -> (i1, i1) {
+  %c4 = spirv.Constant 4 : i32
+  %c5 = spirv.Constant 5 : i32
+  %c6 = spirv.Constant 6 : i32
+
+  // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true
+  // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false
+  %0 = spirv.SGreaterThan %c5, %c6 : i32
+  %1 = spirv.SGreaterThan %c5, %c4 : i32
+
+  // CHECK: return %[[CFALSE]], %[[CTRUE]]
+  return %0, %1 : i1, i1
+}
+
+// CHECK-LABEL: @const_fold_vector_sgt
+func.func @const_fold_vector_sgt() -> vector<3xi1> {
+  %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32>
+  %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32>
+
+  // CHECK: %[[RET:.*]] = spirv.Constant dense<[false, false, true]>
+  %0 = spirv.SGreaterThan %cv0, %cv1 : vector<3xi32>
+
+  // CHECK: return %[[RET]]
+  return %0 : vector<3xi1>
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.SGreaterThanEqual
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @sge_same
+func.func @sge_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) {
+  // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true
+  // CHECK-DAG: %[[CVTRUE:.*]] = spirv.Constant dense<true>
+  %0 = spirv.SGreaterThanEqual %arg0, %arg0 : i32
+  %1 = spirv.SGreaterThanEqual %arg1, %arg1 : vector<3xi32>
+
+  // CHECK: return %[[CTRUE]], %[[CVTRUE]]
+  return %0, %1 : i1, vector<3xi1>
+}
+
+// CHECK-LABEL: @const_fold_scalar_sge
+func.func @const_fold_scalar_sge() -> (i1, i1) {
+  %c4 = spirv.Constant 4 : i32
+  %c5 = spirv.Constant 5 : i32
+  %c6 = spirv.Constant 6 : i32
+
+  // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true
+  // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false
+  %0 = spirv.SGreaterThanEqual %c5, %c6 : i32
+  %1 = spirv.SGreaterThanEqual %c5, %c4 : i32
+
+  // CHECK: return %[[CFALSE]], %[[CTRUE]]
+  return %0, %1 : i1, i1
+}
+
+// CHECK-LABEL: @const_fold_vector_sge
+func.func @const_fold_vector_sge() -> vector<3xi1> {
+  %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32>
+  %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32>
+
+  // CHECK: %[[RET:.*]] = spirv.Constant dense<[true, false, true]>
+  %0 = spirv.SGreaterThanEqual %cv0, %cv1 : vector<3xi32>
+
+  // CHECK: return %[[RET]]
+  return %0 : vector<3xi1>
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.UGreaterThan
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @ugt_same
+func.func @ugt_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) {
+  // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false
+  // CHECK-DAG: %[[CVFALSE:.*]] = spirv.Constant dense<false>
+  %0 = spirv.UGreaterThan %arg0, %arg0 : i32
+  %1 = spirv.UGreaterThan %arg1, %arg1 : vector<3xi32>
+
+  // CHECK: return %[[CFALSE]], %[[CVFALSE]]
+  return %0, %1 : i1, vector<3xi1>
+}
+
+// CHECK-LABEL: @const_fold_scalar_ugt
+func.func @const_fold_scalar_ugt() -> (i1, i1) {
+  %c4 = spirv.Constant 4 : i32
+  %c5 = spirv.Constant 5 : i32
+  %cn6 = spirv.Constant -6 : i32
+
+  // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true
+  // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false
+  %0 = spirv.UGreaterThan %c5, %cn6 : i32
+  %1 = spirv.UGreaterThan %c5, %c4 : i32
+
+  // CHECK: return %[[CFALSE]], %[[CTRUE]]
+  return %0, %1 : i1, i1
+}
+
+// CHECK-LABEL: @const_fold_vector_ugt
+func.func @const_fold_vector_ugt() -> vector<3xi1> {
+  %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32>
+  %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32>
+
+  // CHECK: %[[RET:.*]] = spirv.Constant dense<[false, false, true]>
+  %0 = spirv.UGreaterThan %cv0, %cv1 : vector<3xi32>
+
+  // CHECK: return %[[RET]]
+  return %0 : vector<3xi1>
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.UGreaterThanEqual
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @uge_same
+func.func @uge_same(%arg0 : i32, %arg1 : vector<3xi32>) -> (i1, vector<3xi1>) {
+  // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true
+  // CHECK-DAG: %[[CVTRUE:.*]] = spirv.Constant dense<true>
+  %0 = spirv.UGreaterThanEqual %arg0, %arg0 : i32
+  %1 = spirv.UGreaterThanEqual %arg1, %arg1 : vector<3xi32>
+
+  // CHECK: return %[[CTRUE]], %[[CVTRUE]]
+  return %0, %1 : i1, vector<3xi1>
+}
+
+// CHECK-LABEL: @const_fold_scalar_uge
+func.func @const_fold_scalar_uge() -> (i1, i1) {
+  %c4 = spirv.Constant 4 : i32
+  %c5 = spirv.Constant 5 : i32
+  %cn6 = spirv.Constant -6 : i32
+
+  // CHECK-DAG: %[[CTRUE:.*]] = spirv.Constant true
+  // CHECK-DAG: %[[CFALSE:.*]] = spirv.Constant false
+  %0 = spirv.UGreaterThanEqual %c5, %cn6 : i32
+  %1 = spirv.UGreaterThanEqual %c5, %c4 : i32
+
+  // CHECK: return %[[CFALSE]], %[[CTRUE]]
+  return %0, %1 : i1, i1
+}
+
+// CHECK-LABEL: @const_fold_vector_uge
+func.func @const_fold_vector_uge() -> vector<3xi1> {
+  %cv0 = spirv.Constant dense<[-1, -4, 3]> : vector<3xi32>
+  %cv1 = spirv.Constant dense<[-1, -3, 2]> : vector<3xi32>
+
+  // CHECK: %[[RET:.*]] = spirv.Constant dense<[true, false, true]>
+  %0 = spirv.UGreaterThanEqual %cv0, %cv1 : vector<3xi32>
+
+  // CHECK: return %[[RET]]
+  return %0 : vector<3xi1>
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.SLessThan
 //===----------------------------------------------------------------------===//
-- 
GitLab


From a6b870db091830844431f77eb47aa30fc1d70bed Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Mon, 25 Mar 2024 20:13:53 -0700
Subject: [PATCH 236/404] [RISCV] Enable sub(max, min) lowering for ABDS and
 ABDU (#86592)

We have the ISD nodes for representing signed and unsigned absolute
difference. For RISCV, we have vector min/max in the base vector
extension, so we can expand to the sub(max,min) lowering.

We could almost use the default expansion, but since fixed length
min/max are custom (not legal), the default expansion doesn't cover the
fixed vector cases. The expansion here is just a copy of the generic
code specialized to allow the custom min/max nodes to be created so they
can in turn be legalized to the _vl variants.

Existing DAG combines handle the recognition of absolute difference
idioms and conversion into the respective ISD::ABDS and ISD::ABDU nodes.

This change does have the net effect of potentially pushing a free
floating zero/sign extend after the expansion, and we don't do a great
job of folding that into later expressions. However, since in general
narrowing can reduce required work (by reducing LMUL) this seems like
the right general tradeoff.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |  20 +
 llvm/test/CodeGen/RISCV/rvv/abd.ll            | 124 ++---
 .../CodeGen/RISCV/rvv/fixed-vectors-abd.ll    | 485 +++++-------------
 .../CodeGen/RISCV/rvv/fixed-vectors-sad.ll    | 102 ++--
 4 files changed, 257 insertions(+), 474 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index c3f8a924a1da..e6814c5f71a0 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -819,6 +819,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
       setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT,
                          Legal);
 
+      setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Custom);
+
       // Custom-lower extensions and truncations from/to mask types.
       setOperationAction({ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND},
                          VT, Custom);
@@ -1203,6 +1205,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
         setOperationAction(
             {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX, ISD::ABS}, VT, Custom);
 
+        setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Custom);
+
         // vXi64 MULHS/MULHU requires the V extension instead of Zve64*.
         if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV())
           setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Custom);
@@ -6785,6 +6789,22 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
     if (!Op.getValueType().isVector())
       return lowerSADDSAT_SSUBSAT(Op, DAG);
     return lowerToScalableOp(Op, DAG);
+  case ISD::ABDS:
+  case ISD::ABDU: {
+    SDLoc dl(Op);
+    EVT VT = Op->getValueType(0);
+    SDValue LHS = DAG.getFreeze(Op->getOperand(0));
+    SDValue RHS = DAG.getFreeze(Op->getOperand(1));
+    bool IsSigned = Op->getOpcode() == ISD::ABDS;
+
+    // abds(lhs, rhs) -> sub(smax(lhs,rhs), smin(lhs,rhs))
+    // abdu(lhs, rhs) -> sub(umax(lhs,rhs), umin(lhs,rhs))
+    unsigned MaxOpc = IsSigned ? ISD::SMAX : ISD::UMAX;
+    unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN;
+    SDValue Max = DAG.getNode(MaxOpc, dl, VT, LHS, RHS);
+    SDValue Min = DAG.getNode(MinOpc, dl, VT, LHS, RHS);
+    return DAG.getNode(ISD::SUB, dl, VT, Max, Min);
+  }
   case ISD::ABS:
   case ISD::VP_ABS:
     return lowerABS(Op, DAG);
diff --git a/llvm/test/CodeGen/RISCV/rvv/abd.ll b/llvm/test/CodeGen/RISCV/rvv/abd.ll
index 7c0dc8688602..ddbfbd0b59fa 100644
--- a/llvm/test/CodeGen/RISCV/rvv/abd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/abd.ll
@@ -10,12 +10,9 @@ define <vscale x 16 x i8> @sabd_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
 ; CHECK-LABEL: sabd_b:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
-; CHECK-NEXT:    vwsub.vv v12, v8, v10
-; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v12, 0
-; CHECK-NEXT:    vmax.vv v12, v12, v8
-; CHECK-NEXT:    vsetvli zero, zero, e8, m2, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v12, 0
+; CHECK-NEXT:    vmin.vv v12, v8, v10
+; CHECK-NEXT:    vmax.vv v8, v8, v10
+; CHECK-NEXT:    vsub.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i16>
   %b.sext = sext <vscale x 16 x i8> %b to <vscale x 16 x i16>
@@ -33,9 +30,9 @@ define <vscale x 16 x i8> @sabd_b_promoted_ops(<vscale x 16 x i1> %a, <vscale x
 ; CHECK-NEXT:    vmerge.vim v12, v10, -1, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    vmerge.vim v8, v10, -1, v0
-; CHECK-NEXT:    vsub.vv v8, v12, v8
-; CHECK-NEXT:    vrsub.vi v10, v8, 0
-; CHECK-NEXT:    vmax.vv v8, v8, v10
+; CHECK-NEXT:    vmin.vv v10, v12, v8
+; CHECK-NEXT:    vmax.vv v8, v12, v8
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a.sext = sext <vscale x 16 x i1> %a to <vscale x 16 x i8>
   %b.sext = sext <vscale x 16 x i1> %b to <vscale x 16 x i8>
@@ -48,12 +45,9 @@ define <vscale x 8 x i16> @sabd_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
 ; CHECK-LABEL: sabd_h:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vwsub.vv v12, v8, v10
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v12, 0
-; CHECK-NEXT:    vmax.vv v12, v12, v8
-; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v12, 0
+; CHECK-NEXT:    vmin.vv v12, v8, v10
+; CHECK-NEXT:    vmax.vv v8, v8, v10
+; CHECK-NEXT:    vsub.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %a.sext = sext <vscale x 8 x i16> %a to <vscale x 8 x i32>
   %b.sext = sext <vscale x 8 x i16> %b to <vscale x 8 x i32>
@@ -67,10 +61,11 @@ define <vscale x 8 x i16> @sabd_h_promoted_ops(<vscale x 8 x i8> %a, <vscale x 8
 ; CHECK-LABEL: sabd_h_promoted_ops:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v10, v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vzext.vf2 v8, v10
 ; CHECK-NEXT:    ret
   %a.sext = sext <vscale x 8 x i8> %a to <vscale x 8 x i16>
   %b.sext = sext <vscale x 8 x i8> %b to <vscale x 8 x i16>
@@ -83,12 +78,9 @@ define <vscale x 4 x i32> @sabd_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
 ; CHECK-LABEL: sabd_s:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vwsub.vv v12, v8, v10
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v12, 0
-; CHECK-NEXT:    vmax.vv v12, v12, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v12, 0
+; CHECK-NEXT:    vmin.vv v12, v8, v10
+; CHECK-NEXT:    vmax.vv v8, v8, v10
+; CHECK-NEXT:    vsub.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %a.sext = sext <vscale x 4 x i32> %a to <vscale x 4 x i64>
   %b.sext = sext <vscale x 4 x i32> %b to <vscale x 4 x i64>
@@ -102,10 +94,11 @@ define <vscale x 4 x i32> @sabd_s_promoted_ops(<vscale x 4 x i16> %a, <vscale x
 ; CHECK-LABEL: sabd_s_promoted_ops:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v10, v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vzext.vf2 v8, v10
 ; CHECK-NEXT:    ret
   %a.sext = sext <vscale x 4 x i16> %a to <vscale x 4 x i32>
   %b.sext = sext <vscale x 4 x i16> %b to <vscale x 4 x i32>
@@ -128,10 +121,11 @@ define <vscale x 2 x i64> @sabd_d_promoted_ops(<vscale x 2 x i32> %a, <vscale x
 ; CHECK-LABEL: sabd_d_promoted_ops:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v10, v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vzext.vf2 v8, v10
 ; CHECK-NEXT:    ret
   %a.sext = sext <vscale x 2 x i32> %a to <vscale x 2 x i64>
   %b.sext = sext <vscale x 2 x i32> %b to <vscale x 2 x i64>
@@ -148,12 +142,9 @@ define <vscale x 16 x i8> @uabd_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
 ; CHECK-LABEL: uabd_b:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
-; CHECK-NEXT:    vwsubu.vv v12, v8, v10
-; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v12, 0
-; CHECK-NEXT:    vmax.vv v12, v12, v8
-; CHECK-NEXT:    vsetvli zero, zero, e8, m2, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v12, 0
+; CHECK-NEXT:    vminu.vv v12, v8, v10
+; CHECK-NEXT:    vmaxu.vv v8, v8, v10
+; CHECK-NEXT:    vsub.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %a.zext = zext <vscale x 16 x i8> %a to <vscale x 16 x i16>
   %b.zext = zext <vscale x 16 x i8> %b to <vscale x 16 x i16>
@@ -171,9 +162,9 @@ define <vscale x 16 x i8> @uabd_b_promoted_ops(<vscale x 16 x i1> %a, <vscale x
 ; CHECK-NEXT:    vmerge.vim v12, v10, 1, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    vmerge.vim v8, v10, 1, v0
-; CHECK-NEXT:    vsub.vv v8, v12, v8
-; CHECK-NEXT:    vrsub.vi v10, v8, 0
-; CHECK-NEXT:    vmax.vv v8, v8, v10
+; CHECK-NEXT:    vminu.vv v10, v12, v8
+; CHECK-NEXT:    vmaxu.vv v8, v12, v8
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a.zext = zext <vscale x 16 x i1> %a to <vscale x 16 x i8>
   %b.zext = zext <vscale x 16 x i1> %b to <vscale x 16 x i8>
@@ -186,12 +177,9 @@ define <vscale x 8 x i16> @uabd_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
 ; CHECK-LABEL: uabd_h:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vwsubu.vv v12, v8, v10
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v12, 0
-; CHECK-NEXT:    vmax.vv v12, v12, v8
-; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v12, 0
+; CHECK-NEXT:    vminu.vv v12, v8, v10
+; CHECK-NEXT:    vmaxu.vv v8, v8, v10
+; CHECK-NEXT:    vsub.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %a.zext = zext <vscale x 8 x i16> %a to <vscale x 8 x i32>
   %b.zext = zext <vscale x 8 x i16> %b to <vscale x 8 x i32>
@@ -205,10 +193,11 @@ define <vscale x 8 x i16> @uabd_h_promoted_ops(<vscale x 8 x i8> %a, <vscale x 8
 ; CHECK-LABEL: uabd_h_promoted_ops:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v10, v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vzext.vf2 v8, v10
 ; CHECK-NEXT:    ret
   %a.zext = zext <vscale x 8 x i8> %a to <vscale x 8 x i16>
   %b.zext = zext <vscale x 8 x i8> %b to <vscale x 8 x i16>
@@ -221,12 +210,9 @@ define <vscale x 4 x i32> @uabd_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
 ; CHECK-LABEL: uabd_s:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vwsubu.vv v12, v8, v10
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v12, 0
-; CHECK-NEXT:    vmax.vv v12, v12, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v12, 0
+; CHECK-NEXT:    vminu.vv v12, v8, v10
+; CHECK-NEXT:    vmaxu.vv v8, v8, v10
+; CHECK-NEXT:    vsub.vv v8, v8, v12
 ; CHECK-NEXT:    ret
   %a.zext = zext <vscale x 4 x i32> %a to <vscale x 4 x i64>
   %b.zext = zext <vscale x 4 x i32> %b to <vscale x 4 x i64>
@@ -240,10 +226,11 @@ define <vscale x 4 x i32> @uabd_s_promoted_ops(<vscale x 4 x i16> %a, <vscale x
 ; CHECK-LABEL: uabd_s_promoted_ops:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v10, v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vzext.vf2 v8, v10
 ; CHECK-NEXT:    ret
   %a.zext = zext <vscale x 4 x i16> %a to <vscale x 4 x i32>
   %b.zext = zext <vscale x 4 x i16> %b to <vscale x 4 x i32>
@@ -266,10 +253,11 @@ define <vscale x 2 x i64> @uabd_d_promoted_ops(<vscale x 2 x i32> %a, <vscale x
 ; CHECK-LABEL: uabd_d_promoted_ops:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v10, v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vzext.vf2 v8, v10
 ; CHECK-NEXT:    ret
   %a.zext = zext <vscale x 2 x i32> %a to <vscale x 2 x i64>
   %b.zext = zext <vscale x 2 x i32> %b to <vscale x 2 x i64>
@@ -285,12 +273,9 @@ define <vscale x 4 x i32> @uabd_non_matching_extension(<vscale x 4 x i32> %a, <v
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vzext.vf4 v12, v10
-; CHECK-NEXT:    vwsubu.vv v16, v8, v12
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v16, 0
-; CHECK-NEXT:    vmax.vv v12, v16, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v12, 0
+; CHECK-NEXT:    vminu.vv v10, v8, v12
+; CHECK-NEXT:    vmaxu.vv v8, v8, v12
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a.zext = zext <vscale x 4 x i32> %a to <vscale x 4 x i64>
   %b.zext = zext <vscale x 4 x i8> %b to <vscale x 4 x i64>
@@ -307,10 +292,11 @@ define <vscale x 4 x i32> @uabd_non_matching_promoted_ops(<vscale x 4 x i8> %a,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vwsubu.vv v12, v10, v9
+; CHECK-NEXT:    vminu.vv v8, v10, v9
+; CHECK-NEXT:    vmaxu.vv v9, v10, v9
+; CHECK-NEXT:    vsub.vv v10, v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v12, 0
-; CHECK-NEXT:    vmax.vv v8, v12, v8
+; CHECK-NEXT:    vzext.vf2 v8, v10
 ; CHECK-NEXT:    ret
   %a.zext = zext <vscale x 4 x i8> %a to <vscale x 4 x i32>
   %b.zext = zext <vscale x 4 x i16> %b to <vscale x 4 x i32>
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-abd.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-abd.ll
index 79c0857f90ec..bd1209a17b53 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-abd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-abd.ll
@@ -10,12 +10,9 @@ define <8 x i8> @sabd_8b_as_16b(<8 x i8> %a, <8 x i8> %b) {
 ; CHECK-LABEL: sabd_8b_as_16b:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vwsub.vv v10, v8, v9
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a.sext = sext <8 x i8> %a to <8 x i16>
   %b.sext = sext <8 x i8> %b to <8 x i16>
@@ -29,17 +26,10 @@ define <8 x i8> @sabd_8b_as_32b(<8 x i8> %a, <8 x i8> %b) {
 ;
 ; CHECK-LABEL: sabd_8b_as_32b:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vsext.vf2 v10, v8
-; CHECK-NEXT:    vsext.vf2 v8, v9
-; CHECK-NEXT:    vwsub.vv v12, v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v12, 0
-; CHECK-NEXT:    vmax.vv v8, v12, v8
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vnsrl.wi v10, v8, 0
-; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v10, 0
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a.sext = sext <8 x i8> %a to <8 x i32>
   %b.sext = sext <8 x i8> %b to <8 x i32>
@@ -54,12 +44,9 @@ define <16 x i8> @sabd_16b(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: sabd_16b:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vwsub.vv v10, v8, v9
-; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v10, v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v10, 0
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a.sext = sext <16 x i8> %a to <16 x i16>
   %b.sext = sext <16 x i8> %b to <16 x i16>
@@ -74,12 +61,9 @@ define <4 x i16> @sabd_4h(<4 x i16> %a, <4 x i16> %b) {
 ; CHECK-LABEL: sabd_4h:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vwsub.vv v10, v8, v9
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a.sext = sext <4 x i16> %a to <4 x i32>
   %b.sext = sext <4 x i16> %b to <4 x i32>
@@ -94,10 +78,11 @@ define <4 x i16> @sabd_4h_promoted_ops(<4 x i8> %a, <4 x i8> %b) {
 ; CHECK-LABEL: sabd_4h_promoted_ops:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v9, v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vzext.vf2 v8, v9
 ; CHECK-NEXT:    ret
   %a.sext = sext <4 x i8> %a to <4 x i16>
   %b.sext = sext <4 x i8> %b to <4 x i16>
@@ -111,12 +96,9 @@ define <8 x i16> @sabd_8h(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: sabd_8h:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vwsub.vv v10, v8, v9
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v10, v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v10, 0
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a.sext = sext <8 x i16> %a to <8 x i32>
   %b.sext = sext <8 x i16> %b to <8 x i32>
@@ -131,10 +113,11 @@ define <8 x i16> @sabd_8h_promoted_ops(<8 x i8> %a, <8 x i8> %b) {
 ; CHECK-LABEL: sabd_8h_promoted_ops:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v9, v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vzext.vf2 v8, v9
 ; CHECK-NEXT:    ret
   %a.sext = sext <8 x i8> %a to <8 x i16>
   %b.sext = sext <8 x i8> %b to <8 x i16>
@@ -148,12 +131,9 @@ define <2 x i32> @sabd_2s(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: sabd_2s:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT:    vwsub.vv v10, v8, v9
-; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a.sext = sext <2 x i32> %a to <2 x i64>
   %b.sext = sext <2 x i32> %b to <2 x i64>
@@ -168,10 +148,11 @@ define <2 x i32> @sabd_2s_promoted_ops(<2 x i16> %a, <2 x i16> %b) {
 ; CHECK-LABEL: sabd_2s_promoted_ops:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v9, v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vzext.vf2 v8, v9
 ; CHECK-NEXT:    ret
   %a.sext = sext <2 x i16> %a to <2 x i32>
   %b.sext = sext <2 x i16> %b to <2 x i32>
@@ -185,12 +166,9 @@ define <4 x i32> @sabd_4s(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: sabd_4s:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vwsub.vv v10, v8, v9
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v10, v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v10, 0
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a.sext = sext <4 x i32> %a to <4 x i64>
   %b.sext = sext <4 x i32> %b to <4 x i64>
@@ -205,10 +183,11 @@ define <4 x i32> @sabd_4s_promoted_ops(<4 x i16> %a, <4 x i16> %b) {
 ; CHECK-LABEL: sabd_4s_promoted_ops:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v9, v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vzext.vf2 v8, v9
 ; CHECK-NEXT:    ret
   %a.sext = sext <4 x i16> %a to <4 x i32>
   %b.sext = sext <4 x i16> %b to <4 x i32>
@@ -218,113 +197,13 @@ define <4 x i32> @sabd_4s_promoted_ops(<4 x i16> %a, <4 x i16> %b) {
 }
 
 define <2 x i64> @sabd_2d(<2 x i64> %a, <2 x i64> %b) {
-; RV32-LABEL: sabd_2d:
-; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v10, v8, 1
-; RV32-NEXT:    li a1, 32
-; RV32-NEXT:    vsrl.vx v11, v10, a1
-; RV32-NEXT:    vmv.x.s a3, v11
-; RV32-NEXT:    vsrl.vx v11, v8, a1
-; RV32-NEXT:    vmv.x.s a5, v11
-; RV32-NEXT:    srai t1, a5, 31
-; RV32-NEXT:    vmv.x.s a4, v10
-; RV32-NEXT:    vmv.x.s a0, v8
-; RV32-NEXT:    vslidedown.vi v8, v9, 1
-; RV32-NEXT:    vsrl.vx v10, v8, a1
-; RV32-NEXT:    vmv.x.s a6, v10
-; RV32-NEXT:    vsrl.vx v10, v9, a1
-; RV32-NEXT:    vmv.x.s a7, v10
-; RV32-NEXT:    srai t4, a7, 31
-; RV32-NEXT:    vmv.x.s a1, v9
-; RV32-NEXT:    sltu a2, a0, a1
-; RV32-NEXT:    vmv.x.s t0, v8
-; RV32-NEXT:    mv t5, a2
-; RV32-NEXT:    beq a5, a7, .LBB11_2
-; RV32-NEXT:  # %bb.1:
-; RV32-NEXT:    sltu t5, a5, a7
-; RV32-NEXT:  .LBB11_2:
-; RV32-NEXT:    srai t2, a3, 31
-; RV32-NEXT:    srai t3, a6, 31
-; RV32-NEXT:    sub t6, t1, t4
-; RV32-NEXT:    sltu t5, t6, t5
-; RV32-NEXT:    sltu t1, t1, t4
-; RV32-NEXT:    sltu t4, a4, t0
-; RV32-NEXT:    sub t1, t6, t1
-; RV32-NEXT:    mv t6, t4
-; RV32-NEXT:    beq a3, a6, .LBB11_4
-; RV32-NEXT:  # %bb.3:
-; RV32-NEXT:    sltu t6, a3, a6
-; RV32-NEXT:  .LBB11_4:
-; RV32-NEXT:    sub t1, t1, t5
-; RV32-NEXT:    sub t5, t2, t3
-; RV32-NEXT:    sltu t6, t5, t6
-; RV32-NEXT:    sltu t2, t2, t3
-; RV32-NEXT:    sub t2, t5, t2
-; RV32-NEXT:    sub t2, t2, t6
-; RV32-NEXT:    sub a5, a5, a7
-; RV32-NEXT:    sub a3, a3, a6
-; RV32-NEXT:    sub a3, a3, t4
-; RV32-NEXT:    sub a4, a4, t0
-; RV32-NEXT:    bgez t2, .LBB11_6
-; RV32-NEXT:  # %bb.5:
-; RV32-NEXT:    snez a6, a4
-; RV32-NEXT:    neg a6, a6
-; RV32-NEXT:    sub a3, a6, a3
-; RV32-NEXT:    neg a4, a4
-; RV32-NEXT:  .LBB11_6:
-; RV32-NEXT:    sub a5, a5, a2
-; RV32-NEXT:    sub a0, a0, a1
-; RV32-NEXT:    bgez t1, .LBB11_8
-; RV32-NEXT:  # %bb.7:
-; RV32-NEXT:    snez a1, a0
-; RV32-NEXT:    neg a1, a1
-; RV32-NEXT:    sub a5, a1, a5
-; RV32-NEXT:    neg a0, a0
-; RV32-NEXT:  .LBB11_8:
-; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a5
-; RV32-NEXT:    vmv.v.x v9, a4
-; RV32-NEXT:    vslide1down.vx v9, v9, a3
-; RV32-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
-; RV32-NEXT:    vslideup.vi v8, v9, 1
-; RV32-NEXT:    ret
-;
-; RV64-LABEL: sabd_2d:
-; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-NEXT:    vslidedown.vi v10, v8, 1
-; RV64-NEXT:    vmv.x.s a1, v10
-; RV64-NEXT:    srai a4, a1, 63
-; RV64-NEXT:    vmv.x.s a0, v8
-; RV64-NEXT:    srai a3, a0, 63
-; RV64-NEXT:    vslidedown.vi v8, v9, 1
-; RV64-NEXT:    vmv.x.s a5, v8
-; RV64-NEXT:    srai a6, a5, 63
-; RV64-NEXT:    vmv.x.s a2, v9
-; RV64-NEXT:    srai a7, a2, 63
-; RV64-NEXT:    sltu t0, a0, a2
-; RV64-NEXT:    sub a3, a3, a7
-; RV64-NEXT:    sub a3, a3, t0
-; RV64-NEXT:    sltu a7, a1, a5
-; RV64-NEXT:    sub a4, a4, a6
-; RV64-NEXT:    sub a4, a4, a7
-; RV64-NEXT:    sub a1, a1, a5
-; RV64-NEXT:    bgez a4, .LBB11_2
-; RV64-NEXT:  # %bb.1:
-; RV64-NEXT:    neg a1, a1
-; RV64-NEXT:  .LBB11_2:
-; RV64-NEXT:    sub a0, a0, a2
-; RV64-NEXT:    bgez a3, .LBB11_4
-; RV64-NEXT:  # %bb.3:
-; RV64-NEXT:    neg a0, a0
-; RV64-NEXT:  .LBB11_4:
-; RV64-NEXT:    vmv.s.x v8, a0
-; RV64-NEXT:    vmv.s.x v9, a1
-; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV64-NEXT:    vslideup.vi v8, v9, 1
-; RV64-NEXT:    ret
+; CHECK-LABEL: sabd_2d:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
+; CHECK-NEXT:    ret
   %a.sext = sext <2 x i64> %a to <2 x i128>
   %b.sext = sext <2 x i64> %b to <2 x i128>
   %sub = sub <2 x i128> %a.sext, %b.sext
@@ -338,10 +217,11 @@ define <2 x i64> @sabd_2d_promoted_ops(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: sabd_2d_promoted_ops:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT:    vwsub.vv v10, v8, v9
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v9, v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vzext.vf2 v8, v9
 ; CHECK-NEXT:    ret
   %a.sext = sext <2 x i32> %a to <2 x i64>
   %b.sext = sext <2 x i32> %b to <2 x i64>
@@ -359,12 +239,9 @@ define <8 x i8> @uabd_8b(<8 x i8> %a, <8 x i8> %b) {
 ; CHECK-LABEL: uabd_8b:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a.zext = zext <8 x i8> %a to <8 x i16>
   %b.zext = zext <8 x i8> %b to <8 x i16>
@@ -379,12 +256,9 @@ define <16 x i8> @uabd_16b(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: uabd_16b:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
-; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v10, v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v10, 0
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a.zext = zext <16 x i8> %a to <16 x i16>
   %b.zext = zext <16 x i8> %b to <16 x i16>
@@ -399,12 +273,9 @@ define <4 x i16> @uabd_4h(<4 x i16> %a, <4 x i16> %b) {
 ; CHECK-LABEL: uabd_4h:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a.zext = zext <4 x i16> %a to <4 x i32>
   %b.zext = zext <4 x i16> %b to <4 x i32>
@@ -419,10 +290,11 @@ define <4 x i16> @uabd_4h_promoted_ops(<4 x i8> %a, <4 x i8> %b) {
 ; CHECK-LABEL: uabd_4h_promoted_ops:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v9, v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vzext.vf2 v8, v9
 ; CHECK-NEXT:    ret
   %a.zext = zext <4 x i8> %a to <4 x i16>
   %b.zext = zext <4 x i8> %b to <4 x i16>
@@ -436,12 +308,9 @@ define <8 x i16> @uabd_8h(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: uabd_8h:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v10, v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v10, 0
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a.zext = zext <8 x i16> %a to <8 x i32>
   %b.zext = zext <8 x i16> %b to <8 x i32>
@@ -456,10 +325,11 @@ define <8 x i16> @uabd_8h_promoted_ops(<8 x i8> %a, <8 x i8> %b) {
 ; CHECK-LABEL: uabd_8h_promoted_ops:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v9, v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vzext.vf2 v8, v9
 ; CHECK-NEXT:    ret
   %a.zext = zext <8 x i8> %a to <8 x i16>
   %b.zext = zext <8 x i8> %b to <8 x i16>
@@ -473,12 +343,9 @@ define <2 x i32> @uabd_2s(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: uabd_2s:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
-; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v8, 0
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a.zext = zext <2 x i32> %a to <2 x i64>
   %b.zext = zext <2 x i32> %b to <2 x i64>
@@ -493,10 +360,11 @@ define <2 x i32> @uabd_2s_promoted_ops(<2 x i16> %a, <2 x i16> %b) {
 ; CHECK-LABEL: uabd_2s_promoted_ops:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v9, v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vzext.vf2 v8, v9
 ; CHECK-NEXT:    ret
   %a.zext = zext <2 x i16> %a to <2 x i32>
   %b.zext = zext <2 x i16> %b to <2 x i32>
@@ -510,12 +378,9 @@ define <4 x i32> @uabd_4s(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: uabd_4s:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v10, v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vnsrl.wi v8, v10, 0
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a.zext = zext <4 x i32> %a to <4 x i64>
   %b.zext = zext <4 x i32> %b to <4 x i64>
@@ -530,10 +395,11 @@ define <4 x i32> @uabd_4s_promoted_ops(<4 x i16> %a, <4 x i16> %b) {
 ; CHECK-LABEL: uabd_4s_promoted_ops:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v9, v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vzext.vf2 v8, v9
 ; CHECK-NEXT:    ret
   %a.zext = zext <4 x i16> %a to <4 x i32>
   %b.zext = zext <4 x i16> %b to <4 x i32>
@@ -543,104 +409,13 @@ define <4 x i32> @uabd_4s_promoted_ops(<4 x i16> %a, <4 x i16> %b) {
 }
 
 define <2 x i64> @uabd_2d(<2 x i64> %a, <2 x i64> %b) {
-; RV32-LABEL: uabd_2d:
-; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v10, v8, 1
-; RV32-NEXT:    li a4, 32
-; RV32-NEXT:    vsrl.vx v11, v10, a4
-; RV32-NEXT:    vmv.x.s a0, v11
-; RV32-NEXT:    vsrl.vx v11, v8, a4
-; RV32-NEXT:    vmv.x.s a3, v11
-; RV32-NEXT:    vmv.x.s a1, v10
-; RV32-NEXT:    vmv.x.s a2, v8
-; RV32-NEXT:    vslidedown.vi v8, v9, 1
-; RV32-NEXT:    vsrl.vx v10, v8, a4
-; RV32-NEXT:    vmv.x.s a7, v10
-; RV32-NEXT:    vsrl.vx v10, v9, a4
-; RV32-NEXT:    vmv.x.s a6, v10
-; RV32-NEXT:    vmv.x.s a4, v9
-; RV32-NEXT:    sltu a5, a2, a4
-; RV32-NEXT:    vmv.x.s t0, v8
-; RV32-NEXT:    mv t2, a5
-; RV32-NEXT:    beq a3, a6, .LBB23_2
-; RV32-NEXT:  # %bb.1:
-; RV32-NEXT:    sltu t2, a3, a6
-; RV32-NEXT:  .LBB23_2:
-; RV32-NEXT:    sltu t3, a1, t0
-; RV32-NEXT:    mv t4, t3
-; RV32-NEXT:    beq a0, a7, .LBB23_4
-; RV32-NEXT:  # %bb.3:
-; RV32-NEXT:    sltu t4, a0, a7
-; RV32-NEXT:  .LBB23_4:
-; RV32-NEXT:    sub t1, a3, a6
-; RV32-NEXT:    neg a3, t2
-; RV32-NEXT:    neg a6, t4
-; RV32-NEXT:    sub a0, a0, a7
-; RV32-NEXT:    sub a0, a0, t3
-; RV32-NEXT:    sub a1, a1, t0
-; RV32-NEXT:    bgez a6, .LBB23_6
-; RV32-NEXT:  # %bb.5:
-; RV32-NEXT:    snez a7, a1
-; RV32-NEXT:    neg a7, a7
-; RV32-NEXT:    sub a0, a7, a0
-; RV32-NEXT:  .LBB23_6:
-; RV32-NEXT:    sub a5, t1, a5
-; RV32-NEXT:    sub a2, a2, a4
-; RV32-NEXT:    bltz a3, .LBB23_11
-; RV32-NEXT:  # %bb.7:
-; RV32-NEXT:    bltz a6, .LBB23_12
-; RV32-NEXT:  .LBB23_8:
-; RV32-NEXT:    bgez a3, .LBB23_10
-; RV32-NEXT:  .LBB23_9:
-; RV32-NEXT:    neg a2, a2
-; RV32-NEXT:  .LBB23_10:
-; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a2
-; RV32-NEXT:    vslide1down.vx v8, v8, a5
-; RV32-NEXT:    vmv.v.x v9, a1
-; RV32-NEXT:    vslide1down.vx v9, v9, a0
-; RV32-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
-; RV32-NEXT:    vslideup.vi v8, v9, 1
-; RV32-NEXT:    ret
-; RV32-NEXT:  .LBB23_11:
-; RV32-NEXT:    snez a4, a2
-; RV32-NEXT:    neg a4, a4
-; RV32-NEXT:    sub a5, a4, a5
-; RV32-NEXT:    bgez a6, .LBB23_8
-; RV32-NEXT:  .LBB23_12:
-; RV32-NEXT:    neg a1, a1
-; RV32-NEXT:    bltz a3, .LBB23_9
-; RV32-NEXT:    j .LBB23_10
-;
-; RV64-LABEL: uabd_2d:
-; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-NEXT:    vslidedown.vi v10, v8, 1
-; RV64-NEXT:    vmv.x.s a0, v10
-; RV64-NEXT:    vmv.x.s a1, v8
-; RV64-NEXT:    vslidedown.vi v8, v9, 1
-; RV64-NEXT:    vmv.x.s a4, v8
-; RV64-NEXT:    vmv.x.s a2, v9
-; RV64-NEXT:    sltu a3, a1, a2
-; RV64-NEXT:    neg a3, a3
-; RV64-NEXT:    sltu a5, a0, a4
-; RV64-NEXT:    neg a5, a5
-; RV64-NEXT:    sub a0, a0, a4
-; RV64-NEXT:    bgez a5, .LBB23_2
-; RV64-NEXT:  # %bb.1:
-; RV64-NEXT:    neg a0, a0
-; RV64-NEXT:  .LBB23_2:
-; RV64-NEXT:    sub a1, a1, a2
-; RV64-NEXT:    bgez a3, .LBB23_4
-; RV64-NEXT:  # %bb.3:
-; RV64-NEXT:    neg a1, a1
-; RV64-NEXT:  .LBB23_4:
-; RV64-NEXT:    vmv.s.x v8, a1
-; RV64-NEXT:    vmv.s.x v9, a0
-; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV64-NEXT:    vslideup.vi v8, v9, 1
-; RV64-NEXT:    ret
+; CHECK-LABEL: uabd_2d:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
+; CHECK-NEXT:    ret
   %a.zext = zext <2 x i64> %a to <2 x i128>
   %b.zext = zext <2 x i64> %b to <2 x i128>
   %sub = sub <2 x i128> %a.zext, %b.zext
@@ -654,10 +429,11 @@ define <2 x i64> @uabd_2d_promoted_ops(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: uabd_2d_promoted_ops:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v9, v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vzext.vf2 v8, v9
 ; CHECK-NEXT:    ret
   %a.zext = zext <2 x i32> %a to <2 x i64>
   %b.zext = zext <2 x i32> %b to <2 x i64>
@@ -727,9 +503,9 @@ define <16 x i8> @sabd_v16i8_nsw(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: sabd_v16i8_nsw:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vsub.vv v8, v8, v9
-; CHECK-NEXT:    vrsub.vi v9, v8, 0
+; CHECK-NEXT:    vmin.vv v10, v8, v9
 ; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %sub = sub nsw <16 x i8> %a, %b
   %abs = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %sub, i1 true)
@@ -741,9 +517,9 @@ define <8 x i16> @sabd_v8i16_nsw(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: sabd_v8i16_nsw:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vsub.vv v8, v8, v9
-; CHECK-NEXT:    vrsub.vi v9, v8, 0
+; CHECK-NEXT:    vmin.vv v10, v8, v9
 ; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %sub = sub nsw <8 x i16> %a, %b
   %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 true)
@@ -755,9 +531,9 @@ define <4 x i32> @sabd_v4i32_nsw(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: sabd_v4i32_nsw:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vsub.vv v8, v8, v9
-; CHECK-NEXT:    vrsub.vi v9, v8, 0
+; CHECK-NEXT:    vmin.vv v10, v8, v9
 ; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %sub = sub nsw <4 x i32> %a, %b
   %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 true)
@@ -769,9 +545,9 @@ define <2 x i64> @sabd_v2i64_nsw(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: sabd_v2i64_nsw:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-NEXT:    vsub.vv v8, v8, v9
-; CHECK-NEXT:    vrsub.vi v9, v8, 0
+; CHECK-NEXT:    vmin.vv v10, v8, v9
 ; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %sub = sub nsw <2 x i64> %a, %b
   %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 true)
@@ -783,9 +559,9 @@ define <16 x i8> @smaxmin_v16i8(<16 x i8> %0, <16 x i8> %1) {
 ; CHECK-LABEL: smaxmin_v16i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vmax.vv v10, v8, v9
-; CHECK-NEXT:    vmin.vv v8, v8, v9
-; CHECK-NEXT:    vsub.vv v8, v10, v8
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a = tail call <16 x i8> @llvm.smax.v16i8(<16 x i8> %0, <16 x i8> %1)
   %b = tail call <16 x i8> @llvm.smin.v16i8(<16 x i8> %0, <16 x i8> %1)
@@ -798,9 +574,9 @@ define <8 x i16> @smaxmin_v8i16(<8 x i16> %0, <8 x i16> %1) {
 ; CHECK-LABEL: smaxmin_v8i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vmax.vv v10, v8, v9
-; CHECK-NEXT:    vmin.vv v8, v8, v9
-; CHECK-NEXT:    vsub.vv v8, v10, v8
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a = tail call <8 x i16> @llvm.smax.v8i16(<8 x i16> %0, <8 x i16> %1)
   %b = tail call <8 x i16> @llvm.smin.v8i16(<8 x i16> %0, <8 x i16> %1)
@@ -813,9 +589,9 @@ define <4 x i32> @smaxmin_v4i32(<4 x i32> %0, <4 x i32> %1) {
 ; CHECK-LABEL: smaxmin_v4i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vmax.vv v10, v8, v9
-; CHECK-NEXT:    vmin.vv v8, v8, v9
-; CHECK-NEXT:    vsub.vv v8, v10, v8
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a = tail call <4 x i32> @llvm.smax.v4i32(<4 x i32> %0, <4 x i32> %1)
   %b = tail call <4 x i32> @llvm.smin.v4i32(<4 x i32> %0, <4 x i32> %1)
@@ -828,9 +604,9 @@ define <2 x i64> @smaxmin_v2i64(<2 x i64> %0, <2 x i64> %1) {
 ; CHECK-LABEL: smaxmin_v2i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-NEXT:    vmax.vv v10, v8, v9
-; CHECK-NEXT:    vmin.vv v8, v8, v9
-; CHECK-NEXT:    vsub.vv v8, v10, v8
+; CHECK-NEXT:    vmin.vv v10, v8, v9
+; CHECK-NEXT:    vmax.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a = tail call <2 x i64> @llvm.smax.v2i64(<2 x i64> %0, <2 x i64> %1)
   %b = tail call <2 x i64> @llvm.smin.v2i64(<2 x i64> %0, <2 x i64> %1)
@@ -843,9 +619,9 @@ define <16 x i8> @umaxmin_v16i8(<16 x i8> %0, <16 x i8> %1) {
 ; CHECK-LABEL: umaxmin_v16i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vmaxu.vv v10, v8, v9
-; CHECK-NEXT:    vminu.vv v8, v8, v9
-; CHECK-NEXT:    vsub.vv v8, v10, v8
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a = tail call <16 x i8> @llvm.umax.v16i8(<16 x i8> %0, <16 x i8> %1)
   %b = tail call <16 x i8> @llvm.umin.v16i8(<16 x i8> %0, <16 x i8> %1)
@@ -858,9 +634,9 @@ define <8 x i16> @umaxmin_v8i16(<8 x i16> %0, <8 x i16> %1) {
 ; CHECK-LABEL: umaxmin_v8i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vmaxu.vv v10, v8, v9
-; CHECK-NEXT:    vminu.vv v8, v8, v9
-; CHECK-NEXT:    vsub.vv v8, v10, v8
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a = tail call <8 x i16> @llvm.umax.v8i16(<8 x i16> %0, <8 x i16> %1)
   %b = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %0, <8 x i16> %1)
@@ -873,9 +649,9 @@ define <4 x i32> @umaxmin_v4i32(<4 x i32> %0, <4 x i32> %1) {
 ; CHECK-LABEL: umaxmin_v4i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vmaxu.vv v10, v8, v9
-; CHECK-NEXT:    vminu.vv v8, v8, v9
-; CHECK-NEXT:    vsub.vv v8, v10, v8
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a = tail call <4 x i32> @llvm.umax.v4i32(<4 x i32> %0, <4 x i32> %1)
   %b = tail call <4 x i32> @llvm.umin.v4i32(<4 x i32> %0, <4 x i32> %1)
@@ -888,9 +664,9 @@ define <2 x i64> @umaxmin_v2i64(<2 x i64> %0, <2 x i64> %1) {
 ; CHECK-LABEL: umaxmin_v2i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-NEXT:    vmaxu.vv v10, v8, v9
-; CHECK-NEXT:    vminu.vv v8, v8, v9
-; CHECK-NEXT:    vsub.vv v8, v10, v8
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a = tail call <2 x i64> @llvm.umax.v2i64(<2 x i64> %0, <2 x i64> %1)
   %b = tail call <2 x i64> @llvm.umin.v2i64(<2 x i64> %0, <2 x i64> %1)
@@ -903,9 +679,9 @@ define <16 x i8> @umaxmin_v16i8_com1(<16 x i8> %0, <16 x i8> %1) {
 ; CHECK-LABEL: umaxmin_v16i8_com1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vmaxu.vv v10, v8, v9
-; CHECK-NEXT:    vminu.vv v8, v9, v8
-; CHECK-NEXT:    vsub.vv v8, v10, v8
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    ret
   %a = tail call <16 x i8> @llvm.umax.v16i8(<16 x i8> %0, <16 x i8> %1)
   %b = tail call <16 x i8> @llvm.umin.v16i8(<16 x i8> %1, <16 x i8> %0)
@@ -946,3 +722,6 @@ declare <8 x i16> @llvm.umin.v8i16(<8 x i16>, <8 x i16>)
 declare <4 x i32> @llvm.umin.v4i32(<4 x i32>, <4 x i32>)
 declare <2 x i64> @llvm.umin.v2i64(<2 x i64>, <2 x i64>)
 
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; RV32: {{.*}}
+; RV64: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sad.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sad.ll
index cf177a03a3b4..a4ab67f41595 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sad.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sad.ll
@@ -6,12 +6,14 @@ define signext i16 @sad_4x8_as_i16(<4 x i8> %a, <4 x i8> %b) {
 ; CHECK-LABEL: sad_4x8_as_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
 ; CHECK-NEXT:    vmv.s.x v9, zero
-; CHECK-NEXT:    vredsum.vs v8, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
+; CHECK-NEXT:    vwredsumu.vs v8, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
 entry:
@@ -27,15 +29,13 @@ define signext i32 @sad_4x8_as_i32(<4 x i8> %a, <4 x i8> %b) {
 ; CHECK-LABEL: sad_4x8_as_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
-; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vmv.s.x v9, zero
-; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vwredsumu.vs v8, v8, v9
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vzext.vf4 v9, v8
+; CHECK-NEXT:    vmv.s.x v8, zero
+; CHECK-NEXT:    vredsum.vs v8, v9, v8
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
 entry:
@@ -51,12 +51,14 @@ define signext i16 @sad_16x8_as_i16(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: sad_16x8_as_i16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmv.s.x v9, zero
+; CHECK-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwredsumu.vs v8, v8, v9
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
-; CHECK-NEXT:    vmv.s.x v10, zero
-; CHECK-NEXT:    vredsum.vs v8, v8, v10
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
 entry:
@@ -72,15 +74,13 @@ define signext i32 @sad_16x8_as_i32(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: sad_16x8_as_i32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
-; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmv.s.x v10, zero
-; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vwredsumu.vs v8, v8, v10
+; CHECK-NEXT:    vminu.vv v10, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vzext.vf4 v12, v8
+; CHECK-NEXT:    vmv.s.x v8, zero
+; CHECK-NEXT:    vredsum.vs v8, v12, v8
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
 entry:
@@ -98,43 +98,41 @@ define signext i32 @sad_2block_16xi8_as_i32(ptr %a, ptr %b, i32 signext %stridea
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vle8.v v9, (a1)
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
-; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    add a0, a0, a2
 ; CHECK-NEXT:    add a1, a1, a3
-; CHECK-NEXT:    vle8.v v8, (a0)
-; CHECK-NEXT:    vle8.v v9, (a1)
-; CHECK-NEXT:    vrsub.vi v12, v10, 0
-; CHECK-NEXT:    vmax.vv v12, v10, v12
-; CHECK-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vwsubu.vv v10, v8, v9
-; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
+; CHECK-NEXT:    vle8.v v10, (a0)
+; CHECK-NEXT:    vle8.v v11, (a1)
+; CHECK-NEXT:    vminu.vv v12, v8, v9
+; CHECK-NEXT:    vmaxu.vv v8, v8, v9
+; CHECK-NEXT:    vsub.vv v8, v8, v12
+; CHECK-NEXT:    vminu.vv v9, v10, v11
 ; CHECK-NEXT:    add a0, a0, a2
 ; CHECK-NEXT:    add a1, a1, a3
-; CHECK-NEXT:    vle8.v v14, (a0)
-; CHECK-NEXT:    vle8.v v15, (a1)
-; CHECK-NEXT:    vmax.vv v16, v10, v8
-; CHECK-NEXT:    vwaddu.vv v8, v16, v12
-; CHECK-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vwsubu.vv v12, v14, v15
+; CHECK-NEXT:    vle8.v v12, (a0)
+; CHECK-NEXT:    vle8.v v13, (a1)
+; CHECK-NEXT:    vmaxu.vv v10, v10, v11
+; CHECK-NEXT:    vsub.vv v9, v10, v9
+; CHECK-NEXT:    vwaddu.vv v10, v9, v8
+; CHECK-NEXT:    vminu.vv v8, v12, v13
+; CHECK-NEXT:    vmaxu.vv v9, v12, v13
+; CHECK-NEXT:    vsub.vv v8, v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v14, v12, 0
 ; CHECK-NEXT:    add a0, a0, a2
 ; CHECK-NEXT:    add a1, a1, a3
-; CHECK-NEXT:    vle8.v v16, (a0)
-; CHECK-NEXT:    vle8.v v17, (a1)
-; CHECK-NEXT:    vmax.vv v12, v12, v14
-; CHECK-NEXT:    vwaddu.wv v8, v8, v12
+; CHECK-NEXT:    vle8.v v9, (a0)
+; CHECK-NEXT:    vle8.v v12, (a1)
+; CHECK-NEXT:    vzext.vf2 v14, v8
+; CHECK-NEXT:    vwaddu.vv v16, v14, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vwsubu.vv v12, v16, v17
+; CHECK-NEXT:    vminu.vv v8, v9, v12
+; CHECK-NEXT:    vmaxu.vv v9, v9, v12
+; CHECK-NEXT:    vsub.vv v8, v9, v8
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v14, v12, 0
-; CHECK-NEXT:    vmax.vv v12, v12, v14
-; CHECK-NEXT:    vwaddu.wv v8, v8, v12
+; CHECK-NEXT:    vzext.vf2 v10, v8
+; CHECK-NEXT:    vwaddu.wv v16, v16, v10
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vmv.s.x v12, zero
-; CHECK-NEXT:    vredsum.vs v8, v8, v12
+; CHECK-NEXT:    vmv.s.x v8, zero
+; CHECK-NEXT:    vredsum.vs v8, v16, v8
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
 entry:
-- 
GitLab


From bc31be7949a3d5382be0e15e3957fa957da9de45 Mon Sep 17 00:00:00 2001
From: Nathan Ridge <zeratul976@hotmail.com>
Date: Mon, 25 Mar 2024 23:54:40 -0400
Subject: [PATCH 237/404] [clang][CodeComplete] Handle deref operator in
 getApproximateType (#86466)

This allows completing after `(*this).` in a dependent context.

Fixes https://github.com/clangd/clangd/issues/1952
---
 clang/lib/Sema/SemaCodeComplete.cpp         |  5 +++++
 clang/test/CodeCompletion/member-access.cpp | 20 ++++++++++++++++++--
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index 73e6baa52782..83ebcaf9e765 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -20,6 +20,7 @@
 #include "clang/AST/ExprConcepts.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/NestedNameSpecifier.h"
+#include "clang/AST/OperationKinds.h"
 #include "clang/AST/QualTypeNames.h"
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/AST/Type.h"
@@ -5678,6 +5679,10 @@ QualType getApproximateType(const Expr *E) {
         return getApproximateType(VD->getInit());
     }
   }
+  if (const auto *UO = llvm::dyn_cast<UnaryOperator>(E)) {
+    if (UO->getOpcode() == UnaryOperatorKind::UO_Deref)
+      return UO->getSubExpr()->getType()->getPointeeType();
+  }
   return Unresolved;
 }
 
diff --git a/clang/test/CodeCompletion/member-access.cpp b/clang/test/CodeCompletion/member-access.cpp
index 474b909ab115..9f8c21c0bca6 100644
--- a/clang/test/CodeCompletion/member-access.cpp
+++ b/clang/test/CodeCompletion/member-access.cpp
@@ -348,7 +348,23 @@ namespace function_can_be_call {
     T foo(U, V);
   };
 
-  &S::f
-  // RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:351:7 %s -o - | FileCheck -check-prefix=CHECK_FUNCTION_CAN_BE_CALL %s
+  void test() {
+    &S::f
+  }
+  // RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:352:9 %s -o - | FileCheck -check-prefix=CHECK_FUNCTION_CAN_BE_CALL %s
   // CHECK_FUNCTION_CAN_BE_CALL: COMPLETION: foo : [#T#]foo<<#typename T#>, <#typename U#>>(<#U#>, <#V#>)
 }
+
+namespace deref_dependent_this {
+template <typename T>
+class A {
+  int field;
+
+  void function() {
+    (*this).field;
+// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:364:13 %s -o - | FileCheck -check-prefix=CHECK-DEREF-THIS %s
+// CHECK-DEREF-THIS: field : [#int#]field
+// CHECK-DEREF-THIS: [#void#]function()
+  }
+};
+}
-- 
GitLab


From 7196518f1f32d4d294bcf050790a2ba26b819645 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 25 Mar 2024 21:46:57 -0700
Subject: [PATCH 238/404] [ELF,test] Improve -r incompatibility tests

---
 lld/test/ELF/driver.test | 48 +++++++++++++++-------------------------
 1 file changed, 18 insertions(+), 30 deletions(-)

diff --git a/lld/test/ELF/driver.test b/lld/test/ELF/driver.test
index b8f4584a3267..49deb902aa0f 100644
--- a/lld/test/ELF/driver.test
+++ b/lld/test/ELF/driver.test
@@ -3,55 +3,47 @@
 # RUN: not ld.lld --unknown1 --unkn=own2 -m foo /no/such/file -lnosuchlib \
 # RUN:   2>&1 | FileCheck -check-prefix=UNKNOWN %s
 
-# UNKNOWN: unknown argument '--unknown1'
-# UNKNOWN: unknown argument '--unkn=own2'
-# UNKNOWN: unknown emulation: foo
-# UNKNOWN: cannot open /no/such/file
-# UNKNOWN: unable to find library -lnosuchlib
+# UNKNOWN: error: unknown argument '--unknown1'
+# UNKNOWN: error: unknown argument '--unkn=own2'
+# UNKNOWN: error: unknown emulation: foo
+# UNKNOWN: error: cannot open /no/such/file
+# UNKNOWN: error: unable to find library -lnosuchlib
 
 # RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t
 # RUN: not ld.lld %t -o /no/such/file 2>&1 | FileCheck -check-prefix=MISSING %s
-# MISSING: cannot open output file /no/such/file
+# MISSING: error: cannot open output file /no/such/file
 
 # RUN: ld.lld --help 2>&1 | FileCheck -check-prefix=HELP %s
 # HELP: USAGE:
 # HELP: : supported targets:{{.*}} elf
 
 # RUN: not ld.lld --versin 2>&1 | FileCheck -check-prefix=SPELLVERSION %s
-# SPELLVERSION: unknown argument '--versin', did you mean '--version'
+# SPELLVERSION: error: unknown argument '--versin', did you mean '--version'
 
 ## Attempt to link DSO with -r
 # RUN: ld.lld -shared %t -o %t.so
 # RUN: not ld.lld -r %t.so %t -o /dev/null 2>&1 | FileCheck -check-prefix=ERR %s
-# ERR: attempted static link of dynamic object
+# ERR: error: attempted static link of dynamic object
 
-## Attempt to use -r and -shared together
-# RUN: not ld.lld -r -shared %t -o /dev/null 2>&1 | FileCheck -check-prefix=ERR2 %s
-# ERR2: -r and -shared may not be used together
+# RUN: not ld.lld -r -shared -pie --export-dynamic %t -o /dev/null 2>&1 | FileCheck -check-prefix=ERR2 %s
+# ERR2: error: -r and -shared may not be used together
+# ERR2: error: -r and -pie may not be used together
+# ERR2: error: -r and --export-dynamic may not be used together
 
-## Attempt to use -r and --gdb-index together
-# RUN: not ld.lld -r --gdb-index %t -o /dev/null 2>&1 | FileCheck -check-prefix=ERR4 %s
-# ERR4: -r and --gdb-index may not be used together
+# RUN: not ld.lld -r --icf=all --gdb-index %t -o /dev/null 2>&1 | FileCheck -check-prefix=ERR4 %s
+# ERR4: error: -r and --gdb-index may not be used together
+# ERR4: error: -r and --icf may not be used together
 
-## Attempt to use -r and --icf together
-# RUN: not ld.lld -r --icf=all %t -o /dev/null 2>&1 | FileCheck -check-prefix=ERR5 %s
-# ERR5: -r and --icf may not be used together
-
-## Attempt to use -r and -pie together
-# RUN: not ld.lld -r -pie %t -o /dev/null 2>&1 | FileCheck -check-prefix=ERR6 %s
-# ERR6: -r and -pie may not be used together
-
-## Attempt to use -shared and -pie together
 # RUN: not ld.lld -shared -pie %t -o /dev/null 2>&1 | FileCheck -check-prefix=ERR7 %s
-# ERR7: -shared and -pie may not be used together
+# ERR7: error: -shared and -pie may not be used together
 
 ## "--output=foo" is equivalent to "-o foo".
 # RUN: not ld.lld %t --output=/no/such/file 2>&1 | FileCheck -check-prefix=ERR8 %s
-# ERR8: cannot open output file /no/such/file
+# ERR8: error: cannot open output file /no/such/file
 
 ## "-output=foo" is equivalent to "-o utput=foo".
 # RUN: not ld.lld %t -output=/no/such/file 2>&1 | FileCheck -check-prefix=ERR9 %s
-# ERR9: cannot open output file utput=/no/such/file
+# ERR9: error: cannot open output file utput=/no/such/file
 
 # RUN: ld.lld %t -z foo -o /dev/null 2>&1 | FileCheck -check-prefix=ERR10 %s --implicit-check-not=warning:
 # RUN: ld.lld %t -z foo -z rel -z rela -z max-page-size=1 -z common-page-size=1 -o /dev/null --version 2>&1 | \
@@ -68,10 +60,6 @@
 # RUN: not ld.lld %t -z max-page-size 2>&1 | FileCheck -check-prefix=ERR11 %s
 # ERR11: error: invalid max-page-size:
 
-## Attempt to use -r and --export-dynamic together
-# RUN: not ld.lld -r -export-dynamic %t -o /dev/null 2>&1 | FileCheck -check-prefix=ERR12 %s
-# ERR12: -r and --export-dynamic may not be used together
-
 .globl _start
 _start:
   nop
-- 
GitLab


From 3e046ee0c944b5a55f30b3ce0153791d91dbb134 Mon Sep 17 00:00:00 2001
From: Sam Clegg <sbc@chromium.org>
Date: Mon, 25 Mar 2024 21:50:00 -0700
Subject: [PATCH 239/404] [lld][ELF] Add `--why-extract` for bitcode libcalls
 (#78781)

The Wasm linker already records these and its seems useful to do so.
---
 lld/ELF/Driver.cpp                  | 5 ++++-
 lld/test/ELF/lto/libcall-archive.ll | 6 +++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 048ef0a60618..f14a2376601b 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -2114,8 +2114,11 @@ static void handleUndefinedGlob(StringRef arg) {
 
 static void handleLibcall(StringRef name) {
   Symbol *sym = symtab.find(name);
-  if (sym && sym->isLazy() && isa<BitcodeFile>(sym->file))
+  if (sym && sym->isLazy() && isa<BitcodeFile>(sym->file)) {
+    if (!config->whyExtract.empty())
+      ctx.whyExtractRecords.emplace_back("<libcall>", sym->file, *sym);
     sym->extract();
+  }
 }
 
 static void writeArchiveStats() {
diff --git a/lld/test/ELF/lto/libcall-archive.ll b/lld/test/ELF/lto/libcall-archive.ll
index bd91d0391dc2..84ddc1f55587 100644
--- a/lld/test/ELF/lto/libcall-archive.ll
+++ b/lld/test/ELF/lto/libcall-archive.ll
@@ -4,11 +4,15 @@
 ; RUN: llvm-as -o %t2.o %S/Inputs/libcall-archive.ll
 ; RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux -o %t3.o %S/Inputs/libcall-archive.s
 ; RUN: llvm-ar rcs %t.a %t2.o %t3.o
-; RUN: ld.lld -o %t %t.o %t.a
+; RUN: ld.lld --why-extract=why.txt -o %t %t.o %t.a
+; RUN: FileCheck %s --input-file=why.txt --check-prefix=CHECK-WHY
 ; RUN: llvm-nm %t | FileCheck %s
 ; RUN: ld.lld -o %t2 %t.o --start-lib %t2.o %t3.o --end-lib
 ; RUN: llvm-nm %t2 | FileCheck %s
 
+; CHECK-WHY: reference extracted symbol
+; CHECK-WHY-NEXT: <libcall> {{.*}}tmp.a({{.*}}tmp2.o) memcpy
+
 ; CHECK-NOT: T __sync_val_compare_and_swap_8
 ; CHECK: T _start
 ; CHECK: T memcpy
-- 
GitLab


From 08a321e1b7ffdd72154ae808821d20fe511c83b6 Mon Sep 17 00:00:00 2001
From: Kai Sasaki <lewuathe@gmail.com>
Date: Tue, 26 Mar 2024 13:50:28 +0900
Subject: [PATCH 240/404] [mlir][complex] Canonicalize complex.div by one
 (#85513)

We can canonicalize the complex.div if the divisor is one (real = 1.0,
imag = 0.0) with the input number itself.

Ref: https://www.cuemath.com/numbers/division-of-complex-numbers/
---
 .../mlir/Dialect/Complex/IR/ComplexOps.td     |  2 +
 mlir/lib/Dialect/Complex/IR/ComplexOps.cpp    | 26 +++++++++
 mlir/test/Dialect/Complex/canonicalize.mlir   | 56 +++++++++++++++++++
 3 files changed, 84 insertions(+)

diff --git a/mlir/include/mlir/Dialect/Complex/IR/ComplexOps.td b/mlir/include/mlir/Dialect/Complex/IR/ComplexOps.td
index e19d714cadf8..d660292478b1 100644
--- a/mlir/include/mlir/Dialect/Complex/IR/ComplexOps.td
+++ b/mlir/include/mlir/Dialect/Complex/IR/ComplexOps.td
@@ -225,6 +225,8 @@ def DivOp : ComplexArithmeticOp<"div"> {
     %a = complex.div %b, %c : complex<f32>
     ```
   }];
+
+  let hasFolder = 1;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp b/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp
index 5529dccaf1de..1c81433bc3e9 100644
--- a/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp
+++ b/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp
@@ -369,6 +369,32 @@ OpFoldResult MulOp::fold(FoldAdaptor adaptor) {
   return {};
 }
 
+//===----------------------------------------------------------------------===//
+// DivOp
+//===----------------------------------------------------------------------===//
+
+OpFoldResult DivOp::fold(FoldAdaptor adaptor) {
+  auto rhs = adaptor.getRhs();
+  if (!rhs)
+    return {};
+
+  ArrayAttr arrayAttr = rhs.dyn_cast<ArrayAttr>();
+  if (!arrayAttr || arrayAttr.size() != 2)
+    return {};
+
+  APFloat real = cast<FloatAttr>(arrayAttr[0]).getValue();
+  APFloat imag = cast<FloatAttr>(arrayAttr[1]).getValue();
+
+  if (!imag.isZero())
+    return {};
+
+  // complex.div(a, complex.constant<1.0, 0.0>) -> a
+  if (real == APFloat(real.getSemantics(), 1))
+    return getLhs();
+
+  return {};
+}
+
 //===----------------------------------------------------------------------===//
 // TableGen'd op method definitions
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/Complex/canonicalize.mlir b/mlir/test/Dialect/Complex/canonicalize.mlir
index 64c3f313dda9..16fa0fdf56a1 100644
--- a/mlir/test/Dialect/Complex/canonicalize.mlir
+++ b/mlir/test/Dialect/Complex/canonicalize.mlir
@@ -281,3 +281,59 @@ func.func @double_reverse_bitcast(%arg0 : complex<f32>) -> f64 {
   // CHECK: return %[[R0]] : f64
   func.return %1 : f64
 }
+
+
+// CHECK-LABEL: func @div_one_f16
+//  CHECK-SAME: (%[[ARG0:.*]]: f16, %[[ARG1:.*]]: f16) -> complex<f16>
+func.func @div_one_f16(%arg0: f16, %arg1: f16) -> complex<f16> {
+  %create = complex.create %arg0, %arg1: complex<f16>  
+  %one = complex.constant [1.0 : f16, 0.0 : f16] : complex<f16>
+  %div = complex.div %create, %one : complex<f16>
+  // CHECK: %[[CREATE:.*]] = complex.create %[[ARG0]], %[[ARG1]] : complex<f16>
+  // CHECK-NEXT: return %[[CREATE]]
+  return %div : complex<f16>
+}
+
+// CHECK-LABEL: func @div_one_f32
+//  CHECK-SAME: (%[[ARG0:.*]]: f32, %[[ARG1:.*]]: f32) -> complex<f32>
+func.func @div_one_f32(%arg0: f32, %arg1: f32) -> complex<f32> {
+  %create = complex.create %arg0, %arg1: complex<f32>  
+  %one = complex.constant [1.0 : f32, 0.0 : f32] : complex<f32>
+  %div = complex.div %create, %one : complex<f32>
+  // CHECK: %[[CREATE:.*]] = complex.create %[[ARG0]], %[[ARG1]] : complex<f32>
+  // CHECK-NEXT: return %[[CREATE]]
+  return %div : complex<f32>
+}
+
+// CHECK-LABEL: func @div_one_f64
+//  CHECK-SAME: (%[[ARG0:.*]]: f64, %[[ARG1:.*]]: f64) -> complex<f64>
+func.func @div_one_f64(%arg0: f64, %arg1: f64) -> complex<f64> {
+  %create = complex.create %arg0, %arg1: complex<f64>  
+  %one = complex.constant [1.0 : f64, 0.0 : f64] : complex<f64>
+  %div = complex.div %create, %one : complex<f64>
+  // CHECK: %[[CREATE:.*]] = complex.create %[[ARG0]], %[[ARG1]] : complex<f64>
+  // CHECK-NEXT: return %[[CREATE]]
+  return %div : complex<f64>
+}
+
+// CHECK-LABEL: func @div_one_f80
+//  CHECK-SAME: (%[[ARG0:.*]]: f80, %[[ARG1:.*]]: f80) -> complex<f80>
+func.func @div_one_f80(%arg0: f80, %arg1: f80) -> complex<f80> {
+  %create = complex.create %arg0, %arg1: complex<f80>  
+  %one = complex.constant [1.0 : f80, 0.0 : f80] : complex<f80>
+  %div = complex.div %create, %one : complex<f80>
+  // CHECK: %[[CREATE:.*]] = complex.create %[[ARG0]], %[[ARG1]] : complex<f80>
+  // CHECK-NEXT: return %[[CREATE]]
+  return %div : complex<f80>
+}
+
+// CHECK-LABEL: func @div_one_f128
+//  CHECK-SAME: (%[[ARG0:.*]]: f128, %[[ARG1:.*]]: f128) -> complex<f128>
+func.func @div_one_f128(%arg0: f128, %arg1: f128) -> complex<f128> {
+  %create = complex.create %arg0, %arg1: complex<f128>  
+  %one = complex.constant [1.0 : f128, 0.0 : f128] : complex<f128>
+  %div = complex.div %create, %one : complex<f128>
+  // CHECK: %[[CREATE:.*]] = complex.create %[[ARG0]], %[[ARG1]] : complex<f128>
+  // CHECK-NEXT: return %[[CREATE]]
+  return %div : complex<f128>
+}
-- 
GitLab


From 64a71147024ebcc58655adc28a17c9f17a517aed Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang@amd.com>
Date: Mon, 25 Mar 2024 21:53:19 -0700
Subject: [PATCH 241/404] AMDGPU: Simplify SMInstruction definitions, NFC
 (#86613)

Copy OtherPredicates from Pseudo to Real. Real should inherit predicates
from the corresponding Pseudo
---
 llvm/lib/Target/AMDGPU/SMInstructions.td | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td
index a91fb87998fe..afc9da07bc96 100644
--- a/llvm/lib/Target/AMDGPU/SMInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SMInstructions.td
@@ -67,6 +67,7 @@ class SM_Real <SM_Pseudo ps, string opName = ps.Mnemonic>
   let UseNamedOperandTable = ps.UseNamedOperandTable;
   let SchedRW              = ps.SchedRW;
   let SubtargetPredicate   = ps.SubtargetPredicate;
+  let OtherPredicates      = ps.OtherPredicates;
   let AsmMatchConverter    = ps.AsmMatchConverter;
   let IsAtomicRet          = ps.IsAtomicRet;
   let IsAtomicNoRet        = ps.IsAtomicNoRet;
@@ -1172,11 +1173,9 @@ defm S_LOAD_DWORDX4          : SM_Real_Loads_gfx10<0x002>;
 defm S_LOAD_DWORDX8          : SM_Real_Loads_gfx10<0x003>;
 defm S_LOAD_DWORDX16         : SM_Real_Loads_gfx10<0x004>;
 
-let SubtargetPredicate = HasScalarFlatScratchInsts in {
 defm S_SCRATCH_LOAD_DWORD    : SM_Real_Loads_gfx10<0x005>;
 defm S_SCRATCH_LOAD_DWORDX2  : SM_Real_Loads_gfx10<0x006>;
 defm S_SCRATCH_LOAD_DWORDX4  : SM_Real_Loads_gfx10<0x007>;
-} // End SubtargetPredicate = HasScalarFlatScratchInsts
 
 defm S_BUFFER_LOAD_DWORD     : SM_Real_Loads_gfx10<0x008>;
 defm S_BUFFER_LOAD_DWORDX2   : SM_Real_Loads_gfx10<0x009>;
@@ -1184,19 +1183,15 @@ defm S_BUFFER_LOAD_DWORDX4   : SM_Real_Loads_gfx10<0x00a>;
 defm S_BUFFER_LOAD_DWORDX8   : SM_Real_Loads_gfx10<0x00b>;
 defm S_BUFFER_LOAD_DWORDX16  : SM_Real_Loads_gfx10<0x00c>;
 
-let SubtargetPredicate = HasScalarStores in {
 defm S_STORE_DWORD           : SM_Real_Stores_gfx10<0x010>;
 defm S_STORE_DWORDX2         : SM_Real_Stores_gfx10<0x011>;
 defm S_STORE_DWORDX4         : SM_Real_Stores_gfx10<0x012>;
-let OtherPredicates = [HasScalarFlatScratchInsts] in {
 defm S_SCRATCH_STORE_DWORD   : SM_Real_Stores_gfx10<0x015>;
 defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x016>;
 defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x017>;
-} // End OtherPredicates = [HasScalarFlatScratchInsts]
 defm S_BUFFER_STORE_DWORD    : SM_Real_Stores_gfx10<0x018>;
 defm S_BUFFER_STORE_DWORDX2  : SM_Real_Stores_gfx10<0x019>;
 defm S_BUFFER_STORE_DWORDX4  : SM_Real_Stores_gfx10<0x01a>;
-} // End SubtargetPredicate = HasScalarStores
 
 def S_MEMREALTIME_gfx10              : SMEM_Real_gfx10<0x025, S_MEMREALTIME>;
 def S_MEMTIME_gfx10                  : SMEM_Real_gfx10<0x024, S_MEMTIME>;
@@ -1204,9 +1199,7 @@ def S_GL1_INV_gfx10                  : SMEM_Real_gfx10<0x01f, S_GL1_INV>;
 def S_GET_WAVEID_IN_WORKGROUP_gfx10  : SMEM_Real_gfx10<0x02a, S_GET_WAVEID_IN_WORKGROUP>;
 def S_DCACHE_INV_gfx10               : SMEM_Real_gfx10<0x020, S_DCACHE_INV>;
 
-let SubtargetPredicate = HasScalarStores in {
 def S_DCACHE_WB_gfx10                : SMEM_Real_gfx10<0x021, S_DCACHE_WB>;
-} // End SubtargetPredicate = HasScalarStores
 
 multiclass SM_Real_Probe_gfx10<bits<8> op> {
   defvar ps = NAME;
@@ -1243,8 +1236,6 @@ multiclass SM_Real_Atomics_gfx10<bits<8> op> {
   def _SGPR_IMM_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM_RTN)>;
 }
 
-let SubtargetPredicate = HasScalarAtomics in {
-
 defm S_BUFFER_ATOMIC_SWAP         : SM_Real_Atomics_gfx10 <0x40>;
 defm S_BUFFER_ATOMIC_CMPSWAP      : SM_Real_Atomics_gfx10 <0x41>;
 defm S_BUFFER_ATOMIC_ADD          : SM_Real_Atomics_gfx10 <0x42>;
@@ -1311,8 +1302,6 @@ multiclass SM_Real_Discard_gfx10<bits<8> op> {
 defm S_DCACHE_DISCARD    : SM_Real_Discard_gfx10 <0x28>;
 defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_gfx10 <0x29>;
 
-} // End SubtargetPredicate = HasScalarAtomics
-
 def SMInfoTable : GenericTable {
   let FilterClass = "SM_Real";
   let CppTypeName = "SMInfo";
-- 
GitLab


From c6a65e4b0c80245d766ae2f2f7305b5371d096f5 Mon Sep 17 00:00:00 2001
From: Younan Zhang <zyn7109@gmail.com>
Date: Tue, 26 Mar 2024 12:57:09 +0800
Subject: [PATCH 242/404] [clangd] Support go-to-definition on type hints. The
 protocol part (#85497)

This is in preparation for implementing go-to-definition support on type
inlay hints, switching the `label` field within the InlayHint protocol from a
string to an array of `InlayHintLabelPart`.
---
 clang-tools-extra/clangd/ClangdLSPServer.cpp  |  2 +-
 clang-tools-extra/clangd/InlayHints.cpp       |  5 +-
 clang-tools-extra/clangd/Protocol.cpp         | 31 ++++++++++++
 clang-tools-extra/clangd/Protocol.h           | 47 ++++++++++++++++++-
 clang-tools-extra/clangd/test/inlayHints.test |  6 ++-
 clang-tools-extra/clangd/tool/Check.cpp       |  8 +++-
 .../clangd/unittests/InlayHintTests.cpp       |  9 ++--
 7 files changed, 98 insertions(+), 10 deletions(-)

diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp
index f29dadde2b86..7fd599d4e1a0 100644
--- a/clang-tools-extra/clangd/ClangdLSPServer.cpp
+++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp
@@ -1390,7 +1390,7 @@ void ClangdLSPServer::onClangdInlayHints(const InlayHintsParams &Params,
           // Extension doesn't have paddingLeft/Right so adjust the label
           // accordingly.
           {"label",
-           ((Hint.paddingLeft ? " " : "") + llvm::StringRef(Hint.label) +
+           ((Hint.paddingLeft ? " " : "") + llvm::StringRef(Hint.joinLabels()) +
             (Hint.paddingRight ? " " : ""))
                .str()},
       });
diff --git a/clang-tools-extra/clangd/InlayHints.cpp b/clang-tools-extra/clangd/InlayHints.cpp
index a0ebc631ef82..cd4f1931b3ce 100644
--- a/clang-tools-extra/clangd/InlayHints.cpp
+++ b/clang-tools-extra/clangd/InlayHints.cpp
@@ -977,8 +977,9 @@ private:
       return;
     bool PadLeft = Prefix.consume_front(" ");
     bool PadRight = Suffix.consume_back(" ");
-    Results.push_back(InlayHint{LSPPos, (Prefix + Label + Suffix).str(), Kind,
-                                PadLeft, PadRight, LSPRange});
+    Results.push_back(InlayHint{LSPPos,
+                                /*label=*/{(Prefix + Label + Suffix).str()},
+                                Kind, PadLeft, PadRight, LSPRange});
   }
 
   // Get the range of the main file that *exactly* corresponds to R.
diff --git a/clang-tools-extra/clangd/Protocol.cpp b/clang-tools-extra/clangd/Protocol.cpp
index c6553e00dcae..c08f80442eaa 100644
--- a/clang-tools-extra/clangd/Protocol.cpp
+++ b/clang-tools-extra/clangd/Protocol.cpp
@@ -1501,6 +1501,10 @@ bool operator<(const InlayHint &A, const InlayHint &B) {
   return std::tie(A.position, A.range, A.kind, A.label) <
          std::tie(B.position, B.range, B.kind, B.label);
 }
+std::string InlayHint::joinLabels() const {
+  return llvm::join(llvm::map_range(label, [](auto &L) { return L.value; }),
+                    "");
+}
 
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, InlayHintKind Kind) {
   auto ToString = [](InlayHintKind K) {
@@ -1519,6 +1523,33 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, InlayHintKind Kind) {
   return OS << ToString(Kind);
 }
 
+llvm::json::Value toJSON(const InlayHintLabelPart &L) {
+  llvm::json::Object Result{{"value", L.value}};
+  if (L.tooltip)
+    Result["tooltip"] = *L.tooltip;
+  if (L.location)
+    Result["location"] = *L.location;
+  if (L.command)
+    Result["command"] = *L.command;
+  return Result;
+}
+
+bool operator==(const InlayHintLabelPart &LHS, const InlayHintLabelPart &RHS) {
+  return std::tie(LHS.value, LHS.location) == std::tie(RHS.value, RHS.location);
+}
+
+bool operator<(const InlayHintLabelPart &LHS, const InlayHintLabelPart &RHS) {
+  return std::tie(LHS.value, LHS.location) < std::tie(RHS.value, RHS.location);
+}
+
+llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
+                              const InlayHintLabelPart &L) {
+  OS << L.value;
+  if (L.location)
+    OS << " (" << L.location << ")";
+  return OS;
+}
+
 static const char *toString(OffsetEncoding OE) {
   switch (OE) {
   case OffsetEncoding::UTF8:
diff --git a/clang-tools-extra/clangd/Protocol.h b/clang-tools-extra/clangd/Protocol.h
index 358d4c6feaf8..a0f8b04bc4ff 100644
--- a/clang-tools-extra/clangd/Protocol.h
+++ b/clang-tools-extra/clangd/Protocol.h
@@ -1689,6 +1689,48 @@ enum class InlayHintKind {
 };
 llvm::json::Value toJSON(const InlayHintKind &);
 
+/// An inlay hint label part allows for interactive and composite labels
+/// of inlay hints.
+struct InlayHintLabelPart {
+
+  InlayHintLabelPart() = default;
+
+  InlayHintLabelPart(std::string value,
+                     std::optional<Location> location = std::nullopt)
+      : value(std::move(value)), location(std::move(location)) {}
+
+  /// The value of this label part.
+  std::string value;
+
+  /// The tooltip text when you hover over this label part. Depending on
+  /// the client capability `inlayHint.resolveSupport`, clients might resolve
+  /// this property late using the resolve request.
+  std::optional<MarkupContent> tooltip;
+
+  /// An optional source code location that represents this
+  /// label part.
+  ///
+  /// The editor will use this location for the hover and for code navigation
+  /// features: This part will become a clickable link that resolves to the
+  /// definition of the symbol at the given location (not necessarily the
+  /// location itself), it shows the hover that shows at the given location,
+  /// and it shows a context menu with further code navigation commands.
+  ///
+  /// Depending on the client capability `inlayHint.resolveSupport` clients
+  /// might resolve this property late using the resolve request.
+  std::optional<Location> location;
+
+  /// An optional command for this label part.
+  ///
+  /// Depending on the client capability `inlayHint.resolveSupport` clients
+  /// might resolve this property late using the resolve request.
+  std::optional<Command> command;
+};
+llvm::json::Value toJSON(const InlayHintLabelPart &);
+bool operator==(const InlayHintLabelPart &, const InlayHintLabelPart &);
+bool operator<(const InlayHintLabelPart &, const InlayHintLabelPart &);
+llvm::raw_ostream &operator<<(llvm::raw_ostream &, const InlayHintLabelPart &);
+
 /// Inlay hint information.
 struct InlayHint {
   /// The position of this hint.
@@ -1698,7 +1740,7 @@ struct InlayHint {
   /// InlayHintLabelPart label parts.
   ///
   /// *Note* that neither the string nor the label part can be empty.
-  std::string label;
+  std::vector<InlayHintLabelPart> label;
 
   /// The kind of this hint. Can be omitted in which case the client should fall
   /// back to a reasonable default.
@@ -1724,6 +1766,9 @@ struct InlayHint {
   /// The range allows clients more flexibility of when/how to display the hint.
   /// This is an (unserialized) clangd extension.
   Range range;
+
+  /// Join the label[].value together.
+  std::string joinLabels() const;
 };
 llvm::json::Value toJSON(const InlayHint &);
 bool operator==(const InlayHint &, const InlayHint &);
diff --git a/clang-tools-extra/clangd/test/inlayHints.test b/clang-tools-extra/clangd/test/inlayHints.test
index 8f302dd17a54..e5b3c0fb0b96 100644
--- a/clang-tools-extra/clangd/test/inlayHints.test
+++ b/clang-tools-extra/clangd/test/inlayHints.test
@@ -51,7 +51,11 @@
 # CHECK-NEXT:  "result": [
 # CHECK-NEXT:    {
 # CHECK-NEXT:      "kind": 2,
-# CHECK-NEXT:      "label": "bar:",
+# CHECK-NEXT:      "label": [
+# CHECK-NEXT:        {
+# CHECK-NEXT:          "value": "bar:"
+# CHECK-NEXT:        }
+# CHECK-NEXT:      ],
 # CHECK-NEXT:      "paddingLeft": false,
 # CHECK-NEXT:      "paddingRight": true,
 # CHECK-NEXT:      "position": {
diff --git a/clang-tools-extra/clangd/tool/Check.cpp b/clang-tools-extra/clangd/tool/Check.cpp
index 45e2e1e278de..25005ec1bd04 100644
--- a/clang-tools-extra/clangd/tool/Check.cpp
+++ b/clang-tools-extra/clangd/tool/Check.cpp
@@ -367,7 +367,13 @@ public:
     auto Hints = inlayHints(*AST, LineRange);
 
     for (const auto &Hint : Hints) {
-      vlog("  {0} {1} {2}", Hint.kind, Hint.position, Hint.label);
+      vlog("  {0} {1} [{2}]", Hint.kind, Hint.position, [&] {
+        return llvm::join(llvm::map_range(Hint.label,
+                                          [&](auto &L) {
+                                            return llvm::formatv("{{{0}}", L);
+                                          }),
+                          ", ");
+      }());
     }
   }
 
diff --git a/clang-tools-extra/clangd/unittests/InlayHintTests.cpp b/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
index 0fff0dfca6c9..5b1531eb2fa6 100644
--- a/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
+++ b/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
@@ -25,7 +25,7 @@ namespace clangd {
 
 llvm::raw_ostream &operator<<(llvm::raw_ostream &Stream,
                               const InlayHint &Hint) {
-  return Stream << Hint.label << "@" << Hint.range;
+  return Stream << Hint.joinLabels() << "@" << Hint.range;
 }
 
 namespace {
@@ -57,10 +57,11 @@ struct ExpectedHint {
 
 MATCHER_P2(HintMatcher, Expected, Code, llvm::to_string(Expected)) {
   llvm::StringRef ExpectedView(Expected.Label);
-  if (arg.label != ExpectedView.trim(" ") ||
+  std::string ResultLabel = arg.joinLabels();
+  if (ResultLabel != ExpectedView.trim(" ") ||
       arg.paddingLeft != ExpectedView.starts_with(" ") ||
       arg.paddingRight != ExpectedView.ends_with(" ")) {
-    *result_listener << "label is '" << arg.label << "'";
+    *result_listener << "label is '" << ResultLabel << "'";
     return false;
   }
   if (arg.range != Code.range(Expected.RangeName)) {
@@ -72,7 +73,7 @@ MATCHER_P2(HintMatcher, Expected, Code, llvm::to_string(Expected)) {
   return true;
 }
 
-MATCHER_P(labelIs, Label, "") { return arg.label == Label; }
+MATCHER_P(labelIs, Label, "") { return arg.joinLabels() == Label; }
 
 Config noHintsConfig() {
   Config C;
-- 
GitLab


From 817f453aa576286aaca0a6b0244e6ab08516b80c Mon Sep 17 00:00:00 2001
From: Shih-Po Hung <shihpo.hung@sifive.com>
Date: Tue, 26 Mar 2024 14:15:40 +0800
Subject: [PATCH 243/404] [RISCV][TTI] Refactor getCastInstrCost to exit early
 (#86619)

To reduce the indentation by using early returns, this patch hoist the
return for illegal type and non vector type earlier.

It should mostly be an NFC.
---
 .../Target/RISCV/RISCVTargetTransformInfo.cpp | 133 +++++++++---------
 1 file changed, 65 insertions(+), 68 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index f75b3d3caa62..65142a03f0a6 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -897,76 +897,73 @@ InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
                                                TTI::CastContextHint CCH,
                                                TTI::TargetCostKind CostKind,
                                                const Instruction *I) {
-  if (isa<VectorType>(Dst) && isa<VectorType>(Src)) {
-    // FIXME: Need to compute legalizing cost for illegal types.
-    if (!isTypeLegal(Src) || !isTypeLegal(Dst))
-      return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
-
-    // Skip if element size of Dst or Src is bigger than ELEN.
-    if (Src->getScalarSizeInBits() > ST->getELen() ||
-        Dst->getScalarSizeInBits() > ST->getELen())
-      return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
-
-    int ISD = TLI->InstructionOpcodeToISD(Opcode);
-    assert(ISD && "Invalid opcode");
-
-    // FIXME: Need to consider vsetvli and lmul.
-    int PowDiff = (int)Log2_32(Dst->getScalarSizeInBits()) -
-                  (int)Log2_32(Src->getScalarSizeInBits());
-    switch (ISD) {
-    case ISD::SIGN_EXTEND:
-    case ISD::ZERO_EXTEND:
-      if (Src->getScalarSizeInBits() == 1) {
-        // We do not use vsext/vzext to extend from mask vector.
-        // Instead we use the following instructions to extend from mask vector:
-        // vmv.v.i v8, 0
-        // vmerge.vim v8, v8, -1, v0
-        return 2;
-      }
-      return 1;
-    case ISD::TRUNCATE:
-      if (Dst->getScalarSizeInBits() == 1) {
-        // We do not use several vncvt to truncate to mask vector. So we could
-        // not use PowDiff to calculate it.
-        // Instead we use the following instructions to truncate to mask vector:
-        // vand.vi v8, v8, 1
-        // vmsne.vi v0, v8, 0
-        return 2;
-      }
-      [[fallthrough]];
-    case ISD::FP_EXTEND:
-    case ISD::FP_ROUND:
-      // Counts of narrow/widen instructions.
-      return std::abs(PowDiff);
-    case ISD::FP_TO_SINT:
-    case ISD::FP_TO_UINT:
-    case ISD::SINT_TO_FP:
-    case ISD::UINT_TO_FP:
-      if (Src->getScalarSizeInBits() == 1 || Dst->getScalarSizeInBits() == 1) {
-        // The cost of convert from or to mask vector is different from other
-        // cases. We could not use PowDiff to calculate it.
-        // For mask vector to fp, we should use the following instructions:
-        // vmv.v.i v8, 0
-        // vmerge.vim v8, v8, -1, v0
-        // vfcvt.f.x.v v8, v8
-
-        // And for fp vector to mask, we use:
-        // vfncvt.rtz.x.f.w v9, v8
-        // vand.vi v8, v9, 1
-        // vmsne.vi v0, v8, 0
-        return 3;
-      }
-      if (std::abs(PowDiff) <= 1)
-        return 1;
-      // Backend could lower (v[sz]ext i8 to double) to vfcvt(v[sz]ext.f8 i8),
-      // so it only need two conversion.
-      if (Src->isIntOrIntVectorTy())
-        return 2;
-      // Counts of narrow/widen instructions.
-      return std::abs(PowDiff);
+  bool IsVectorType = isa<VectorType>(Dst) && isa<VectorType>(Src);
+  bool IsTypeLegal = isTypeLegal(Src) && isTypeLegal(Dst) &&
+                     (Src->getScalarSizeInBits() <= ST->getELen()) &&
+                     (Dst->getScalarSizeInBits() <= ST->getELen());
+
+  // FIXME: Need to compute legalizing cost for illegal types.
+  if (!IsVectorType || !IsTypeLegal)
+    return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
+
+  int ISD = TLI->InstructionOpcodeToISD(Opcode);
+  assert(ISD && "Invalid opcode");
+
+  // FIXME: Need to consider vsetvli and lmul.
+  int PowDiff = (int)Log2_32(Dst->getScalarSizeInBits()) -
+                (int)Log2_32(Src->getScalarSizeInBits());
+  switch (ISD) {
+  case ISD::SIGN_EXTEND:
+  case ISD::ZERO_EXTEND:
+    if (Src->getScalarSizeInBits() == 1) {
+      // We do not use vsext/vzext to extend from mask vector.
+      // Instead we use the following instructions to extend from mask vector:
+      // vmv.v.i v8, 0
+      // vmerge.vim v8, v8, -1, v0
+      return 2;
     }
+    return 1;
+  case ISD::TRUNCATE:
+    if (Dst->getScalarSizeInBits() == 1) {
+      // We do not use several vncvt to truncate to mask vector. So we could
+      // not use PowDiff to calculate it.
+      // Instead we use the following instructions to truncate to mask vector:
+      // vand.vi v8, v8, 1
+      // vmsne.vi v0, v8, 0
+      return 2;
+    }
+    [[fallthrough]];
+  case ISD::FP_EXTEND:
+  case ISD::FP_ROUND:
+    // Counts of narrow/widen instructions.
+    return std::abs(PowDiff);
+  case ISD::FP_TO_SINT:
+  case ISD::FP_TO_UINT:
+  case ISD::SINT_TO_FP:
+  case ISD::UINT_TO_FP:
+    if (Src->getScalarSizeInBits() == 1 || Dst->getScalarSizeInBits() == 1) {
+      // The cost of convert from or to mask vector is different from other
+      // cases. We could not use PowDiff to calculate it.
+      // For mask vector to fp, we should use the following instructions:
+      // vmv.v.i v8, 0
+      // vmerge.vim v8, v8, -1, v0
+      // vfcvt.f.x.v v8, v8
+
+      // And for fp vector to mask, we use:
+      // vfncvt.rtz.x.f.w v9, v8
+      // vand.vi v8, v9, 1
+      // vmsne.vi v0, v8, 0
+      return 3;
+    }
+    if (std::abs(PowDiff) <= 1)
+      return 1;
+    // Backend could lower (v[sz]ext i8 to double) to vfcvt(v[sz]ext.f8 i8),
+    // so it only need two conversion.
+    if (Src->isIntOrIntVectorTy())
+      return 2;
+    // Counts of narrow/widen instructions.
+    return std::abs(PowDiff);
   }
-  return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
 }
 
 unsigned RISCVTTIImpl::getEstimatedVLFor(VectorType *Ty) {
-- 
GitLab


From 5dc0c75aabb9811e03cc8025905fed6dc2dd7bda Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung@sifive.com>
Date: Mon, 25 Mar 2024 23:31:08 -0700
Subject: [PATCH 244/404] [RISCV][TTI] Fix missing return in the end of
 function

---
 llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 65142a03f0a6..4bcb9acdae19 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -964,6 +964,7 @@ InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
     // Counts of narrow/widen instructions.
     return std::abs(PowDiff);
   }
+  return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
 }
 
 unsigned RISCVTTIImpl::getEstimatedVLFor(VectorType *Ty) {
-- 
GitLab


From 5c663aa9ae4656c46d09914994859118759b426b Mon Sep 17 00:00:00 2001
From: ZhangYin <zhangyin2018@iscas.ac.cn>
Date: Tue, 26 Mar 2024 14:38:20 +0800
Subject: [PATCH 245/404] [libc++] <experimental/simd> Reduce types for
 template parameter U in tests (#83781)

---
 .../simd/simd.class/simd_ctor_broadcast.pass.cpp     |  4 ++--
 .../simd/simd.class/simd_ctor_conversion.pass.cpp    |  4 ++--
 .../simd/simd.class/simd_ctor_load.pass.cpp          |  6 +++---
 .../simd_mask_ctor_conversion.pass.cpp               |  4 ++--
 .../simd.reference/reference_assignment.pass.cpp     |  6 +++---
 libcxx/test/std/experimental/simd/test_utils.h       | 12 ++++++++++++
 6 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/libcxx/test/std/experimental/simd/simd.class/simd_ctor_broadcast.pass.cpp b/libcxx/test/std/experimental/simd/simd.class/simd_ctor_broadcast.pass.cpp
index 679cb1aa167c..8a291632a8ab 100644
--- a/libcxx/test/std/experimental/simd/simd.class/simd_ctor_broadcast.pass.cpp
+++ b/libcxx/test/std/experimental/simd/simd.class/simd_ctor_broadcast.pass.cpp
@@ -41,7 +41,7 @@ struct CheckSimdBroadcastCtorFromVectorizedType {
     std::array<T, array_size> expected_value;
     std::fill(expected_value.begin(), expected_value.end(), 3);
 
-    types::for_each(arithmetic_no_bool_types(), BroadCastHelper<T, SimdAbi, array_size>(expected_value));
+    types::for_each(simd_test_types(), BroadCastHelper<T, SimdAbi, array_size>(expected_value));
   }
 };
 
@@ -110,7 +110,7 @@ template <class T, std::size_t>
 struct CheckBroadcastCtorTraits {
   template <class SimdAbi>
   void operator()() {
-    types::for_each(arithmetic_no_bool_types(), CheckBroadcastCtorTraitsHelper<T, SimdAbi>());
+    types::for_each(simd_test_types(), CheckBroadcastCtorTraitsHelper<T, SimdAbi>());
 
     static_assert(!has_broadcast_ctor<no_implicit_type<T>, T, SimdAbi>::value);
     static_assert(has_broadcast_ctor<implicit_type<T>, T, SimdAbi>::value);
diff --git a/libcxx/test/std/experimental/simd/simd.class/simd_ctor_conversion.pass.cpp b/libcxx/test/std/experimental/simd/simd.class/simd_ctor_conversion.pass.cpp
index 7ce4bed9c7db..653704cd245a 100644
--- a/libcxx/test/std/experimental/simd/simd.class/simd_ctor_conversion.pass.cpp
+++ b/libcxx/test/std/experimental/simd/simd.class/simd_ctor_conversion.pass.cpp
@@ -46,7 +46,7 @@ struct CheckConversionSimdCtor {
     for (size_t i = 0; i < array_size; ++i)
       expected_value[i] = static_cast<T>(i);
 
-    types::for_each(arithmetic_no_bool_types(), ConversionHelper<T, SimdAbi, array_size>(expected_value));
+    types::for_each(simd_test_types(), ConversionHelper<T, SimdAbi, array_size>(expected_value));
   }
 };
 
@@ -70,7 +70,7 @@ struct CheckConversionSimdCtorTraits {
   void operator()() {
     constexpr std::size_t array_size = ex::simd_size_v<T, SimdAbi>;
 
-    types::for_each(arithmetic_no_bool_types(), CheckConversionSimdCtorTraitsHelper<T, SimdAbi, array_size>());
+    types::for_each(simd_test_types(), CheckConversionSimdCtorTraitsHelper<T, SimdAbi, array_size>());
   }
 };
 
diff --git a/libcxx/test/std/experimental/simd/simd.class/simd_ctor_load.pass.cpp b/libcxx/test/std/experimental/simd/simd.class/simd_ctor_load.pass.cpp
index 3992f3e450cb..3f663d9c9735 100644
--- a/libcxx/test/std/experimental/simd/simd.class/simd_ctor_load.pass.cpp
+++ b/libcxx/test/std/experimental/simd/simd.class/simd_ctor_load.pass.cpp
@@ -59,9 +59,9 @@ struct CheckSimdLoadCtor {
   void operator()() {
     constexpr std::size_t array_size = ex::simd_size_v<T, SimdAbi>;
 
-    types::for_each(arithmetic_no_bool_types(), ElementAlignedLoadCtorHelper<T, SimdAbi, array_size>());
-    types::for_each(arithmetic_no_bool_types(), VectorAlignedLoadCtorHelper<T, SimdAbi, array_size>());
-    types::for_each(arithmetic_no_bool_types(), OveralignedLoadCtorHelper<T, SimdAbi, array_size>());
+    types::for_each(simd_test_types(), ElementAlignedLoadCtorHelper<T, SimdAbi, array_size>());
+    types::for_each(simd_test_types(), VectorAlignedLoadCtorHelper<T, SimdAbi, array_size>());
+    types::for_each(simd_test_types(), OveralignedLoadCtorHelper<T, SimdAbi, array_size>());
   }
 };
 
diff --git a/libcxx/test/std/experimental/simd/simd.mask.class/simd_mask_ctor_conversion.pass.cpp b/libcxx/test/std/experimental/simd/simd.mask.class/simd_mask_ctor_conversion.pass.cpp
index dc0764f937fc..7910b2cc2522 100644
--- a/libcxx/test/std/experimental/simd/simd.mask.class/simd_mask_ctor_conversion.pass.cpp
+++ b/libcxx/test/std/experimental/simd/simd.mask.class/simd_mask_ctor_conversion.pass.cpp
@@ -42,7 +42,7 @@ struct CheckConversionMaskCtor {
     constexpr std::size_t array_size = ex::simd_size_v<T, SimdAbi>;
     std::array<bool, array_size> expected_value{};
 
-    types::for_each(arithmetic_no_bool_types(), ConversionHelper<T, SimdAbi, array_size>(expected_value));
+    types::for_each(simd_test_types(), ConversionHelper<T, SimdAbi, array_size>(expected_value));
   }
 };
 
@@ -65,7 +65,7 @@ struct CheckConversionMaskCtorTraits {
   void operator()() {
     constexpr std::size_t array_size = ex::simd_size_v<T, SimdAbi>;
 
-    types::for_each(arithmetic_no_bool_types(), CheckConversionMaskCtorTraitsHelper<T, SimdAbi, array_size>());
+    types::for_each(simd_test_types(), CheckConversionMaskCtorTraitsHelper<T, SimdAbi, array_size>());
   }
 };
 
diff --git a/libcxx/test/std/experimental/simd/simd.reference/reference_assignment.pass.cpp b/libcxx/test/std/experimental/simd/simd.reference/reference_assignment.pass.cpp
index cfef55a868ba..013adb368593 100644
--- a/libcxx/test/std/experimental/simd/simd.reference/reference_assignment.pass.cpp
+++ b/libcxx/test/std/experimental/simd/simd.reference/reference_assignment.pass.cpp
@@ -72,10 +72,10 @@ template <class T, std::size_t>
 struct CheckReferenceAssignment {
   template <class SimdAbi>
   void operator()() {
-    types::for_each(arithmetic_no_bool_types(), CheckSimdReferenceAssignmentHelper<T, SimdAbi>());
-    types::for_each(arithmetic_no_bool_types(), CheckMaskReferenceAssignmentHelper<T, SimdAbi>());
+    types::for_each(simd_test_types(), CheckSimdReferenceAssignmentHelper<T, SimdAbi>());
+    types::for_each(simd_test_types(), CheckMaskReferenceAssignmentHelper<T, SimdAbi>());
 
-    types::for_each(arithmetic_no_bool_types(), CheckReferenceAssignmentTraitsHelper<T, SimdAbi>());
+    types::for_each(simd_test_types(), CheckReferenceAssignmentTraitsHelper<T, SimdAbi>());
   }
 };
 
diff --git a/libcxx/test/std/experimental/simd/test_utils.h b/libcxx/test/std/experimental/simd/test_utils.h
index b3679b51e50b..9b4416085751 100644
--- a/libcxx/test/std/experimental/simd/test_utils.h
+++ b/libcxx/test/std/experimental/simd/test_utils.h
@@ -48,6 +48,18 @@ using arithmetic_no_bool_types = types::concatenate_t<types::integer_types, type
 using arithmetic_no_bool_types = types::concatenate_t<types::integer_types, types::floating_point_types>;
 #endif
 
+// For interfaces with vectorizable type template parameters, we only use some common or boundary types
+// as template parameters for testing to ensure that the compilation time of a single test does not exceed.
+using simd_test_types =
+    types::type_list<char,
+                     unsigned,
+                     int,
+#ifndef TEST_HAS_NO_INT128
+                     __int128_t,
+#endif
+                     float,
+                     double>;
+
 template <template <class T, std::size_t N> class Func>
 void test_all_simd_abi() {
   types::for_each(arithmetic_no_bool_types(), TestAllSimdAbiFunctor<Func>());
-- 
GitLab


From 6ff61914a1ef2c05702ba73cf31ffdeed59c1740 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Tue, 26 Mar 2024 00:04:19 -0700
Subject: [PATCH 246/404] [Driver] Reject -fptrauth-intrinsics on non-AArch64
 targets

And add a driver test missing from the original patch #65996.
---
 clang/include/clang/Driver/Options.td | 8 ++------
 clang/lib/Driver/ToolChains/Clang.cpp | 7 +++----
 clang/test/Driver/aarch64-ptrauth.c   | 5 +++++
 3 files changed, 10 insertions(+), 10 deletions(-)
 create mode 100644 clang/test/Driver/aarch64-ptrauth.c

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index b0d90c776b58..29066ea14280 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -4105,12 +4105,8 @@ defm strict_return : BoolFOption<"strict-return",
             " of a non-void function as unreachable">,
   PosFlag<SetTrue>>;
 
-let Group = f_Group in {
-  let Visibility = [ClangOption,CC1Option] in {
-    def fptrauth_intrinsics : Flag<["-"], "fptrauth-intrinsics">,
-      HelpText<"Enable pointer authentication intrinsics">;
-  }
-  def fno_ptrauth_intrinsics : Flag<["-"], "fno-ptrauth-intrinsics">;
+let Flags = [TargetSpecific] in {
+defm ptrauth_intrinsics : OptInCC1FFlag<"ptrauth-intrinsics", "Enable pointer authentication intrinsics">;
 }
 
 def fenable_matrix : Flag<["-"], "fenable-matrix">, Group<f_Group>,
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 86a287db72a4..3bcacff7724c 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1776,6 +1776,9 @@ void Clang::AddAArch64TargetArgs(const ArgList &Args,
   }
 
   AddUnalignedAccessWarning(CmdArgs);
+
+  Args.addOptInFlag(CmdArgs, options::OPT_fptrauth_intrinsics,
+                    options::OPT_fno_ptrauth_intrinsics);
 }
 
 void Clang::AddLoongArchTargetArgs(const ArgList &Args,
@@ -7258,10 +7261,6 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // -fno-common is the default, set -fcommon only when that flag is set.
   Args.addOptInFlag(CmdArgs, options::OPT_fcommon, options::OPT_fno_common);
 
-  if (Args.hasFlag(options::OPT_fptrauth_intrinsics,
-                   options::OPT_fno_ptrauth_intrinsics, false))
-    CmdArgs.push_back("-fptrauth-intrinsics");
-
   // -fsigned-bitfields is default, and clang doesn't yet support
   // -funsigned-bitfields.
   if (!Args.hasFlag(options::OPT_fsigned_bitfields,
diff --git a/clang/test/Driver/aarch64-ptrauth.c b/clang/test/Driver/aarch64-ptrauth.c
new file mode 100644
index 000000000000..1a69b2c6edfb
--- /dev/null
+++ b/clang/test/Driver/aarch64-ptrauth.c
@@ -0,0 +1,5 @@
+// RUN: %clang -### -c --target=aarch64 -fno-ptrauth-intrinsics -fptrauth-intrinsics %s 2>&1 | FileCheck %s --check-prefix=INTRIN
+// INTRIN: "-cc1"{{.*}} "-fptrauth-intrinsics"
+
+// RUN: not %clang -### -c --target=x86_64 -fptrauth-intrinsics %s 2>&1 | FileCheck %s --check-prefix=ERR
+// ERR: error: unsupported option '-fptrauth-intrinsics' for target '{{.*}}'
-- 
GitLab


From 6a6f9bf38e65ec45b32da4b578e2830341a9b364 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Tue, 26 Mar 2024 08:35:03 +0100
Subject: [PATCH 247/404] [clang] Fix assertion failure when printing atomic
 apvalues (#85259)

When printing an `_Atomic(some struct type)`, we would later run into an
assertion because we do a `Ty->castAs<RecordType>()`, which doesn't work
with an `AtomicType`.
---
 clang/lib/AST/APValue.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/clang/lib/AST/APValue.cpp b/clang/lib/AST/APValue.cpp
index 4eae308ef5b3..d8042321319a 100644
--- a/clang/lib/AST/APValue.cpp
+++ b/clang/lib/AST/APValue.cpp
@@ -704,6 +704,9 @@ void APValue::printPretty(raw_ostream &Out, const PrintingPolicy &Policy,
     return;
   }
 
+  if (const auto *AT = Ty->getAs<AtomicType>())
+    Ty = AT->getValueType();
+
   switch (getKind()) {
   case APValue::None:
     Out << "<out of lifetime>";
-- 
GitLab


From 14c30189fb8782535ac9a5a52160e3fc62e7e78c Mon Sep 17 00:00:00 2001
From: Bevin Hansson <59652494+bevin-hansson@users.noreply.github.com>
Date: Tue, 26 Mar 2024 10:08:22 +0100
Subject: [PATCH 248/404] [ExpandLargeFpConvert] Fix incorrect values in
 fp-to-int conversion. (#86514)

The IR for a double-to-i129 conversion looks like this in one of the
blocks in compiler-rt:

  %cmp5.i = icmp ult i16 %3, -129, !dbg !24

But in ExpandLargeFpConvert, it looks like:

  %13 = icmp ult i129 %12, 4294967167, !dbg !19

ExpandLargeFpConvert is wrong; the value should have been
signed before negating, but instead we get a very large
unsigned value. Another value in the same pass also has this
issue.
---
 llvm/lib/CodeGen/ExpandLargeFpConvert.cpp     |  11 +-
 llvm/test/CodeGen/AMDGPU/fptoi.i128.ll        | 400 +++++++++---------
 .../X86/expand-large-fp-convert-fptosi129.ll  |  16 +-
 .../X86/expand-large-fp-convert-fptoui129.ll  |  16 +-
 4 files changed, 218 insertions(+), 225 deletions(-)

diff --git a/llvm/lib/CodeGen/ExpandLargeFpConvert.cpp b/llvm/lib/CodeGen/ExpandLargeFpConvert.cpp
index 308f13c19f75..4ec966e56d6e 100644
--- a/llvm/lib/CodeGen/ExpandLargeFpConvert.cpp
+++ b/llvm/lib/CodeGen/ExpandLargeFpConvert.cpp
@@ -175,9 +175,10 @@ static void expandFPToI(Instruction *FPToI) {
   // if.end:
   Builder.SetInsertPoint(IfEnd);
   Value *Add1 = Builder.CreateAdd(
-      And2, ConstantInt::getSigned(IntTy, -int64_t(ExponentBias + BitWidth)));
-  Value *Cmp3 =
-      Builder.CreateICmpULT(Add1, ConstantInt::getSigned(IntTy, -BitWidth));
+      And2, ConstantInt::getSigned(
+                IntTy, -static_cast<int64_t>(ExponentBias + BitWidth)));
+  Value *Cmp3 = Builder.CreateICmpULT(
+      Add1, ConstantInt::getSigned(IntTy, -static_cast<int64_t>(BitWidth)));
   Builder.CreateCondBr(Cmp3, IfThen5, IfEnd9);
 
   // if.then5:
@@ -203,8 +204,8 @@ static void expandFPToI(Instruction *FPToI) {
   // if.else:
   Builder.SetInsertPoint(IfElse);
   Value *Sub15 = Builder.CreateAdd(
-      And2,
-      ConstantInt::getSigned(IntTy, -(ExponentBias + FPMantissaWidth)));
+      And2, ConstantInt::getSigned(
+                IntTy, -static_cast<int64_t>(ExponentBias + FPMantissaWidth)));
   Value *Shl = Builder.CreateShl(Or, Sub15);
   Value *Mul16 = Builder.CreateMul(Shl, Sign);
   Builder.CreateBr(End);
diff --git a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
index a69418d43641..66bf0d5abb73 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
@@ -22,35 +22,31 @@ define i128 @fptosi_f64_to_i128(double %x) {
 ; SDAG-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffffb81, v6
 ; SDAG-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v7, vcc
 ; SDAG-NEXT:    v_addc_co_u32_e32 v2, vcc, -1, v7, vcc
+; SDAG-NEXT:    s_movk_i32 s4, 0xff7f
 ; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, -1, v7, vcc
-; SDAG-NEXT:    s_mov_b64 s[6:7], 0xffffff7f
-; SDAG-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
-; SDAG-NEXT:    v_cmp_lt_i64_e64 s[4:5], -1, v[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
-; SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
-; SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
-; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; SDAG-NEXT:    v_and_b32_e32 v0, 1, v0
-; SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; SDAG-NEXT:    s_mov_b32 s5, -1
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[0:1]
+; SDAG-NEXT:    v_cmp_eq_u64_e64 s[6:7], -1, v[2:3]
+; SDAG-NEXT:    v_cmp_lt_i64_e32 vcc, -1, v[4:5]
+; SDAG-NEXT:    s_and_b64 s[4:5], s[6:7], s[4:5]
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; SDAG-NEXT:    s_xor_b64 s[10:11], exec, s[6:7]
 ; SDAG-NEXT:    s_cbranch_execz .LBB0_7
 ; SDAG-NEXT:  ; %bb.2: ; %fp-to-i-if-end9
-; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; SDAG-NEXT:    v_add_co_u32_e32 v9, vcc, -1, v0
-; SDAG-NEXT:    v_addc_co_u32_e64 v10, s[6:7], 0, -1, vcc
-; SDAG-NEXT:    s_mov_b64 s[6:7], 0x432
+; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; SDAG-NEXT:    v_add_co_u32_e64 v9, s[4:5], -1, v0
+; SDAG-NEXT:    v_addc_co_u32_e64 v10, s[4:5], 0, -1, s[4:5]
+; SDAG-NEXT:    s_mov_b64 s[4:5], 0x432
 ; SDAG-NEXT:    v_and_b32_e32 v0, 0xfffff, v5
-; SDAG-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[6:7]
-; SDAG-NEXT:    v_cndmask_b32_e64 v8, -1, 0, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v11, -1, 1, s[4:5]
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[6:7]
+; SDAG-NEXT:    v_cndmask_b32_e64 v8, -1, 0, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v11, -1, 1, vcc
 ; SDAG-NEXT:    v_or_b32_e32 v5, 0x100000, v0
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; SDAG-NEXT:    s_xor_b64 s[12:13], exec, s[6:7]
 ; SDAG-NEXT:    s_cbranch_execz .LBB0_4
 ; SDAG-NEXT:  ; %bb.3: ; %fp-to-i-if-else
@@ -59,37 +55,37 @@ define i128 @fptosi_f64_to_i128(double %x) {
 ; SDAG-NEXT:    v_add_u32_e32 v7, 0xfffffbcd, v6
 ; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v0, v[4:5]
 ; SDAG-NEXT:    v_lshlrev_b64 v[2:3], v2, v[4:5]
-; SDAG-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v7
-; SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v7
+; SDAG-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[4:5]
 ; SDAG-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, v7
 ; SDAG-NEXT:    v_cndmask_b32_e64 v6, 0, v1, s[6:7]
-; SDAG-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v0, s[4:5]
 ; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v7, v[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[6:7]
-; SDAG-NEXT:    v_cndmask_b32_e32 v12, 0, v0, vcc
-; SDAG-NEXT:    v_cndmask_b32_e32 v7, 0, v1, vcc
-; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[6:7], v12, v11, 0
+; SDAG-NEXT:    v_cndmask_b32_e64 v12, 0, v0, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v7, 0, v1, s[4:5]
+; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v12, v11, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v3, 0
 ; SDAG-NEXT:    v_mul_lo_u32 v13, v8, v2
-; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[6:7], v11, v2, 0
+; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v11, v2, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v2, v1
 ; SDAG-NEXT:    v_mul_lo_u32 v6, v11, v6
-; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[6:7], v7, v11, v[2:3]
+; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v7, v11, v[2:3]
 ; SDAG-NEXT:    v_mul_lo_u32 v10, v10, v12
 ; SDAG-NEXT:    v_add3_u32 v5, v5, v6, v13
 ; SDAG-NEXT:    v_mov_b32_e32 v6, v2
 ; SDAG-NEXT:    v_mov_b32_e32 v2, v3
-; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[6:7], v12, v8, v[1:2]
-; SDAG-NEXT:    v_mad_u64_u32 v[3:4], s[6:7], v9, v12, v[4:5]
-; SDAG-NEXT:    v_add_co_u32_e32 v5, vcc, v6, v2
-; SDAG-NEXT:    v_addc_co_u32_e64 v6, s[6:7], 0, 0, vcc
+; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v12, v8, v[1:2]
+; SDAG-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v9, v12, v[4:5]
+; SDAG-NEXT:    v_add_co_u32_e64 v5, s[4:5], v6, v2
+; SDAG-NEXT:    v_addc_co_u32_e64 v6, s[4:5], 0, 0, s[4:5]
 ; SDAG-NEXT:    v_mul_lo_u32 v9, v9, v7
-; SDAG-NEXT:    v_mad_u64_u32 v[5:6], s[6:7], v7, v8, v[5:6]
+; SDAG-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v7, v8, v[5:6]
 ; SDAG-NEXT:    ; implicit-def: $vgpr11
 ; SDAG-NEXT:    ; implicit-def: $vgpr8
 ; SDAG-NEXT:    v_add3_u32 v4, v10, v4, v9
-; SDAG-NEXT:    v_add_co_u32_e32 v2, vcc, v5, v3
-; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, v6, v4, vcc
+; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v5, v3
+; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[4:5], v6, v4, s[4:5]
 ; SDAG-NEXT:    ; implicit-def: $vgpr6_vgpr7
 ; SDAG-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; SDAG-NEXT:    ; implicit-def: $vgpr9
@@ -100,37 +96,37 @@ define i128 @fptosi_f64_to_i128(double %x) {
 ; SDAG-NEXT:  ; %bb.5: ; %fp-to-i-if-then12
 ; SDAG-NEXT:    v_sub_u32_e32 v2, 0x433, v6
 ; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v2, v[4:5]
-; SDAG-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v2
+; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v2
 ; SDAG-NEXT:    v_cmp_eq_u32_e64 s[6:7], 0, v2
-; SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, v0, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, v1, s[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v6, v0, v4, s[6:7]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v5, v1, v5, s[6:7]
-; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[6:7], v6, v11, 0
+; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v6, v11, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; SDAG-NEXT:    v_mad_u64_u32 v[3:4], s[6:7], v5, v11, v[1:2]
+; SDAG-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v5, v11, v[1:2]
 ; SDAG-NEXT:    v_mov_b32_e32 v7, v4
 ; SDAG-NEXT:    v_mov_b32_e32 v4, v2
-; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[6:7], v6, v8, v[3:4]
-; SDAG-NEXT:    v_add_co_u32_e32 v2, vcc, v7, v2
-; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[6:7], 0, 0, vcc
-; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[6:7], v5, v8, v[2:3]
-; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[6:7], v9, v6, v[2:3]
-; SDAG-NEXT:    v_mad_u64_u32 v[3:4], s[6:7], v10, v6, v[3:4]
+; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v6, v8, v[3:4]
+; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v7, v2
+; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, s[4:5]
+; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v5, v8, v[2:3]
+; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v9, v6, v[2:3]
+; SDAG-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v10, v6, v[3:4]
 ; SDAG-NEXT:    v_mad_i32_i24 v3, v9, v5, v3
 ; SDAG-NEXT:  .LBB0_6: ; %Flow1
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[12:13]
 ; SDAG-NEXT:  .LBB0_7: ; %Flow2
-; SDAG-NEXT:    s_andn2_saveexec_b64 s[6:7], s[10:11]
+; SDAG-NEXT:    s_andn2_saveexec_b64 s[4:5], s[10:11]
 ; SDAG-NEXT:  ; %bb.8: ; %fp-to-i-if-then5
 ; SDAG-NEXT:    v_bfrev_b32_e32 v0, 1
 ; SDAG-NEXT:    v_bfrev_b32_e32 v1, -2
-; SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, -1, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v3, v0, v1, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
 ; SDAG-NEXT:    v_mov_b32_e32 v0, v2
 ; SDAG-NEXT:    v_mov_b32_e32 v1, v2
 ; SDAG-NEXT:  ; %bb.9: ; %Flow3
-; SDAG-NEXT:    s_or_b64 exec, exec, s[6:7]
+; SDAG-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SDAG-NEXT:  .LBB0_10: ; %fp-to-i-cleanup
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -158,14 +154,16 @@ define i128 @fptosi_f64_to_i128(double %x) {
 ; GISEL-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffffb81, v6
 ; GISEL-NEXT:    v_mov_b32_e32 v2, 0xffffff80
 ; GISEL-NEXT:    v_addc_co_u32_e64 v1, s[6:7], 0, -1, vcc
-; GISEL-NEXT:    v_mov_b32_e32 v3, 0
+; GISEL-NEXT:    v_mov_b32_e32 v3, -1
 ; GISEL-NEXT:    v_addc_co_u32_e64 v8, s[6:7], 0, -1, s[6:7]
 ; GISEL-NEXT:    v_cmp_ge_u64_e32 vcc, v[0:1], v[2:3]
 ; GISEL-NEXT:    v_addc_co_u32_e64 v9, s[6:7], 0, -1, s[6:7]
 ; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GISEL-NEXT:    v_cmp_le_u64_e32 vcc, -1, v[8:9]
 ; GISEL-NEXT:    v_cmp_lt_i64_e64 s[4:5], -1, v[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, 1, v0, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT:    v_cmp_eq_u64_e32 vcc, -1, v[8:9]
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
 ; GISEL-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GISEL-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
@@ -396,35 +394,31 @@ define i128 @fptoui_f64_to_i128(double %x) {
 ; SDAG-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffffb81, v6
 ; SDAG-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v7, vcc
 ; SDAG-NEXT:    v_addc_co_u32_e32 v2, vcc, -1, v7, vcc
+; SDAG-NEXT:    s_movk_i32 s4, 0xff7f
 ; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, -1, v7, vcc
-; SDAG-NEXT:    s_mov_b64 s[6:7], 0xffffff7f
-; SDAG-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
-; SDAG-NEXT:    v_cmp_lt_i64_e64 s[4:5], -1, v[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
-; SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
-; SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
-; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; SDAG-NEXT:    v_and_b32_e32 v0, 1, v0
-; SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; SDAG-NEXT:    s_mov_b32 s5, -1
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[0:1]
+; SDAG-NEXT:    v_cmp_eq_u64_e64 s[6:7], -1, v[2:3]
+; SDAG-NEXT:    v_cmp_lt_i64_e32 vcc, -1, v[4:5]
+; SDAG-NEXT:    s_and_b64 s[4:5], s[6:7], s[4:5]
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; SDAG-NEXT:    s_xor_b64 s[10:11], exec, s[6:7]
 ; SDAG-NEXT:    s_cbranch_execz .LBB1_7
 ; SDAG-NEXT:  ; %bb.2: ; %fp-to-i-if-end9
-; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; SDAG-NEXT:    v_add_co_u32_e32 v9, vcc, -1, v0
-; SDAG-NEXT:    v_addc_co_u32_e64 v10, s[6:7], 0, -1, vcc
-; SDAG-NEXT:    s_mov_b64 s[6:7], 0x432
+; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; SDAG-NEXT:    v_add_co_u32_e64 v9, s[4:5], -1, v0
+; SDAG-NEXT:    v_addc_co_u32_e64 v10, s[4:5], 0, -1, s[4:5]
+; SDAG-NEXT:    s_mov_b64 s[4:5], 0x432
 ; SDAG-NEXT:    v_and_b32_e32 v0, 0xfffff, v5
-; SDAG-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[6:7]
-; SDAG-NEXT:    v_cndmask_b32_e64 v8, -1, 0, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v11, -1, 1, s[4:5]
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[6:7]
+; SDAG-NEXT:    v_cndmask_b32_e64 v8, -1, 0, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v11, -1, 1, vcc
 ; SDAG-NEXT:    v_or_b32_e32 v5, 0x100000, v0
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; SDAG-NEXT:    s_xor_b64 s[12:13], exec, s[6:7]
 ; SDAG-NEXT:    s_cbranch_execz .LBB1_4
 ; SDAG-NEXT:  ; %bb.3: ; %fp-to-i-if-else
@@ -433,37 +427,37 @@ define i128 @fptoui_f64_to_i128(double %x) {
 ; SDAG-NEXT:    v_add_u32_e32 v7, 0xfffffbcd, v6
 ; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v0, v[4:5]
 ; SDAG-NEXT:    v_lshlrev_b64 v[2:3], v2, v[4:5]
-; SDAG-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v7
-; SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v7
+; SDAG-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[4:5]
 ; SDAG-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, v7
 ; SDAG-NEXT:    v_cndmask_b32_e64 v6, 0, v1, s[6:7]
-; SDAG-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v0, s[4:5]
 ; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v7, v[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[6:7]
-; SDAG-NEXT:    v_cndmask_b32_e32 v12, 0, v0, vcc
-; SDAG-NEXT:    v_cndmask_b32_e32 v7, 0, v1, vcc
-; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[6:7], v12, v11, 0
+; SDAG-NEXT:    v_cndmask_b32_e64 v12, 0, v0, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v7, 0, v1, s[4:5]
+; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v12, v11, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v3, 0
 ; SDAG-NEXT:    v_mul_lo_u32 v13, v8, v2
-; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[6:7], v11, v2, 0
+; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v11, v2, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v2, v1
 ; SDAG-NEXT:    v_mul_lo_u32 v6, v11, v6
-; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[6:7], v7, v11, v[2:3]
+; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v7, v11, v[2:3]
 ; SDAG-NEXT:    v_mul_lo_u32 v10, v10, v12
 ; SDAG-NEXT:    v_add3_u32 v5, v5, v6, v13
 ; SDAG-NEXT:    v_mov_b32_e32 v6, v2
 ; SDAG-NEXT:    v_mov_b32_e32 v2, v3
-; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[6:7], v12, v8, v[1:2]
-; SDAG-NEXT:    v_mad_u64_u32 v[3:4], s[6:7], v9, v12, v[4:5]
-; SDAG-NEXT:    v_add_co_u32_e32 v5, vcc, v6, v2
-; SDAG-NEXT:    v_addc_co_u32_e64 v6, s[6:7], 0, 0, vcc
+; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v12, v8, v[1:2]
+; SDAG-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v9, v12, v[4:5]
+; SDAG-NEXT:    v_add_co_u32_e64 v5, s[4:5], v6, v2
+; SDAG-NEXT:    v_addc_co_u32_e64 v6, s[4:5], 0, 0, s[4:5]
 ; SDAG-NEXT:    v_mul_lo_u32 v9, v9, v7
-; SDAG-NEXT:    v_mad_u64_u32 v[5:6], s[6:7], v7, v8, v[5:6]
+; SDAG-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v7, v8, v[5:6]
 ; SDAG-NEXT:    ; implicit-def: $vgpr11
 ; SDAG-NEXT:    ; implicit-def: $vgpr8
 ; SDAG-NEXT:    v_add3_u32 v4, v10, v4, v9
-; SDAG-NEXT:    v_add_co_u32_e32 v2, vcc, v5, v3
-; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, v6, v4, vcc
+; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v5, v3
+; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[4:5], v6, v4, s[4:5]
 ; SDAG-NEXT:    ; implicit-def: $vgpr6_vgpr7
 ; SDAG-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; SDAG-NEXT:    ; implicit-def: $vgpr9
@@ -474,37 +468,37 @@ define i128 @fptoui_f64_to_i128(double %x) {
 ; SDAG-NEXT:  ; %bb.5: ; %fp-to-i-if-then12
 ; SDAG-NEXT:    v_sub_u32_e32 v2, 0x433, v6
 ; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v2, v[4:5]
-; SDAG-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v2
+; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v2
 ; SDAG-NEXT:    v_cmp_eq_u32_e64 s[6:7], 0, v2
-; SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; SDAG-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, v0, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, v1, s[4:5]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v6, v0, v4, s[6:7]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v5, v1, v5, s[6:7]
-; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[6:7], v6, v11, 0
+; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v6, v11, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; SDAG-NEXT:    v_mad_u64_u32 v[3:4], s[6:7], v5, v11, v[1:2]
+; SDAG-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v5, v11, v[1:2]
 ; SDAG-NEXT:    v_mov_b32_e32 v7, v4
 ; SDAG-NEXT:    v_mov_b32_e32 v4, v2
-; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[6:7], v6, v8, v[3:4]
-; SDAG-NEXT:    v_add_co_u32_e32 v2, vcc, v7, v2
-; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[6:7], 0, 0, vcc
-; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[6:7], v5, v8, v[2:3]
-; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[6:7], v9, v6, v[2:3]
-; SDAG-NEXT:    v_mad_u64_u32 v[3:4], s[6:7], v10, v6, v[3:4]
+; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v6, v8, v[3:4]
+; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v7, v2
+; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[4:5], 0, 0, s[4:5]
+; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v5, v8, v[2:3]
+; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v9, v6, v[2:3]
+; SDAG-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v10, v6, v[3:4]
 ; SDAG-NEXT:    v_mad_i32_i24 v3, v9, v5, v3
 ; SDAG-NEXT:  .LBB1_6: ; %Flow1
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[12:13]
 ; SDAG-NEXT:  .LBB1_7: ; %Flow2
-; SDAG-NEXT:    s_andn2_saveexec_b64 s[6:7], s[10:11]
+; SDAG-NEXT:    s_andn2_saveexec_b64 s[4:5], s[10:11]
 ; SDAG-NEXT:  ; %bb.8: ; %fp-to-i-if-then5
 ; SDAG-NEXT:    v_bfrev_b32_e32 v0, 1
 ; SDAG-NEXT:    v_bfrev_b32_e32 v1, -2
-; SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, -1, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v3, v0, v1, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
 ; SDAG-NEXT:    v_mov_b32_e32 v0, v2
 ; SDAG-NEXT:    v_mov_b32_e32 v1, v2
 ; SDAG-NEXT:  ; %bb.9: ; %Flow3
-; SDAG-NEXT:    s_or_b64 exec, exec, s[6:7]
+; SDAG-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SDAG-NEXT:  .LBB1_10: ; %fp-to-i-cleanup
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -532,14 +526,16 @@ define i128 @fptoui_f64_to_i128(double %x) {
 ; GISEL-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffffb81, v6
 ; GISEL-NEXT:    v_mov_b32_e32 v2, 0xffffff80
 ; GISEL-NEXT:    v_addc_co_u32_e64 v1, s[6:7], 0, -1, vcc
-; GISEL-NEXT:    v_mov_b32_e32 v3, 0
+; GISEL-NEXT:    v_mov_b32_e32 v3, -1
 ; GISEL-NEXT:    v_addc_co_u32_e64 v8, s[6:7], 0, -1, s[6:7]
 ; GISEL-NEXT:    v_cmp_ge_u64_e32 vcc, v[0:1], v[2:3]
 ; GISEL-NEXT:    v_addc_co_u32_e64 v9, s[6:7], 0, -1, s[6:7]
 ; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GISEL-NEXT:    v_cmp_le_u64_e32 vcc, -1, v[8:9]
 ; GISEL-NEXT:    v_cmp_lt_i64_e64 s[4:5], -1, v[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, 1, v0, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT:    v_cmp_eq_u64_e32 vcc, -1, v[8:9]
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
 ; GISEL-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GISEL-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
@@ -769,36 +765,32 @@ define i128 @fptosi_f32_to_i128(float %x) {
 ; SDAG-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffff01, v5
 ; SDAG-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v6, vcc
 ; SDAG-NEXT:    v_addc_co_u32_e32 v2, vcc, -1, v6, vcc
+; SDAG-NEXT:    s_movk_i32 s4, 0xff7f
 ; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, -1, v6, vcc
-; SDAG-NEXT:    s_mov_b64 s[6:7], 0xffffff7f
-; SDAG-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
-; SDAG-NEXT:    v_cmp_lt_i32_e64 s[4:5], -1, v4
-; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
-; SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
-; SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
-; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; SDAG-NEXT:    v_and_b32_e32 v0, 1, v0
-; SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; SDAG-NEXT:    s_mov_b32 s5, -1
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[0:1]
+; SDAG-NEXT:    v_cmp_eq_u64_e64 s[6:7], -1, v[2:3]
+; SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v4
+; SDAG-NEXT:    s_and_b64 s[4:5], s[6:7], s[4:5]
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; SDAG-NEXT:    s_xor_b64 s[10:11], exec, s[6:7]
 ; SDAG-NEXT:    s_cbranch_execz .LBB2_7
 ; SDAG-NEXT:  ; %bb.2: ; %fp-to-i-if-end9
-; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; SDAG-NEXT:    v_add_co_u32_e32 v9, vcc, -1, v0
-; SDAG-NEXT:    v_addc_co_u32_e64 v11, s[6:7], 0, -1, vcc
-; SDAG-NEXT:    s_mov_b64 s[6:7], 0x95
+; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; SDAG-NEXT:    v_add_co_u32_e64 v9, s[4:5], -1, v0
+; SDAG-NEXT:    v_addc_co_u32_e64 v11, s[4:5], 0, -1, s[4:5]
+; SDAG-NEXT:    s_mov_b64 s[4:5], 0x95
 ; SDAG-NEXT:    v_and_b32_e32 v0, 0x7fffff, v4
-; SDAG-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[5:6]
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[5:6]
 ; SDAG-NEXT:    v_mov_b32_e32 v7, 0
-; SDAG-NEXT:    v_cndmask_b32_e64 v8, -1, 0, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v10, -1, 1, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v8, -1, 0, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v10, -1, 1, vcc
 ; SDAG-NEXT:    v_or_b32_e32 v6, 0x800000, v0
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; SDAG-NEXT:    s_xor_b64 s[12:13], exec, s[6:7]
 ; SDAG-NEXT:    s_cbranch_execz .LBB2_4
 ; SDAG-NEXT:  ; %bb.3: ; %fp-to-i-if-else
@@ -807,37 +799,37 @@ define i128 @fptosi_f32_to_i128(float %x) {
 ; SDAG-NEXT:    v_add_u32_e32 v4, 0xffffff6a, v5
 ; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v0, v[6:7]
 ; SDAG-NEXT:    v_lshlrev_b64 v[2:3], v2, v[6:7]
-; SDAG-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v4
-; SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v4
+; SDAG-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[4:5]
 ; SDAG-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, v4
 ; SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, v1, s[6:7]
-; SDAG-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v0, s[4:5]
 ; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v4, v[6:7]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[6:7]
-; SDAG-NEXT:    v_cndmask_b32_e32 v13, 0, v0, vcc
-; SDAG-NEXT:    v_cndmask_b32_e32 v12, 0, v1, vcc
-; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[6:7], v13, v10, 0
+; SDAG-NEXT:    v_cndmask_b32_e64 v13, 0, v0, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v12, 0, v1, s[4:5]
+; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v13, v10, 0
 ; SDAG-NEXT:    v_mul_lo_u32 v14, v8, v2
 ; SDAG-NEXT:    v_mul_lo_u32 v15, v10, v3
 ; SDAG-NEXT:    v_mov_b32_e32 v6, v1
-; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[6:7], v12, v10, v[6:7]
-; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[6:7], v10, v2, 0
+; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v12, v10, v[6:7]
+; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v10, v2, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v6, v5
 ; SDAG-NEXT:    v_mov_b32_e32 v5, v7
-; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[6:7], v13, v8, v[4:5]
+; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v13, v8, v[4:5]
 ; SDAG-NEXT:    v_add3_u32 v3, v3, v15, v14
-; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[6:7], v9, v13, v[2:3]
-; SDAG-NEXT:    v_add_co_u32_e32 v5, vcc, v6, v5
-; SDAG-NEXT:    v_addc_co_u32_e64 v6, s[6:7], 0, 0, vcc
+; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v9, v13, v[2:3]
+; SDAG-NEXT:    v_add_co_u32_e64 v5, s[4:5], v6, v5
+; SDAG-NEXT:    v_addc_co_u32_e64 v6, s[4:5], 0, 0, s[4:5]
 ; SDAG-NEXT:    v_mul_lo_u32 v3, v9, v12
 ; SDAG-NEXT:    v_mul_lo_u32 v7, v11, v13
-; SDAG-NEXT:    v_mad_u64_u32 v[5:6], s[6:7], v12, v8, v[5:6]
+; SDAG-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v12, v8, v[5:6]
 ; SDAG-NEXT:    ; implicit-def: $vgpr10
 ; SDAG-NEXT:    ; implicit-def: $vgpr8
 ; SDAG-NEXT:    ; implicit-def: $vgpr9
 ; SDAG-NEXT:    v_add3_u32 v3, v7, v2, v3
-; SDAG-NEXT:    v_add_co_u32_e32 v2, vcc, v5, v1
-; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, v6, v3, vcc
+; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v5, v1
+; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[4:5], v6, v3, s[4:5]
 ; SDAG-NEXT:    ; implicit-def: $vgpr5_vgpr6
 ; SDAG-NEXT:    v_mov_b32_e32 v1, v4
 ; SDAG-NEXT:    ; implicit-def: $vgpr6_vgpr7
@@ -847,29 +839,29 @@ define i128 @fptosi_f32_to_i128(float %x) {
 ; SDAG-NEXT:  ; %bb.5: ; %fp-to-i-if-then12
 ; SDAG-NEXT:    v_sub_u32_e32 v2, 0x96, v5
 ; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v2, v[6:7]
-; SDAG-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v2
-; SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; SDAG-NEXT:    v_cndmask_b32_e32 v3, v0, v6, vcc
-; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[12:13], v3, v10, 0
+; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v2
+; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, v0, s[4:5]
+; SDAG-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v2
+; SDAG-NEXT:    v_cndmask_b32_e64 v3, v0, v6, s[4:5]
+; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v3, v10, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[12:13], v3, v8, v[1:2]
+; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v3, v8, v[1:2]
 ; SDAG-NEXT:    v_mov_b32_e32 v1, v5
-; SDAG-NEXT:    v_mad_i64_i32 v[2:3], s[12:13], v9, v3, v[1:2]
+; SDAG-NEXT:    v_mad_i64_i32 v[2:3], s[4:5], v9, v3, v[1:2]
 ; SDAG-NEXT:    v_mov_b32_e32 v1, v4
 ; SDAG-NEXT:  .LBB2_6: ; %Flow1
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; SDAG-NEXT:  .LBB2_7: ; %Flow2
-; SDAG-NEXT:    s_andn2_saveexec_b64 s[6:7], s[10:11]
+; SDAG-NEXT:    s_andn2_saveexec_b64 s[4:5], s[10:11]
 ; SDAG-NEXT:  ; %bb.8: ; %fp-to-i-if-then5
 ; SDAG-NEXT:    v_bfrev_b32_e32 v0, 1
 ; SDAG-NEXT:    v_bfrev_b32_e32 v1, -2
-; SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, -1, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v3, v0, v1, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
 ; SDAG-NEXT:    v_mov_b32_e32 v0, v2
 ; SDAG-NEXT:    v_mov_b32_e32 v1, v2
 ; SDAG-NEXT:  ; %bb.9: ; %Flow3
-; SDAG-NEXT:    s_or_b64 exec, exec, s[6:7]
+; SDAG-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SDAG-NEXT:  .LBB2_10: ; %fp-to-i-cleanup
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -897,14 +889,16 @@ define i128 @fptosi_f32_to_i128(float %x) {
 ; GISEL-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffff01, v6
 ; GISEL-NEXT:    v_mov_b32_e32 v2, 0xffffff80
 ; GISEL-NEXT:    v_addc_co_u32_e64 v1, s[6:7], 0, -1, vcc
-; GISEL-NEXT:    v_mov_b32_e32 v3, 0
+; GISEL-NEXT:    v_mov_b32_e32 v3, -1
 ; GISEL-NEXT:    v_addc_co_u32_e64 v8, s[6:7], 0, -1, s[6:7]
 ; GISEL-NEXT:    v_cmp_ge_u64_e32 vcc, v[0:1], v[2:3]
 ; GISEL-NEXT:    v_addc_co_u32_e64 v9, s[6:7], 0, -1, s[6:7]
 ; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GISEL-NEXT:    v_cmp_le_u64_e32 vcc, -1, v[8:9]
 ; GISEL-NEXT:    v_cmp_lt_i32_e64 s[4:5], -1, v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, 1, v0, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT:    v_cmp_eq_u64_e32 vcc, -1, v[8:9]
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
 ; GISEL-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GISEL-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
@@ -1129,36 +1123,32 @@ define i128 @fptoui_f32_to_i128(float %x) {
 ; SDAG-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffff01, v5
 ; SDAG-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v6, vcc
 ; SDAG-NEXT:    v_addc_co_u32_e32 v2, vcc, -1, v6, vcc
+; SDAG-NEXT:    s_movk_i32 s4, 0xff7f
 ; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, -1, v6, vcc
-; SDAG-NEXT:    s_mov_b64 s[6:7], 0xffffff7f
-; SDAG-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
-; SDAG-NEXT:    v_cmp_lt_i32_e64 s[4:5], -1, v4
-; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; SDAG-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
-; SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
-; SDAG-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
-; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; SDAG-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; SDAG-NEXT:    v_and_b32_e32 v0, 1, v0
-; SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; SDAG-NEXT:    s_mov_b32 s5, -1
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[0:1]
+; SDAG-NEXT:    v_cmp_eq_u64_e64 s[6:7], -1, v[2:3]
+; SDAG-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v4
+; SDAG-NEXT:    s_and_b64 s[4:5], s[6:7], s[4:5]
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; SDAG-NEXT:    s_xor_b64 s[10:11], exec, s[6:7]
 ; SDAG-NEXT:    s_cbranch_execz .LBB3_7
 ; SDAG-NEXT:  ; %bb.2: ; %fp-to-i-if-end9
-; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; SDAG-NEXT:    v_add_co_u32_e32 v9, vcc, -1, v0
-; SDAG-NEXT:    v_addc_co_u32_e64 v11, s[6:7], 0, -1, vcc
-; SDAG-NEXT:    s_mov_b64 s[6:7], 0x95
+; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; SDAG-NEXT:    v_add_co_u32_e64 v9, s[4:5], -1, v0
+; SDAG-NEXT:    v_addc_co_u32_e64 v11, s[4:5], 0, -1, s[4:5]
+; SDAG-NEXT:    s_mov_b64 s[4:5], 0x95
 ; SDAG-NEXT:    v_and_b32_e32 v0, 0x7fffff, v4
-; SDAG-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[5:6]
+; SDAG-NEXT:    v_cmp_lt_u64_e64 s[4:5], s[4:5], v[5:6]
 ; SDAG-NEXT:    v_mov_b32_e32 v7, 0
-; SDAG-NEXT:    v_cndmask_b32_e64 v8, -1, 0, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v10, -1, 1, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v8, -1, 0, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v10, -1, 1, vcc
 ; SDAG-NEXT:    v_or_b32_e32 v6, 0x800000, v0
 ; SDAG-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; SDAG-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; SDAG-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; SDAG-NEXT:    s_xor_b64 s[12:13], exec, s[6:7]
 ; SDAG-NEXT:    s_cbranch_execz .LBB3_4
 ; SDAG-NEXT:  ; %bb.3: ; %fp-to-i-if-else
@@ -1167,37 +1157,37 @@ define i128 @fptoui_f32_to_i128(float %x) {
 ; SDAG-NEXT:    v_add_u32_e32 v4, 0xffffff6a, v5
 ; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v0, v[6:7]
 ; SDAG-NEXT:    v_lshlrev_b64 v[2:3], v2, v[6:7]
-; SDAG-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v4
-; SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v4
+; SDAG-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[4:5]
 ; SDAG-NEXT:    v_cmp_ne_u32_e64 s[6:7], 0, v4
 ; SDAG-NEXT:    v_cndmask_b32_e64 v3, 0, v1, s[6:7]
-; SDAG-NEXT:    v_cndmask_b32_e32 v2, v2, v0, vcc
+; SDAG-NEXT:    v_cndmask_b32_e64 v2, v2, v0, s[4:5]
 ; SDAG-NEXT:    v_lshlrev_b64 v[0:1], v4, v[6:7]
 ; SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, v2, s[6:7]
-; SDAG-NEXT:    v_cndmask_b32_e32 v13, 0, v0, vcc
-; SDAG-NEXT:    v_cndmask_b32_e32 v12, 0, v1, vcc
-; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[6:7], v13, v10, 0
+; SDAG-NEXT:    v_cndmask_b32_e64 v13, 0, v0, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v12, 0, v1, s[4:5]
+; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v13, v10, 0
 ; SDAG-NEXT:    v_mul_lo_u32 v14, v8, v2
 ; SDAG-NEXT:    v_mul_lo_u32 v15, v10, v3
 ; SDAG-NEXT:    v_mov_b32_e32 v6, v1
-; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[6:7], v12, v10, v[6:7]
-; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[6:7], v10, v2, 0
+; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v12, v10, v[6:7]
+; SDAG-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v10, v2, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v6, v5
 ; SDAG-NEXT:    v_mov_b32_e32 v5, v7
-; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[6:7], v13, v8, v[4:5]
+; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v13, v8, v[4:5]
 ; SDAG-NEXT:    v_add3_u32 v3, v3, v15, v14
-; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[6:7], v9, v13, v[2:3]
-; SDAG-NEXT:    v_add_co_u32_e32 v5, vcc, v6, v5
-; SDAG-NEXT:    v_addc_co_u32_e64 v6, s[6:7], 0, 0, vcc
+; SDAG-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v9, v13, v[2:3]
+; SDAG-NEXT:    v_add_co_u32_e64 v5, s[4:5], v6, v5
+; SDAG-NEXT:    v_addc_co_u32_e64 v6, s[4:5], 0, 0, s[4:5]
 ; SDAG-NEXT:    v_mul_lo_u32 v3, v9, v12
 ; SDAG-NEXT:    v_mul_lo_u32 v7, v11, v13
-; SDAG-NEXT:    v_mad_u64_u32 v[5:6], s[6:7], v12, v8, v[5:6]
+; SDAG-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v12, v8, v[5:6]
 ; SDAG-NEXT:    ; implicit-def: $vgpr10
 ; SDAG-NEXT:    ; implicit-def: $vgpr8
 ; SDAG-NEXT:    ; implicit-def: $vgpr9
 ; SDAG-NEXT:    v_add3_u32 v3, v7, v2, v3
-; SDAG-NEXT:    v_add_co_u32_e32 v2, vcc, v5, v1
-; SDAG-NEXT:    v_addc_co_u32_e32 v3, vcc, v6, v3, vcc
+; SDAG-NEXT:    v_add_co_u32_e64 v2, s[4:5], v5, v1
+; SDAG-NEXT:    v_addc_co_u32_e64 v3, s[4:5], v6, v3, s[4:5]
 ; SDAG-NEXT:    ; implicit-def: $vgpr5_vgpr6
 ; SDAG-NEXT:    v_mov_b32_e32 v1, v4
 ; SDAG-NEXT:    ; implicit-def: $vgpr6_vgpr7
@@ -1207,29 +1197,29 @@ define i128 @fptoui_f32_to_i128(float %x) {
 ; SDAG-NEXT:  ; %bb.5: ; %fp-to-i-if-then12
 ; SDAG-NEXT:    v_sub_u32_e32 v2, 0x96, v5
 ; SDAG-NEXT:    v_lshrrev_b64 v[0:1], v2, v[6:7]
-; SDAG-NEXT:    v_cmp_gt_u32_e32 vcc, 64, v2
-; SDAG-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
-; SDAG-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; SDAG-NEXT:    v_cndmask_b32_e32 v3, v0, v6, vcc
-; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[12:13], v3, v10, 0
+; SDAG-NEXT:    v_cmp_gt_u32_e64 s[4:5], 64, v2
+; SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, v0, s[4:5]
+; SDAG-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v2
+; SDAG-NEXT:    v_cndmask_b32_e64 v3, v0, v6, s[4:5]
+; SDAG-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v3, v10, 0
 ; SDAG-NEXT:    v_mov_b32_e32 v2, 0
-; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[12:13], v3, v8, v[1:2]
+; SDAG-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v3, v8, v[1:2]
 ; SDAG-NEXT:    v_mov_b32_e32 v1, v5
-; SDAG-NEXT:    v_mad_i64_i32 v[2:3], s[12:13], v9, v3, v[1:2]
+; SDAG-NEXT:    v_mad_i64_i32 v[2:3], s[4:5], v9, v3, v[1:2]
 ; SDAG-NEXT:    v_mov_b32_e32 v1, v4
 ; SDAG-NEXT:  .LBB3_6: ; %Flow1
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; SDAG-NEXT:  .LBB3_7: ; %Flow2
-; SDAG-NEXT:    s_andn2_saveexec_b64 s[6:7], s[10:11]
+; SDAG-NEXT:    s_andn2_saveexec_b64 s[4:5], s[10:11]
 ; SDAG-NEXT:  ; %bb.8: ; %fp-to-i-if-then5
 ; SDAG-NEXT:    v_bfrev_b32_e32 v0, 1
 ; SDAG-NEXT:    v_bfrev_b32_e32 v1, -2
-; SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, -1, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v3, v0, v1, s[4:5]
+; SDAG-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; SDAG-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
 ; SDAG-NEXT:    v_mov_b32_e32 v0, v2
 ; SDAG-NEXT:    v_mov_b32_e32 v1, v2
 ; SDAG-NEXT:  ; %bb.9: ; %Flow3
-; SDAG-NEXT:    s_or_b64 exec, exec, s[6:7]
+; SDAG-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; SDAG-NEXT:  .LBB3_10: ; %fp-to-i-cleanup
 ; SDAG-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -1257,14 +1247,16 @@ define i128 @fptoui_f32_to_i128(float %x) {
 ; GISEL-NEXT:    v_add_co_u32_e32 v0, vcc, 0xffffff01, v6
 ; GISEL-NEXT:    v_mov_b32_e32 v2, 0xffffff80
 ; GISEL-NEXT:    v_addc_co_u32_e64 v1, s[6:7], 0, -1, vcc
-; GISEL-NEXT:    v_mov_b32_e32 v3, 0
+; GISEL-NEXT:    v_mov_b32_e32 v3, -1
 ; GISEL-NEXT:    v_addc_co_u32_e64 v8, s[6:7], 0, -1, s[6:7]
 ; GISEL-NEXT:    v_cmp_ge_u64_e32 vcc, v[0:1], v[2:3]
 ; GISEL-NEXT:    v_addc_co_u32_e64 v9, s[6:7], 0, -1, s[6:7]
 ; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GISEL-NEXT:    v_cmp_le_u64_e32 vcc, -1, v[8:9]
 ; GISEL-NEXT:    v_cmp_lt_i32_e64 s[4:5], -1, v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, 1, v0, vcc
+; GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT:    v_cmp_eq_u64_e32 vcc, -1, v[8:9]
+; GISEL-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
 ; GISEL-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GISEL-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
diff --git a/llvm/test/Transforms/ExpandLargeFpConvert/X86/expand-large-fp-convert-fptosi129.ll b/llvm/test/Transforms/ExpandLargeFpConvert/X86/expand-large-fp-convert-fptosi129.ll
index 77bbd5f0bb42..75130c27f473 100644
--- a/llvm/test/Transforms/ExpandLargeFpConvert/X86/expand-large-fp-convert-fptosi129.ll
+++ b/llvm/test/Transforms/ExpandLargeFpConvert/X86/expand-large-fp-convert-fptosi129.ll
@@ -27,7 +27,7 @@ define i129 @floattosi129(float %a) {
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[FP_TO_I_CLEANUP:%.*]], label [[FP_TO_I_IF_END:%.*]]
 ; CHECK:       fp-to-i-if-end:
 ; CHECK-NEXT:    [[TMP9:%.*]] = add i129 [[TMP5]], -256
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp ult i129 [[TMP9]], 4294967167
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ult i129 [[TMP9]], -129
 ; CHECK-NEXT:    br i1 [[TMP10]], label [[FP_TO_I_IF_THEN5:%.*]], label [[FP_TO_I_IF_END9:%.*]]
 ; CHECK:       fp-to-i-if-then5:
 ; CHECK-NEXT:    [[TMP11:%.*]] = select i1 [[TMP2]], i129 340282366920938463463374607431768211455, i129 -340282366920938463463374607431768211456
@@ -41,7 +41,7 @@ define i129 @floattosi129(float %a) {
 ; CHECK-NEXT:    [[TMP15:%.*]] = mul i129 [[TMP14]], [[TMP3]]
 ; CHECK-NEXT:    br label [[FP_TO_I_CLEANUP]]
 ; CHECK:       fp-to-i-if-else:
-; CHECK-NEXT:    [[TMP16:%.*]] = add i129 [[TMP5]], 4294967146
+; CHECK-NEXT:    [[TMP16:%.*]] = add i129 [[TMP5]], -150
 ; CHECK-NEXT:    [[TMP17:%.*]] = shl i129 [[TMP7]], [[TMP16]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = mul i129 [[TMP17]], [[TMP3]]
 ; CHECK-NEXT:    br label [[FP_TO_I_CLEANUP]]
@@ -68,7 +68,7 @@ define i129 @doubletosi129(double %a) {
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[FP_TO_I_CLEANUP:%.*]], label [[FP_TO_I_IF_END:%.*]]
 ; CHECK:       fp-to-i-if-end:
 ; CHECK-NEXT:    [[TMP9:%.*]] = add i129 [[TMP5]], -1152
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp ult i129 [[TMP9]], 4294967167
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ult i129 [[TMP9]], -129
 ; CHECK-NEXT:    br i1 [[TMP10]], label [[FP_TO_I_IF_THEN5:%.*]], label [[FP_TO_I_IF_END9:%.*]]
 ; CHECK:       fp-to-i-if-then5:
 ; CHECK-NEXT:    [[TMP11:%.*]] = select i1 [[TMP2]], i129 340282366920938463463374607431768211455, i129 -340282366920938463463374607431768211456
@@ -82,7 +82,7 @@ define i129 @doubletosi129(double %a) {
 ; CHECK-NEXT:    [[TMP15:%.*]] = mul i129 [[TMP14]], [[TMP3]]
 ; CHECK-NEXT:    br label [[FP_TO_I_CLEANUP]]
 ; CHECK:       fp-to-i-if-else:
-; CHECK-NEXT:    [[TMP16:%.*]] = add i129 [[TMP5]], 4294966221
+; CHECK-NEXT:    [[TMP16:%.*]] = add i129 [[TMP5]], -1075
 ; CHECK-NEXT:    [[TMP17:%.*]] = shl i129 [[TMP7]], [[TMP16]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = mul i129 [[TMP17]], [[TMP3]]
 ; CHECK-NEXT:    br label [[FP_TO_I_CLEANUP]]
@@ -110,7 +110,7 @@ define i129 @x86_fp80tosi129(x86_fp80 %a) {
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[FP_TO_I_CLEANUP:%.*]], label [[FP_TO_I_IF_END:%.*]]
 ; CHECK:       fp-to-i-if-end:
 ; CHECK-NEXT:    [[TMP10:%.*]] = add i129 [[TMP6]], -16512
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp ult i129 [[TMP10]], 4294967167
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ult i129 [[TMP10]], -129
 ; CHECK-NEXT:    br i1 [[TMP11]], label [[FP_TO_I_IF_THEN5:%.*]], label [[FP_TO_I_IF_END9:%.*]]
 ; CHECK:       fp-to-i-if-then5:
 ; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP3]], i129 340282366920938463463374607431768211455, i129 -340282366920938463463374607431768211456
@@ -124,7 +124,7 @@ define i129 @x86_fp80tosi129(x86_fp80 %a) {
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul i129 [[TMP15]], [[TMP4]]
 ; CHECK-NEXT:    br label [[FP_TO_I_CLEANUP]]
 ; CHECK:       fp-to-i-if-else:
-; CHECK-NEXT:    [[TMP17:%.*]] = add i129 [[TMP6]], 4294950801
+; CHECK-NEXT:    [[TMP17:%.*]] = add i129 [[TMP6]], -16495
 ; CHECK-NEXT:    [[TMP18:%.*]] = shl i129 [[TMP8]], [[TMP17]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = mul i129 [[TMP18]], [[TMP4]]
 ; CHECK-NEXT:    br label [[FP_TO_I_CLEANUP]]
@@ -151,7 +151,7 @@ define i129 @fp128tosi129(fp128 %a) {
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[FP_TO_I_CLEANUP:%.*]], label [[FP_TO_I_IF_END:%.*]]
 ; CHECK:       fp-to-i-if-end:
 ; CHECK-NEXT:    [[TMP9:%.*]] = add i129 [[TMP5]], -16512
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp ult i129 [[TMP9]], 4294967167
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ult i129 [[TMP9]], -129
 ; CHECK-NEXT:    br i1 [[TMP10]], label [[FP_TO_I_IF_THEN5:%.*]], label [[FP_TO_I_IF_END9:%.*]]
 ; CHECK:       fp-to-i-if-then5:
 ; CHECK-NEXT:    [[TMP11:%.*]] = select i1 [[TMP2]], i129 340282366920938463463374607431768211455, i129 -340282366920938463463374607431768211456
@@ -165,7 +165,7 @@ define i129 @fp128tosi129(fp128 %a) {
 ; CHECK-NEXT:    [[TMP15:%.*]] = mul i129 [[TMP14]], [[TMP3]]
 ; CHECK-NEXT:    br label [[FP_TO_I_CLEANUP]]
 ; CHECK:       fp-to-i-if-else:
-; CHECK-NEXT:    [[TMP16:%.*]] = add i129 [[TMP5]], 4294950801
+; CHECK-NEXT:    [[TMP16:%.*]] = add i129 [[TMP5]], -16495
 ; CHECK-NEXT:    [[TMP17:%.*]] = shl i129 [[TMP7]], [[TMP16]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = mul i129 [[TMP17]], [[TMP3]]
 ; CHECK-NEXT:    br label [[FP_TO_I_CLEANUP]]
diff --git a/llvm/test/Transforms/ExpandLargeFpConvert/X86/expand-large-fp-convert-fptoui129.ll b/llvm/test/Transforms/ExpandLargeFpConvert/X86/expand-large-fp-convert-fptoui129.ll
index 67d9eb533a3e..ed630d7934c3 100644
--- a/llvm/test/Transforms/ExpandLargeFpConvert/X86/expand-large-fp-convert-fptoui129.ll
+++ b/llvm/test/Transforms/ExpandLargeFpConvert/X86/expand-large-fp-convert-fptoui129.ll
@@ -27,7 +27,7 @@ define i129 @floattoui129(float %a) {
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[FP_TO_I_CLEANUP:%.*]], label [[FP_TO_I_IF_END:%.*]]
 ; CHECK:       fp-to-i-if-end:
 ; CHECK-NEXT:    [[TMP9:%.*]] = add i129 [[TMP5]], -256
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp ult i129 [[TMP9]], 4294967167
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ult i129 [[TMP9]], -129
 ; CHECK-NEXT:    br i1 [[TMP10]], label [[FP_TO_I_IF_THEN5:%.*]], label [[FP_TO_I_IF_END9:%.*]]
 ; CHECK:       fp-to-i-if-then5:
 ; CHECK-NEXT:    [[TMP11:%.*]] = select i1 [[TMP2]], i129 340282366920938463463374607431768211455, i129 -340282366920938463463374607431768211456
@@ -41,7 +41,7 @@ define i129 @floattoui129(float %a) {
 ; CHECK-NEXT:    [[TMP15:%.*]] = mul i129 [[TMP14]], [[TMP3]]
 ; CHECK-NEXT:    br label [[FP_TO_I_CLEANUP]]
 ; CHECK:       fp-to-i-if-else:
-; CHECK-NEXT:    [[TMP16:%.*]] = add i129 [[TMP5]], 4294967146
+; CHECK-NEXT:    [[TMP16:%.*]] = add i129 [[TMP5]], -150
 ; CHECK-NEXT:    [[TMP17:%.*]] = shl i129 [[TMP7]], [[TMP16]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = mul i129 [[TMP17]], [[TMP3]]
 ; CHECK-NEXT:    br label [[FP_TO_I_CLEANUP]]
@@ -68,7 +68,7 @@ define i129 @doubletoui129(double %a) {
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[FP_TO_I_CLEANUP:%.*]], label [[FP_TO_I_IF_END:%.*]]
 ; CHECK:       fp-to-i-if-end:
 ; CHECK-NEXT:    [[TMP9:%.*]] = add i129 [[TMP5]], -1152
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp ult i129 [[TMP9]], 4294967167
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ult i129 [[TMP9]], -129
 ; CHECK-NEXT:    br i1 [[TMP10]], label [[FP_TO_I_IF_THEN5:%.*]], label [[FP_TO_I_IF_END9:%.*]]
 ; CHECK:       fp-to-i-if-then5:
 ; CHECK-NEXT:    [[TMP11:%.*]] = select i1 [[TMP2]], i129 340282366920938463463374607431768211455, i129 -340282366920938463463374607431768211456
@@ -82,7 +82,7 @@ define i129 @doubletoui129(double %a) {
 ; CHECK-NEXT:    [[TMP15:%.*]] = mul i129 [[TMP14]], [[TMP3]]
 ; CHECK-NEXT:    br label [[FP_TO_I_CLEANUP]]
 ; CHECK:       fp-to-i-if-else:
-; CHECK-NEXT:    [[TMP16:%.*]] = add i129 [[TMP5]], 4294966221
+; CHECK-NEXT:    [[TMP16:%.*]] = add i129 [[TMP5]], -1075
 ; CHECK-NEXT:    [[TMP17:%.*]] = shl i129 [[TMP7]], [[TMP16]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = mul i129 [[TMP17]], [[TMP3]]
 ; CHECK-NEXT:    br label [[FP_TO_I_CLEANUP]]
@@ -110,7 +110,7 @@ define i129 @x86_fp80toui129(x86_fp80 %a) {
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[FP_TO_I_CLEANUP:%.*]], label [[FP_TO_I_IF_END:%.*]]
 ; CHECK:       fp-to-i-if-end:
 ; CHECK-NEXT:    [[TMP10:%.*]] = add i129 [[TMP6]], -16512
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp ult i129 [[TMP10]], 4294967167
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ult i129 [[TMP10]], -129
 ; CHECK-NEXT:    br i1 [[TMP11]], label [[FP_TO_I_IF_THEN5:%.*]], label [[FP_TO_I_IF_END9:%.*]]
 ; CHECK:       fp-to-i-if-then5:
 ; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP3]], i129 340282366920938463463374607431768211455, i129 -340282366920938463463374607431768211456
@@ -124,7 +124,7 @@ define i129 @x86_fp80toui129(x86_fp80 %a) {
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul i129 [[TMP15]], [[TMP4]]
 ; CHECK-NEXT:    br label [[FP_TO_I_CLEANUP]]
 ; CHECK:       fp-to-i-if-else:
-; CHECK-NEXT:    [[TMP17:%.*]] = add i129 [[TMP6]], 4294950801
+; CHECK-NEXT:    [[TMP17:%.*]] = add i129 [[TMP6]], -16495
 ; CHECK-NEXT:    [[TMP18:%.*]] = shl i129 [[TMP8]], [[TMP17]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = mul i129 [[TMP18]], [[TMP4]]
 ; CHECK-NEXT:    br label [[FP_TO_I_CLEANUP]]
@@ -151,7 +151,7 @@ define i129 @fp128toui129(fp128 %a) {
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[FP_TO_I_CLEANUP:%.*]], label [[FP_TO_I_IF_END:%.*]]
 ; CHECK:       fp-to-i-if-end:
 ; CHECK-NEXT:    [[TMP9:%.*]] = add i129 [[TMP5]], -16512
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp ult i129 [[TMP9]], 4294967167
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ult i129 [[TMP9]], -129
 ; CHECK-NEXT:    br i1 [[TMP10]], label [[FP_TO_I_IF_THEN5:%.*]], label [[FP_TO_I_IF_END9:%.*]]
 ; CHECK:       fp-to-i-if-then5:
 ; CHECK-NEXT:    [[TMP11:%.*]] = select i1 [[TMP2]], i129 340282366920938463463374607431768211455, i129 -340282366920938463463374607431768211456
@@ -165,7 +165,7 @@ define i129 @fp128toui129(fp128 %a) {
 ; CHECK-NEXT:    [[TMP15:%.*]] = mul i129 [[TMP14]], [[TMP3]]
 ; CHECK-NEXT:    br label [[FP_TO_I_CLEANUP]]
 ; CHECK:       fp-to-i-if-else:
-; CHECK-NEXT:    [[TMP16:%.*]] = add i129 [[TMP5]], 4294950801
+; CHECK-NEXT:    [[TMP16:%.*]] = add i129 [[TMP5]], -16495
 ; CHECK-NEXT:    [[TMP17:%.*]] = shl i129 [[TMP7]], [[TMP16]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = mul i129 [[TMP17]], [[TMP3]]
 ; CHECK-NEXT:    br label [[FP_TO_I_CLEANUP]]
-- 
GitLab


From 4d315ff382de912e5129b417c997116851088d4b Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Tue, 26 Mar 2024 09:11:35 +0000
Subject: [PATCH 249/404] [GlobalISel] Add CTLZ known bits. (#86436)

Replicated from SDAG.
---
 llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp        | 11 +++++++++++
 llvm/test/CodeGen/AArch64/setcc_knownbits.ll          |  4 +---
 .../AMDGPU/GlobalISel/legalize-ctlz-zero-undef.mir    |  4 +---
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
index 2e2cc9a95bd9..914022ab7301 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
@@ -589,6 +589,17 @@ void GISelKnownBits::computeKnownBitsImpl(Register R, KnownBits &Known,
     }
     break;
   }
+  case TargetOpcode::G_CTLZ:
+  case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
+    KnownBits SrcOpKnown;
+    computeKnownBitsImpl(MI.getOperand(1).getReg(), SrcOpKnown, DemandedElts,
+                         Depth + 1);
+    // If we have a known 1, its position is our upper bound.
+    unsigned PossibleLZ = SrcOpKnown.countMaxLeadingZeros();
+    unsigned LowBits = llvm::bit_width(PossibleLZ);
+    Known.Zero.setBitsFrom(LowBits);
+    break;
+  }
   }
 
   assert(!Known.hasConflict() && "Bits known to be one AND zero?");
diff --git a/llvm/test/CodeGen/AArch64/setcc_knownbits.ll b/llvm/test/CodeGen/AArch64/setcc_knownbits.ll
index 6a337bf02e09..8be63b04d8ce 100644
--- a/llvm/test/CodeGen/AArch64/setcc_knownbits.ll
+++ b/llvm/test/CodeGen/AArch64/setcc_knownbits.ll
@@ -68,9 +68,7 @@ define i1 @lshr_ctlz_undef_cmpeq_one_i64(i64 %in) {
 ; CHECK-GI-LABEL: lshr_ctlz_undef_cmpeq_one_i64:
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    clz x8, x0
-; CHECK-GI-NEXT:    lsr x8, x8, #6
-; CHECK-GI-NEXT:    cmp x8, #1
-; CHECK-GI-NEXT:    cset w0, eq
+; CHECK-GI-NEXT:    lsr w0, w8, #6
 ; CHECK-GI-NEXT:    ret
   %ctlz = call i64 @llvm.ctlz.i64(i64 %in, i1 -1)
   %lshr = lshr i64 %ctlz, 6
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-undef.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-undef.mir
index e9f8180aae66..fed277d7d10d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-undef.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-undef.mir
@@ -64,9 +64,7 @@ body: |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; CHECK-NEXT: [[CTLZ_ZERO_UNDEF:%[0-9]+]]:_(s32) = G_CTLZ_ZERO_UNDEF [[COPY]](s32)
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[CTLZ_ZERO_UNDEF]], [[C]]
-    ; CHECK-NEXT: $vgpr0 = COPY [[AND]](s32)
+    ; CHECK-NEXT: $vgpr0 = COPY [[CTLZ_ZERO_UNDEF]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s16) = G_CTLZ_ZERO_UNDEF %0
     %2:_(s32) = G_ZEXT %1
-- 
GitLab


From eb70b485a91361eee83d3744d1bd3e4c3a23692f Mon Sep 17 00:00:00 2001
From: Christian Sigg <chsigg@users.noreply.github.com>
Date: Tue, 26 Mar 2024 10:23:47 +0100
Subject: [PATCH 250/404] [mlir][bazel] Export more headers by a single target
 only. (#86637)

Ideally, header files should be used by only one target, but this is
hard because CMake is less strict with headers (no layering check). But
even with bazel, headers should only be exported once in the `hdrs`
attribute. Other targets may use them in the `srcs` attribute to avoid
circular dependencies.
---
 .../include/mlir/Dialect/LLVMIR/LLVMDialect.h |  1 -
 .../llvm-project-overlay/mlir/BUILD.bazel     | 51 ++++++++++---------
 2 files changed, 27 insertions(+), 25 deletions(-)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h
index 06df4a601b7a..9341a5a11cd6 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.h
@@ -30,7 +30,6 @@
 #include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Support/ThreadLocalCache.h"
-#include "mlir/Transforms/Mem2Reg.h"
 #include "llvm/ADT/PointerEmbeddedInt.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/LLVMContext.h"
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 71fb97bff62f..1d9260e022c5 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -3823,9 +3823,10 @@ cc_library(
     srcs = glob([
         "lib/Dialect/*.cpp",
     ]),
-    hdrs = glob([
-        "include/mlir/Dialect/*.h",
-    ]),
+    hdrs = glob(
+        include = ["include/mlir/Dialect/*.h"],
+        exclude = ["include/mlir/Dialect/CommonFolders.h"],
+    ),
     includes = ["include"],
     deps = [
         ":IR",
@@ -4534,6 +4535,7 @@ cc_library(
         ":ArithDialect",
         ":BufferizationInterfaces",
         ":CastInterfaces",
+        ":CommonFolders",
         ":ControlFlowInterfaces",
         ":Dialect",
         ":FuncDialect",
@@ -5336,9 +5338,7 @@ cc_library(
             "include/mlir/Dialect/LLVMIR/VCIX*.h",
             "include/mlir/Dialect/LLVMIR/*X86Vector*.h",
         ],
-    ) + [
-        "include/mlir/Transforms/Mem2Reg.h",
-    ],
+    ),
     includes = ["include"],
     deps = [
         ":CallOpInterfaces",
@@ -8635,17 +8635,26 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "ParseUtilities",
+    hdrs = ["include/mlir/Tools/ParseUtilities.h"],
+    includes = ["include"],
+    deps = [
+        ":IR",
+        ":Parser",
+    ],
+)
+
 cc_library(
     name = "TranslateLib",
     srcs = glob([
         "lib/Tools/mlir-translate/*.cpp",
-    ]) + [
-        "include/mlir/Tools/ParseUtilities.h",
-    ],
+    ]),
     hdrs = glob(["include/mlir/Tools/mlir-translate/*.h"]),
     includes = ["include"],
     deps = [
         ":IR",
+        ":ParseUtilities",
         ":Parser",
         ":Support",
         "//llvm:Support",
@@ -8655,6 +8664,7 @@ cc_library(
 cc_library(
     name = "ToLLVMIRTranslation",
     srcs = [
+        "lib/Target/LLVMIR/AttrKindDetail.h",
         "lib/Target/LLVMIR/DebugTranslation.cpp",
         "lib/Target/LLVMIR/DebugTranslation.h",
         "lib/Target/LLVMIR/LoopAnnotationTranslation.cpp",
@@ -8667,7 +8677,6 @@ cc_library(
         "include/mlir/Target/LLVMIR/LLVMTranslationInterface.h",
         "include/mlir/Target/LLVMIR/ModuleTranslation.h",
         "include/mlir/Target/LLVMIR/TypeToLLVM.h",
-        "lib/Target/LLVMIR/AttrKindDetail.h",
     ],
     includes = ["include"],
     deps = [
@@ -8988,9 +8997,11 @@ cc_library(
 cc_library(
     name = "FromLLVMIRTranslation",
     srcs = [
+        "lib/Target/LLVMIR/AttrKindDetail.h",
         "lib/Target/LLVMIR/DataLayoutImporter.cpp",
         "lib/Target/LLVMIR/DataLayoutImporter.h",
         "lib/Target/LLVMIR/DebugImporter.cpp",
+        "lib/Target/LLVMIR/DebugImporter.h",
         "lib/Target/LLVMIR/LoopAnnotationImporter.cpp",
         "lib/Target/LLVMIR/LoopAnnotationImporter.h",
         "lib/Target/LLVMIR/ModuleImport.cpp",
@@ -9001,8 +9012,6 @@ cc_library(
         "include/mlir/Target/LLVMIR/LLVMImportInterface.h",
         "include/mlir/Target/LLVMIR/ModuleImport.h",
         "include/mlir/Target/LLVMIR/TypeFromLLVM.h",
-        "lib/Target/LLVMIR/AttrKindDetail.h",
-        "lib/Target/LLVMIR/DebugImporter.h",
     ],
     includes = ["include"],
     deps = [
@@ -9109,10 +9118,7 @@ cc_library(
 
 cc_library(
     name = "MlirOptLib",
-    srcs = [
-        "include/mlir/Tools/ParseUtilities.h",
-        "lib/Tools/mlir-opt/MlirOptMain.cpp",
-    ],
+    srcs = ["lib/Tools/mlir-opt/MlirOptMain.cpp"],
     hdrs = ["include/mlir/Tools/mlir-opt/MlirOptMain.h"],
     includes = ["include"],
     deps = [
@@ -9121,6 +9127,7 @@ cc_library(
         ":Debug",
         ":IR",
         ":IRDLDialect",
+        ":ParseUtilities",
         ":Parser",
         ":Pass",
         ":PluginsLib",
@@ -9393,10 +9400,7 @@ cc_binary(
 cc_library(
     name = "MlirJitRunner",
     srcs = ["lib/ExecutionEngine/JitRunner.cpp"],
-    hdrs = [
-        "include/mlir/ExecutionEngine/JitRunner.h",
-        "include/mlir/Tools/ParseUtilities.h",
-    ],
+    hdrs = ["include/mlir/ExecutionEngine/JitRunner.h"],
     includes = ["include"],
     deps = [
         ":AllPassesAndDialects",
@@ -9407,6 +9411,7 @@ cc_library(
         ":LLVMToLLVMIRTranslation",
         ":OpenACCToLLVMIRTranslation",
         ":OpenMPToLLVMIRTranslation",
+        ":ParseUtilities",
         ":Parser",
         ":SCFToControlFlow",
         ":Support",
@@ -13786,13 +13791,11 @@ cc_library(
 cc_library(
     name = "MlirReduceLib",
     srcs = ["lib/Tools/mlir-reduce/MlirReduceMain.cpp"],
-    hdrs = [
-        "include/mlir/Tools/ParseUtilities.h",
-        "include/mlir/Tools/mlir-reduce/MlirReduceMain.h",
-    ],
+    hdrs = ["include/mlir/Tools/mlir-reduce/MlirReduceMain.h"],
     includes = ["include"],
     deps = [
         ":IR",
+        ":ParseUtilities",
         ":Parser",
         ":Pass",
         ":Reducer",
-- 
GitLab


From 3f8431ec66ffcfaf1bd864261f425b12ab1b59b8 Mon Sep 17 00:00:00 2001
From: XChy <xxs_chy@outlook.com>
Date: Tue, 26 Mar 2024 17:31:55 +0800
Subject: [PATCH 251/404] [SCCIterator] Union MST node by rank correctly
 (#86389)

Fixes #85975
---
 llvm/include/llvm/ADT/SCCIterator.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/ADT/SCCIterator.h b/llvm/include/llvm/ADT/SCCIterator.h
index e743ae7c11ed..3bd103c13f19 100644
--- a/llvm/include/llvm/ADT/SCCIterator.h
+++ b/llvm/include/llvm/ADT/SCCIterator.h
@@ -281,14 +281,14 @@ class scc_member_iterator {
     if (G1 == G2)
       return false;
 
-    // Make the smaller rank tree a direct child or the root of high rank tree.
-    if (G1->Rank < G1->Rank)
+    // Make the smaller rank tree a direct child of high rank tree.
+    if (G1->Rank < G2->Rank)
       G1->Group = G2;
     else {
       G2->Group = G1;
       // If the ranks are the same, increment root of one tree by one.
       if (G1->Rank == G2->Rank)
-        G2->Rank++;
+        G1->Rank++;
     }
     return true;
   }
-- 
GitLab


From 4fa736ba41fce885047bd5a242230b216ea85c70 Mon Sep 17 00:00:00 2001
From: NagyDonat <donat.nagy@ericsson.com>
Date: Tue, 26 Mar 2024 10:33:12 +0100
Subject: [PATCH 252/404] [Clang] NFC Silence compiler warning spam (#86532)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This non-functional change eliminates the compiler warning
```
/repo/llvm-project/clang/include/clang/Basic/SyncScope.h: In member function ‘virtual bool clang::AtomicScopeGenericModel::isValid(unsigned int) const’:
/repo/llvm-project/clang/include/clang/Basic/SyncScope.h:255:14: warning: comparison of unsigned expression >= 0 is always true [-Wtype-limits]
     return S >= static_cast<unsigned>(System) &&
            ~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
```
which was appearing repeatedly, as this header is included into very
many source files (through various chains of 6-7 header files).
---
 clang/include/clang/Basic/SyncScope.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/clang/include/clang/Basic/SyncScope.h b/clang/include/clang/Basic/SyncScope.h
index bc7ec7b5cf77..45beff41afa1 100644
--- a/clang/include/clang/Basic/SyncScope.h
+++ b/clang/include/clang/Basic/SyncScope.h
@@ -252,8 +252,7 @@ public:
   }
 
   bool isValid(unsigned S) const override {
-    return S >= static_cast<unsigned>(System) &&
-           S <= static_cast<unsigned>(Last);
+    return S <= static_cast<unsigned>(Last);
   }
 
   ArrayRef<unsigned> getRuntimeValues() const override {
-- 
GitLab


From fbc247367a86e8a711584801586ec9616268d889 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Tue, 26 Mar 2024 09:48:06 +0000
Subject: [PATCH 253/404] [AArch64][GlobalISel] Legalization for small
 anyext/sext/zext (#86438)

Similar to #85625, some of the codegen is still far from optimal but
this helps fix quite a few fallback cases.
---
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    |  22 +-
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    |   5 +-
 .../GlobalISel/legalize-build-vector.mir      |  27 +-
 .../GlobalISel/legalize-insert-vector-elt.mir |  29 +-
 .../AArch64/GlobalISel/legalize-select.mir    |  57 ++--
 .../legalize-shuffle-vector-widen-crash.ll    |  14 +-
 .../AArch64/GlobalISel/legalize-xtn.mir       |  18 +-
 llvm/test/CodeGen/AArch64/aarch64-smull.ll    |  53 +++-
 .../AArch64/arm64-extract-insert-varidx.ll    |  34 +--
 llvm/test/CodeGen/AArch64/bitcast.ll          | 145 ++++++---
 llvm/test/CodeGen/AArch64/bswap.ll            |  26 +-
 llvm/test/CodeGen/AArch64/fptoi.ll            |  45 ++-
 llvm/test/CodeGen/AArch64/itofp.ll            | 278 +++++++++++++-----
 llvm/test/CodeGen/AArch64/load.ll             |   3 +-
 .../AArch64/neon-bitwise-instructions.ll      |  46 ++-
 llvm/test/CodeGen/AArch64/shift.ll            | 225 ++++++++++----
 16 files changed, 714 insertions(+), 313 deletions(-)

diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 1b25da8833e4..c3a23ea0ad37 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -5421,14 +5421,22 @@ LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
   case TargetOpcode::G_FPTOUI:
   case TargetOpcode::G_SITOFP:
   case TargetOpcode::G_UITOFP: {
-    if (TypeIdx != 0)
-      return UnableToLegalize;
     Observer.changingInstr(MI);
-    LLT SrcTy = LLT::fixed_vector(
-        MoreTy.getNumElements(),
-        MRI.getType(MI.getOperand(1).getReg()).getElementType());
-    moreElementsVectorSrc(MI, SrcTy, 1);
-    moreElementsVectorDst(MI, MoreTy, 0);
+    LLT SrcExtTy;
+    LLT DstExtTy;
+    if (TypeIdx == 0) {
+      DstExtTy = MoreTy;
+      SrcExtTy = LLT::fixed_vector(
+          MoreTy.getNumElements(),
+          MRI.getType(MI.getOperand(1).getReg()).getElementType());
+    } else {
+      DstExtTy = LLT::fixed_vector(
+          MoreTy.getNumElements(),
+          MRI.getType(MI.getOperand(0).getReg()).getElementType());
+      SrcExtTy = MoreTy;
+    }
+    moreElementsVectorSrc(MI, SrcExtTy, 1);
+    moreElementsVectorDst(MI, DstExtTy, 0);
     Observer.changedInstr(MI);
     return Legalized;
   }
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 34e2c1d9c8e2..33dba6a5c61e 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -611,7 +611,9 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
                Query.Types[0].isVector() &&
                (Query.Types[1].getScalarSizeInBits() == 8 ||
                 Query.Types[1].getScalarSizeInBits() == 16);
-      });
+      })
+      .clampMinNumElements(1, s8, 8)
+      .clampMinNumElements(1, s16, 4);
 
   getActionDefinitionsBuilder(G_TRUNC)
       .legalFor({{v2s32, v2s64}, {v4s16, v4s32}, {v8s8, v8s16}})
@@ -630,7 +632,6 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       })
       .clampMinNumElements(0, s8, 8)
       .clampMinNumElements(0, s16, 4)
-      .clampMinNumElements(0, s32, 2)
       .alwaysLegal();
 
   getActionDefinitionsBuilder(G_SEXT_INREG)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-build-vector.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-build-vector.mir
index c9556e27c634..a63d8b9c1377 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-build-vector.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-build-vector.mir
@@ -121,10 +121,11 @@ body: |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s16) = COPY $h0
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s16) = COPY $h1
-    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[COPY]](s16)
-    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[COPY1]](s16)
-    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ANYEXT]](s32), [[ANYEXT1]](s32)
-    ; CHECK-NEXT: $d0 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[COPY]](s16), [[COPY1]](s16), [[DEF]](s16), [[DEF]](s16)
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(<4 x s32>) = G_ANYEXT [[BUILD_VECTOR]](<4 x s16>)
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s32>), [[UV1:%[0-9]+]]:_(<2 x s32>) = G_UNMERGE_VALUES [[ANYEXT]](<4 x s32>)
+    ; CHECK-NEXT: $d0 = COPY [[UV]](<2 x s32>)
     ; CHECK-NEXT: RET_ReallyLR
     %0:_(s16) = COPY $h0
     %1:_(s16) = COPY $h1
@@ -141,8 +142,8 @@ body: |
     ; CHECK-LABEL: name: widen_v2s8
     ; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY [[DEF]](s32)
-    ; CHECK-NEXT: %3:_(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[DEF]](s32)
-    ; CHECK-NEXT: $d0 = COPY %3(<2 x s32>)
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[DEF]](s32)
+    ; CHECK-NEXT: $d0 = COPY [[BUILD_VECTOR]](<2 x s32>)
     ; CHECK-NEXT: RET_ReallyLR
     %0:_(s8) = G_IMPLICIT_DEF
     %1:_(s8) = G_IMPLICIT_DEF
@@ -157,12 +158,14 @@ name:            widen_v4s8
 body: |
   bb.0:
     ; CHECK-LABEL: name: widen_v4s8
-    ; CHECK: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s16) = COPY [[DEF]](s16)
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s16) = COPY [[DEF]](s16)
-    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s16) = COPY [[DEF]](s16)
-    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[COPY]](s16), [[COPY1]](s16), [[COPY2]](s16), [[DEF]](s16)
-    ; CHECK-NEXT: $d0 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; CHECK: [[DEF:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF2:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF3:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF1]](s8), [[DEF2]](s8), [[DEF3]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(<8 x s16>) = G_ANYEXT [[BUILD_VECTOR]](<8 x s8>)
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(<4 x s16>), [[UV1:%[0-9]+]]:_(<4 x s16>) = G_UNMERGE_VALUES [[ANYEXT]](<8 x s16>)
+    ; CHECK-NEXT: $d0 = COPY [[UV]](<4 x s16>)
     ; CHECK-NEXT: RET_ReallyLR
     %0:_(s8) = G_IMPLICIT_DEF
     %1:_(s8) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-insert-vector-elt.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-insert-vector-elt.mir
index e12353c7ef5b..d3db2432e84c 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-insert-vector-elt.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-insert-vector-elt.mir
@@ -235,31 +235,32 @@ body:             |
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK-NEXT:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
+  ; CHECK-NEXT:   [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY2]](s32)
+  ; CHECK-NEXT:   [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[COPY3]](s32)
   ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
-  ; CHECK-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
-  ; CHECK-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
-  ; CHECK-NEXT:   [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY3]](s32)
-  ; CHECK-NEXT:   [[DEF2:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
-  ; CHECK-NEXT:   [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16), [[TRUNC2]](s16), [[DEF2]](s16)
+  ; CHECK-NEXT:   [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s8>) = G_BUILD_VECTOR [[TRUNC]](s8), [[TRUNC1]](s8), [[TRUNC2]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8)
+  ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(<8 x s16>) = G_ANYEXT [[BUILD_VECTOR]](<8 x s8>)
+  ; CHECK-NEXT:   [[UV:%[0-9]+]]:_(<4 x s16>), [[UV1:%[0-9]+]]:_(<4 x s16>) = G_UNMERGE_VALUES [[ANYEXT]](<8 x s16>)
   ; CHECK-NEXT:   [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
-  ; CHECK-NEXT:   [[IVEC:%[0-9]+]]:_(<4 x s16>) = G_INSERT_VECTOR_ELT [[BUILD_VECTOR]], [[C2]](s16), [[C1]](s64)
-  ; CHECK-NEXT:   [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[IVEC]](<4 x s16>)
-  ; CHECK-NEXT:   [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[UV]](s16)
-  ; CHECK-NEXT:   [[TRUNC4:%[0-9]+]]:_(s8) = G_TRUNC [[UV1]](s16)
-  ; CHECK-NEXT:   [[TRUNC5:%[0-9]+]]:_(s8) = G_TRUNC [[UV2]](s16)
+  ; CHECK-NEXT:   [[IVEC:%[0-9]+]]:_(<4 x s16>) = G_INSERT_VECTOR_ELT [[UV]], [[C2]](s16), [[C1]](s64)
+  ; CHECK-NEXT:   [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[IVEC]](<4 x s16>)
+  ; CHECK-NEXT:   [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[UV2]](s16)
+  ; CHECK-NEXT:   [[TRUNC4:%[0-9]+]]:_(s8) = G_TRUNC [[UV3]](s16)
+  ; CHECK-NEXT:   [[TRUNC5:%[0-9]+]]:_(s8) = G_TRUNC [[UV4]](s16)
   ; CHECK-NEXT:   [[BUILD_VECTOR1:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[TRUNC3]](s8), [[TRUNC4]](s8), [[TRUNC5]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8)
   ; CHECK-NEXT:   [[BUILD_VECTOR2:%[0-9]+]]:_(<16 x s8>) = G_BUILD_VECTOR [[C]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8)
   ; CHECK-NEXT:   [[SHUF:%[0-9]+]]:_(<16 x s8>) = G_SHUFFLE_VECTOR [[BUILD_VECTOR1]](<16 x s8>), [[BUILD_VECTOR2]], shufflemask(0, 16, 16, 16, 1, 16, 16, 16, 2, 16, 16, 16, undef, undef, undef, undef)
   ; CHECK-NEXT:   [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[SHUF]](<16 x s8>)
   ; CHECK-NEXT:   [[UITOFP:%[0-9]+]]:_(<4 x s32>) = G_UITOFP [[BITCAST]](<4 x s32>)
-  ; CHECK-NEXT:   [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UITOFP]](<4 x s32>)
-  ; CHECK-NEXT:   G_STORE [[UV4]](s32), [[COPY]](p0) :: (store (s32), align 16)
+  ; CHECK-NEXT:   [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[UITOFP]](<4 x s32>)
+  ; CHECK-NEXT:   G_STORE [[UV6]](s32), [[COPY]](p0) :: (store (s32), align 16)
   ; CHECK-NEXT:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
   ; CHECK-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C3]](s64)
-  ; CHECK-NEXT:   G_STORE [[UV5]](s32), [[PTR_ADD]](p0) :: (store (s32) into unknown-address + 4)
+  ; CHECK-NEXT:   G_STORE [[UV7]](s32), [[PTR_ADD]](p0) :: (store (s32) into unknown-address + 4)
   ; CHECK-NEXT:   [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
   ; CHECK-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C4]](s64)
-  ; CHECK-NEXT:   G_STORE [[UV6]](s32), [[PTR_ADD1]](p0) :: (store (s32) into unknown-address + 8, align 8)
+  ; CHECK-NEXT:   G_STORE [[UV8]](s32), [[PTR_ADD1]](p0) :: (store (s32) into unknown-address + 8, align 8)
   ; CHECK-NEXT:   G_BR %bb.1
   bb.1:
     liveins: $w1, $w2, $w3, $x0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-select.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-select.mir
index 63a26dcfea47..e49a94c12ed4 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-select.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-select.mir
@@ -293,41 +293,44 @@ body:             |
     ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), %w0(s32), [[C]]
     ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[ICMP2]], 1
     ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
-    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s16) = COPY [[DEF1]](s16)
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s16) = COPY [[DEF1]](s16)
-    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s16) = COPY [[DEF1]](s16)
-    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[COPY]](s16), [[COPY1]](s16), [[COPY2]](s16), [[DEF1]](s16)
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8)
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(<8 x s16>) = G_ANYEXT [[BUILD_VECTOR]](<8 x s8>)
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(<4 x s16>), [[UV1:%[0-9]+]]:_(<4 x s16>) = G_UNMERGE_VALUES [[ANYEXT]](<8 x s16>)
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[SEXT_INREG]](s32)
-    ; CHECK-NEXT: [[IVEC:%[0-9]+]]:_(<4 x s16>) = G_INSERT_VECTOR_ELT [[BUILD_VECTOR]], [[TRUNC]](s16), [[C1]](s64)
-    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[IVEC]](<4 x s16>)
-    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[UV]](s16)
-    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[UV1]](s16)
-    ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[UV2]](s16)
-    ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s8) = G_TRUNC [[UV3]](s16)
-    ; CHECK-NEXT: [[DEF2:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
-    ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<8 x s8>) = G_BUILD_VECTOR [[TRUNC1]](s8), [[TRUNC2]](s8), [[TRUNC3]](s8), [[TRUNC4]](s8), [[DEF2]](s8), [[DEF2]](s8), [[DEF2]](s8), [[DEF2]](s8)
-    ; CHECK-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<8 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF2]](s8), [[DEF2]](s8), [[DEF2]](s8), [[DEF2]](s8)
+    ; CHECK-NEXT: [[IVEC:%[0-9]+]]:_(<4 x s16>) = G_INSERT_VECTOR_ELT [[UV]], [[TRUNC]](s16), [[C1]](s64)
+    ; CHECK-NEXT: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[IVEC]](<4 x s16>)
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[UV2]](s16)
+    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[UV3]](s16)
+    ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[UV4]](s16)
+    ; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s8) = G_TRUNC [[UV5]](s16)
+    ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<8 x s8>) = G_BUILD_VECTOR [[TRUNC1]](s8), [[TRUNC2]](s8), [[TRUNC3]](s8), [[TRUNC4]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8)
+    ; CHECK-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<8 x s8>) = G_BUILD_VECTOR [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8)
     ; CHECK-NEXT: [[SHUF:%[0-9]+]]:_(<8 x s8>) = G_SHUFFLE_VECTOR [[BUILD_VECTOR1]](<8 x s8>), [[BUILD_VECTOR2]], shufflemask(0, 0, 0, 0, undef, undef, undef, undef)
-    ; CHECK-NEXT: [[UV4:%[0-9]+]]:_(<4 x s8>), [[UV5:%[0-9]+]]:_(<4 x s8>) = G_UNMERGE_VALUES [[SHUF]](<8 x s8>)
-    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
-    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s16) = COPY [[C2]](s16)
-    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s16) = COPY [[C2]](s16)
-    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s16) = COPY [[C2]](s16)
-    ; CHECK-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[COPY3]](s16), [[COPY4]](s16), [[COPY5]](s16), [[C2]](s16)
-    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[UV4]](<4 x s8>)
-    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<4 x s16>) = G_XOR [[ANYEXT]], [[BUILD_VECTOR3]]
+    ; CHECK-NEXT: [[UV6:%[0-9]+]]:_(<4 x s8>), [[UV7:%[0-9]+]]:_(<4 x s8>) = G_UNMERGE_VALUES [[SHUF]](<8 x s8>)
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s8) = G_CONSTANT i8 1
+    ; CHECK-NEXT: [[UV8:%[0-9]+]]:_(s8), [[UV9:%[0-9]+]]:_(s8), [[UV10:%[0-9]+]]:_(s8), [[UV11:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[UV6]](<4 x s8>)
+    ; CHECK-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<8 x s8>) = G_BUILD_VECTOR [[UV8]](s8), [[UV9]](s8), [[UV10]](s8), [[UV11]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8)
+    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(<8 x s16>) = G_ANYEXT [[BUILD_VECTOR3]](<8 x s8>)
+    ; CHECK-NEXT: [[UV12:%[0-9]+]]:_(<4 x s16>), [[UV13:%[0-9]+]]:_(<4 x s16>) = G_UNMERGE_VALUES [[ANYEXT1]](<8 x s16>)
+    ; CHECK-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<8 x s8>) = G_BUILD_VECTOR [[C2]](s8), [[C2]](s8), [[C2]](s8), [[C2]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8)
+    ; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(<8 x s16>) = G_ANYEXT [[BUILD_VECTOR4]](<8 x s8>)
+    ; CHECK-NEXT: [[UV14:%[0-9]+]]:_(<4 x s16>), [[UV15:%[0-9]+]]:_(<4 x s16>) = G_UNMERGE_VALUES [[ANYEXT2]](<8 x s16>)
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<4 x s16>) = G_XOR [[UV12]], [[UV14]]
     ; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(<4 x s16>) = G_TRUNC [[ICMP]](<4 x s32>)
-    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[UV4]](<4 x s8>)
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(<4 x s16>) = G_AND [[TRUNC5]], [[ANYEXT1]]
+    ; CHECK-NEXT: [[UV16:%[0-9]+]]:_(s8), [[UV17:%[0-9]+]]:_(s8), [[UV18:%[0-9]+]]:_(s8), [[UV19:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[UV6]](<4 x s8>)
+    ; CHECK-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<8 x s8>) = G_BUILD_VECTOR [[UV16]](s8), [[UV17]](s8), [[UV18]](s8), [[UV19]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8), [[DEF1]](s8)
+    ; CHECK-NEXT: [[ANYEXT3:%[0-9]+]]:_(<8 x s16>) = G_ANYEXT [[BUILD_VECTOR5]](<8 x s8>)
+    ; CHECK-NEXT: [[UV20:%[0-9]+]]:_(<4 x s16>), [[UV21:%[0-9]+]]:_(<4 x s16>) = G_UNMERGE_VALUES [[ANYEXT3]](<8 x s16>)
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(<4 x s16>) = G_AND [[TRUNC5]], [[UV20]]
     ; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(<4 x s16>) = G_TRUNC [[ICMP1]](<4 x s32>)
     ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(<4 x s16>) = G_AND [[TRUNC6]], [[XOR]]
     ; CHECK-NEXT: [[OR:%[0-9]+]]:_(<4 x s16>) = G_OR [[AND]], [[AND1]]
-    ; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(<4 x s32>) = G_ANYEXT [[OR]](<4 x s16>)
+    ; CHECK-NEXT: [[ANYEXT4:%[0-9]+]]:_(<4 x s32>) = G_ANYEXT [[OR]](<4 x s16>)
     ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; CHECK-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[C3]](s32), [[C3]](s32), [[C3]](s32), [[C3]](s32)
-    ; CHECK-NEXT: %zext_select:_(<4 x s32>) = G_AND [[ANYEXT2]], [[BUILD_VECTOR4]]
+    ; CHECK-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[C3]](s32), [[C3]](s32), [[C3]](s32), [[C3]](s32)
+    ; CHECK-NEXT: %zext_select:_(<4 x s32>) = G_AND [[ANYEXT4]], [[BUILD_VECTOR6]]
     ; CHECK-NEXT: $q0 = COPY %zext_select(<4 x s32>)
     ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %w0:_(s32) = COPY $w0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector-widen-crash.ll b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector-widen-crash.ll
index 42a8f51002f2..f7efaeaa5070 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector-widen-crash.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-shuffle-vector-widen-crash.ll
@@ -1,16 +1,24 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -global-isel-abort=2 -global-isel -o - %s | FileCheck %s
+; RUN: llc -global-isel -o - %s | FileCheck %s
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 target triple = "arm64-apple-macosx11.0.0"
 
 declare i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32>) #0
 
-; This test currently falls back but ensures we don't crash.
-
 define i32 @bar() {
 ; CHECK-LABEL: bar:
 ; CHECK:       ; %bb.0: ; %bb
 ; CHECK-NEXT:    movi.2d v0, #0000000000000000
+; CHECK-NEXT:    mov b1, v0[1]
+; CHECK-NEXT:    mov b2, v0[2]
+; CHECK-NEXT:    mov b3, v0[3]
+; CHECK-NEXT:    mov.h v0[1], v1[0]
+; CHECK-NEXT:    mov.h v2[1], v3[0]
+; CHECK-NEXT:    ushll.4s v0, v0, #0
+; CHECK-NEXT:    ushll.4s v1, v2, #0
+; CHECK-NEXT:    mov.d v0[1], v1[0]
+; CHECK-NEXT:    movi.4s v1, #1
+; CHECK-NEXT:    and.16b v0, v0, v1
 ; CHECK-NEXT:    addv.4s s0, v0
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-xtn.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-xtn.mir
index ed40a2ff7ea7..e729f027baa7 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-xtn.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-xtn.mir
@@ -541,17 +541,13 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
     ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
-    ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32), [[DEF]](s32), [[DEF]](s32)
-    ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF]](s32), [[DEF]](s32), [[DEF]](s32)
-    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(<4 x s16>) = G_TRUNC [[BUILD_VECTOR]](<4 x s32>)
-    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(<4 x s16>) = G_TRUNC [[BUILD_VECTOR1]](<4 x s32>)
-    ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[TRUNC]](<4 x s16>), [[TRUNC1]](<4 x s16>)
-    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(<8 x s8>) = G_TRUNC [[CONCAT_VECTORS]](<8 x s16>)
-    ; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s8>), [[UV3:%[0-9]+]]:_(<2 x s8>), [[UV4:%[0-9]+]]:_(<2 x s8>), [[UV5:%[0-9]+]]:_(<2 x s8>) = G_UNMERGE_VALUES [[TRUNC2]](<8 x s8>)
-    ; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s8>) = G_CONCAT_VECTORS [[UV2]](<2 x s8>), [[UV2]](<2 x s8>)
-    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[CONCAT_VECTORS1]](<4 x s8>)
-    ; CHECK-NEXT: $d0 = COPY [[ANYEXT]](<4 x s16>)
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[UV]](s32)
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[UV1]](s32)
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s8>) = G_BUILD_VECTOR [[TRUNC]](s8), [[TRUNC1]](s8), [[TRUNC]](s8), [[TRUNC1]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8), [[DEF]](s8)
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(<8 x s16>) = G_ANYEXT [[BUILD_VECTOR]](<8 x s8>)
+    ; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<4 x s16>), [[UV3:%[0-9]+]]:_(<4 x s16>) = G_UNMERGE_VALUES [[ANYEXT]](<8 x s16>)
+    ; CHECK-NEXT: $d0 = COPY [[UV2]](<4 x s16>)
     ; CHECK-NEXT: RET_ReallyLR implicit $d0
     %0:_(<2 x s32>) = COPY $d0
     %1:_(<2 x s8>) = G_TRUNC %0(<2 x s32>)
diff --git a/llvm/test/CodeGen/AArch64/aarch64-smull.ll b/llvm/test/CodeGen/AArch64/aarch64-smull.ll
index dbc5417e2313..61a4f64ac2bf 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-smull.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-smull.ll
@@ -3,8 +3,7 @@
 ; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+sve < %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SVE
 ; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
-; CHECK-GI:       warning: Instruction selection used fallback path for smull_zext_v4i16_v4i32
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for pmlsl2_v8i16_uzp1
+; CHECK-GI:       warning: Instruction selection used fallback path for pmlsl2_v8i16_uzp1
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for smlsl2_v8i16_uzp1
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for umlsl2_v8i16_uzp1
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for smlsl2_v4i32_uzp1
@@ -189,13 +188,49 @@ define <8 x i32> @smull_zext_v8i8_v8i32_top_bit_is_1(ptr %A, ptr %B) nounwind {
 }
 
 define <4 x i32> @smull_zext_v4i16_v4i32(ptr %A, ptr %B) nounwind {
-; CHECK-LABEL: smull_zext_v4i16_v4i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr s0, [x0]
-; CHECK-NEXT:    ldr d1, [x1]
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    smull v0.4s, v0.4h, v1.4h
-; CHECK-NEXT:    ret
+; CHECK-NEON-LABEL: smull_zext_v4i16_v4i32:
+; CHECK-NEON:       // %bb.0:
+; CHECK-NEON-NEXT:    ldr s0, [x0]
+; CHECK-NEON-NEXT:    ldr d1, [x1]
+; CHECK-NEON-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-NEON-NEXT:    smull v0.4s, v0.4h, v1.4h
+; CHECK-NEON-NEXT:    ret
+;
+; CHECK-SVE-LABEL: smull_zext_v4i16_v4i32:
+; CHECK-SVE:       // %bb.0:
+; CHECK-SVE-NEXT:    ldr s0, [x0]
+; CHECK-SVE-NEXT:    ldr d1, [x1]
+; CHECK-SVE-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SVE-NEXT:    smull v0.4s, v0.4h, v1.4h
+; CHECK-SVE-NEXT:    ret
+;
+; CHECK-GI-LABEL: smull_zext_v4i16_v4i32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    uxtb w8, w8
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov b2, v0.b[2]
+; CHECK-GI-NEXT:    mov b3, v0.b[3]
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    uxtb w9, w9
+; CHECK-GI-NEXT:    uxtb w10, w10
+; CHECK-GI-NEXT:    uxtb w11, w11
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    fmov s2, w10
+; CHECK-GI-NEXT:    fmov s3, w11
+; CHECK-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NEXT:    mov v2.h[1], v3.h[0]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v1.4s, v2.4h, #0
+; CHECK-GI-NEXT:    ldr d2, [x1]
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT:    sshll v1.4s, v2.4h, #0
+; CHECK-GI-NEXT:    mul v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    ret
   %load.A = load <4 x i8>, ptr %A
   %load.B = load <4 x i16>, ptr %B
   %zext.A = zext <4 x i8> %load.A to <4 x i32>
diff --git a/llvm/test/CodeGen/AArch64/arm64-extract-insert-varidx.ll b/llvm/test/CodeGen/AArch64/arm64-extract-insert-varidx.ll
index a1e069368527..bc399c8d4ff0 100644
--- a/llvm/test/CodeGen/AArch64/arm64-extract-insert-varidx.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-extract-insert-varidx.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
 ; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -mattr=+neon -fp-contract=fast -aarch64-enable-sink-fold=true | FileCheck %s --check-prefix=CHECK-SDAG
-; RUN: llc < %s -global-isel -global-isel-abort=2 -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -mattr=+neon -fp-contract=fast -aarch64-enable-sink-fold=true | FileCheck %s --check-prefix=CHECK-GISEL
+; RUN: llc < %s -global-isel -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -mattr=+neon -fp-contract=fast -aarch64-enable-sink-fold=true | FileCheck %s --check-prefix=CHECK-GISEL
 
 define <4 x i8> @test_varidx_extract_v8s8(<8 x i8> %x, i32 %idx) {
 ; CHECK-SDAG-LABEL: test_varidx_extract_v8s8:
@@ -29,20 +29,20 @@ define <4 x i8> @test_varidx_extract_v8s8(<8 x i8> %x, i32 %idx) {
 ; CHECK-GISEL-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-GISEL-NEXT:    mov w9, w0
 ; CHECK-GISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GISEL-NEXT:    mov b1, v0.b[1]
 ; CHECK-GISEL-NEXT:    add x8, sp, #8
-; CHECK-GISEL-NEXT:    str d0, [sp, #8]
 ; CHECK-GISEL-NEXT:    and x9, x9, #0x7
-; CHECK-GISEL-NEXT:    mov b2, v0.b[1]
+; CHECK-GISEL-NEXT:    str d0, [sp, #8]
 ; CHECK-GISEL-NEXT:    mov b3, v0.b[2]
 ; CHECK-GISEL-NEXT:    lsl x10, x9, #1
 ; CHECK-GISEL-NEXT:    mov b0, v0.b[3]
 ; CHECK-GISEL-NEXT:    sub x9, x10, x9
-; CHECK-GISEL-NEXT:    ldrb w8, [x8, x9]
-; CHECK-GISEL-NEXT:    fmov s1, w8
-; CHECK-GISEL-NEXT:    mov v1.h[1], v2.h[0]
-; CHECK-GISEL-NEXT:    mov v1.h[2], v3.h[0]
-; CHECK-GISEL-NEXT:    mov v1.h[3], v0.h[0]
-; CHECK-GISEL-NEXT:    fmov d0, d1
+; CHECK-GISEL-NEXT:    ldr b2, [x8, x9]
+; CHECK-GISEL-NEXT:    mov v2.b[1], v1.b[0]
+; CHECK-GISEL-NEXT:    mov v2.b[2], v3.b[0]
+; CHECK-GISEL-NEXT:    mov v2.b[3], v0.b[0]
+; CHECK-GISEL-NEXT:    ushll v0.8h, v2.8b, #0
+; CHECK-GISEL-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GISEL-NEXT:    add sp, sp, #16
 ; CHECK-GISEL-NEXT:    ret
   %tmp = extractelement <8 x i8> %x, i32 %idx
@@ -176,17 +176,15 @@ define <2 x i16> @test_varidx_extract_v4s16(<4 x i16> %x, i32 %idx) {
 ; CHECK-GISEL:       // %bb.0:
 ; CHECK-GISEL-NEXT:    sub sp, sp, #16
 ; CHECK-GISEL-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-GISEL-NEXT:    mov w9, w0
-; CHECK-GISEL-NEXT:    mov w8, #2 // =0x2
-; CHECK-GISEL-NEXT:    add x10, sp, #8
-; CHECK-GISEL-NEXT:    and x9, x9, #0x3
 ; CHECK-GISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GISEL-NEXT:    mov w9, w0
+; CHECK-GISEL-NEXT:    mov h1, v0.h[1]
+; CHECK-GISEL-NEXT:    add x8, sp, #8
 ; CHECK-GISEL-NEXT:    str d0, [sp, #8]
-; CHECK-GISEL-NEXT:    madd x8, x9, x8, x10
-; CHECK-GISEL-NEXT:    umov w9, v0.h[1]
-; CHECK-GISEL-NEXT:    fmov s1, w9
-; CHECK-GISEL-NEXT:    ldr h0, [x8]
-; CHECK-GISEL-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GISEL-NEXT:    and x9, x9, #0x3
+; CHECK-GISEL-NEXT:    ldr h0, [x8, x9, lsl #1]
+; CHECK-GISEL-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GISEL-NEXT:    ushll v0.4s, v0.4h, #0
 ; CHECK-GISEL-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GISEL-NEXT:    add sp, sp, #16
 ; CHECK-GISEL-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/bitcast.ll b/llvm/test/CodeGen/AArch64/bitcast.ll
index b7c02d6855a9..e0851fd8739e 100644
--- a/llvm/test/CodeGen/AArch64/bitcast.ll
+++ b/llvm/test/CodeGen/AArch64/bitcast.ll
@@ -1,14 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 ; PR23065: SCALAR_TO_VECTOR implies the top elements 1 to N-1 of the N-element vector are undefined.
 
-; CHECK-GI:       warning: Instruction selection used fallback path for bitcast_i32_v4i8
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for bitcast_i32_v2i16
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for bitcast_v2i16_v4i8
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for bitcast_v4i8_v2i16
-
 define <4 x i16> @foo1(<2 x i32> %a) {
 ; CHECK-SD-LABEL: foo1:
 ; CHECK-SD:       // %bb.0:
@@ -80,12 +75,26 @@ define i32 @bitcast_v4i8_i32(<4 x i8> %a, <4 x i8> %b){
 }
 
 define <4 x i8> @bitcast_i32_v4i8(i32 %a, i32 %b){
-; CHECK-LABEL: bitcast_i32_v4i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add w8, w0, w1
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    zip1 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: bitcast_i32_v4i8:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    add w8, w0, w1
+; CHECK-SD-NEXT:    fmov s0, w8
+; CHECK-SD-NEXT:    zip1 v0.8b, v0.8b, v0.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: bitcast_i32_v4i8:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    add w8, w0, w1
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov b2, v0.b[2]
+; CHECK-GI-NEXT:    mov b3, v0.b[3]
+; CHECK-GI-NEXT:    mov v0.b[1], v1.b[0]
+; CHECK-GI-NEXT:    mov v0.b[2], v2.b[0]
+; CHECK-GI-NEXT:    mov v0.b[3], v3.b[0]
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
   %c = add i32 %a, %b
   %d = bitcast i32 %c to <4 x i8>
   ret <4 x i8> %d
@@ -119,13 +128,23 @@ define i32 @bitcast_v2i16_i32(<2 x i16> %a, <2 x i16> %b){
 }
 
 define <2 x i16> @bitcast_i32_v2i16(i32 %a, i32 %b){
-; CHECK-LABEL: bitcast_i32_v2i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add w8, w0, w1
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: bitcast_i32_v2i16:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    add w8, w0, w1
+; CHECK-SD-NEXT:    fmov s0, w8
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: bitcast_i32_v2i16:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    add w8, w0, w1
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
   %c = add i32 %a, %b
   %d = bitcast i32 %c to <2 x i16>
   ret <2 x i16> %d
@@ -382,40 +401,72 @@ define <8 x i16> @bitcast_v16i8_v8i16(<16 x i8> %a, <16 x i8> %b){
 ; ===== Smaller/Larger Width Vectors with Legal Element Sizes =====
 
 define <4 x i8> @bitcast_v2i16_v4i8(<2 x i16> %a, <2 x i16> %b){
-; CHECK-LABEL: bitcast_v2i16_v4i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #16
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    add v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    mov w8, v0.s[1]
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    strh w9, [sp, #12]
-; CHECK-NEXT:    strh w8, [sp, #14]
-; CHECK-NEXT:    ldr s0, [sp, #12]
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: bitcast_v2i16_v4i8:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sub sp, sp, #16
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT:    add v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-SD-NEXT:    fmov w9, s0
+; CHECK-SD-NEXT:    strh w9, [sp, #12]
+; CHECK-SD-NEXT:    strh w8, [sp, #14]
+; CHECK-SD-NEXT:    ldr s0, [sp, #12]
+; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    add sp, sp, #16
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: bitcast_v2i16_v4i8:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    add v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov b2, v0.b[2]
+; CHECK-GI-NEXT:    mov b3, v0.b[3]
+; CHECK-GI-NEXT:    mov v0.b[1], v1.b[0]
+; CHECK-GI-NEXT:    mov v0.b[2], v2.b[0]
+; CHECK-GI-NEXT:    mov v0.b[3], v3.b[0]
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
   %c = add <2 x i16> %a, %b
   %d = bitcast <2 x i16> %c to <4 x i8>
   ret <4 x i8> %d
 }
 
 define <2 x i16> @bitcast_v4i8_v2i16(<4 x i8> %a, <4 x i8> %b){
-; CHECK-LABEL: bitcast_v4i8_v2i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #16
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    add v0.4h, v0.4h, v1.4h
-; CHECK-NEXT:    add x8, sp, #12
-; CHECK-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
-; CHECK-NEXT:    str s0, [sp, #12]
-; CHECK-NEXT:    ld1 { v0.h }[0], [x8]
-; CHECK-NEXT:    orr x8, x8, #0x2
-; CHECK-NEXT:    ld1 { v0.h }[2], [x8]
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: bitcast_v4i8_v2i16:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sub sp, sp, #16
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT:    add v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT:    add x8, sp, #12
+; CHECK-SD-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-SD-NEXT:    str s0, [sp, #12]
+; CHECK-SD-NEXT:    ld1 { v0.h }[0], [x8]
+; CHECK-SD-NEXT:    orr x8, x8, #0x2
+; CHECK-SD-NEXT:    ld1 { v0.h }[2], [x8]
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    add sp, sp, #16
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: bitcast_v4i8_v2i16:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    add v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    mov h3, v0.h[3]
+; CHECK-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NEXT:    mov v0.h[2], v2.h[0]
+; CHECK-GI-NEXT:    mov v0.h[3], v3.h[0]
+; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
   %c = add <4 x i8> %a, %b
   %d = bitcast <4 x i8> %c to <2 x i16>
   ret <2 x i16> %d
diff --git a/llvm/test/CodeGen/AArch64/bswap.ll b/llvm/test/CodeGen/AArch64/bswap.ll
index f4221accfcbc..071613b9cc01 100644
--- a/llvm/test/CodeGen/AArch64/bswap.ll
+++ b/llvm/test/CodeGen/AArch64/bswap.ll
@@ -1,8 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -mtriple=aarch64 %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-
-; CHECK-GI:         warning: Instruction selection used fallback path for bswap_v2i16
+; RUN: llc -mtriple=aarch64 -global-isel %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 ; ====== Scalar Tests =====
 define i16 @bswap_i16(i16 %a){
@@ -103,11 +101,23 @@ declare <2 x i64> @llvm.bswap.v2i64(<2 x i64>)
 ; ===== Smaller/Larger Width Vectors with Legal Element Sizes =====
 
 define <2 x i16> @bswap_v2i16(<2 x i16> %a){
-; CHECK-LABEL: bswap_v2i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rev32 v0.8b, v0.8b
-; CHECK-NEXT:    ushr v0.2s, v0.2s, #16
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: bswap_v2i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    rev32 v0.8b, v0.8b
+; CHECK-SD-NEXT:    ushr v0.2s, v0.2s, #16
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: bswap_v2i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NEXT:    rev16 v0.8b, v0.8b
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
 entry:
   %res = call <2 x i16> @llvm.bswap.v2i16(<2 x i16> %a)
   ret <2 x i16> %res
diff --git a/llvm/test/CodeGen/AArch64/fptoi.ll b/llvm/test/CodeGen/AArch64/fptoi.ll
index 7af01b53dae7..01585d02adcb 100644
--- a/llvm/test/CodeGen/AArch64/fptoi.ll
+++ b/llvm/test/CodeGen/AArch64/fptoi.ll
@@ -1,13 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16
 ; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16
-; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16
-; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16
-
-; CHECK-GI-FP16:       warning: Instruction selection used fallback path for fptos_v2f16_v2i16
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for fptou_v2f16_v2i16
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for fptos_v2f16_v2i8
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for fptou_v2f16_v2i8
+; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16
+; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16
 
 define i64 @fptos_f64_i64(double %a) {
 ; CHECK-LABEL: fptos_f64_i64:
@@ -5184,8 +5179,13 @@ define <2 x i16> @fptos_v2f16_v2i16(<2 x half> %a) {
 ;
 ; CHECK-GI-FP16-LABEL: fptos_v2f16_v2i16:
 ; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-FP16-NEXT:    ushll v0.4s, v0.4h, #0
 ; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-FP16-NEXT:    ret
 entry:
@@ -5212,8 +5212,13 @@ define <2 x i16> @fptou_v2f16_v2i16(<2 x half> %a) {
 ;
 ; CHECK-GI-FP16-LABEL: fptou_v2f16_v2i16:
 ; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-FP16-NEXT:    fcvtzu v0.4h, v0.4h
+; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-FP16-NEXT:    ushll v0.4s, v0.4h, #0
 ; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-FP16-NEXT:    ret
 entry:
@@ -5658,8 +5663,13 @@ define <2 x i8> @fptos_v2f16_v2i8(<2 x half> %a) {
 ;
 ; CHECK-GI-FP16-LABEL: fptos_v2f16_v2i8:
 ; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-FP16-NEXT:    ushll v0.4s, v0.4h, #0
 ; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-FP16-NEXT:    ret
 entry:
@@ -5686,8 +5696,13 @@ define <2 x i8> @fptou_v2f16_v2i8(<2 x half> %a) {
 ;
 ; CHECK-GI-FP16-LABEL: fptou_v2f16_v2i8:
 ; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-FP16-NEXT:    fcvtzu v0.4h, v0.4h
+; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-FP16-NEXT:    ushll v0.4s, v0.4h, #0
 ; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-FP16-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/AArch64/itofp.ll b/llvm/test/CodeGen/AArch64/itofp.ll
index ceac37f91238..f5a7b5dc9f49 100644
--- a/llvm/test/CodeGen/AArch64/itofp.ll
+++ b/llvm/test/CodeGen/AArch64/itofp.ll
@@ -4,13 +4,6 @@
 ; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16
 ; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16
 
-; CHECK-GI:       warning: Instruction selection used fallback path for stofp_v3i8_v3f64
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v3i8_v3f64
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for stofp_v3i8_v3f32
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for utofp_v3i8_v3f32
-; CHECK-GI-NOFP16-NEXT:  warning: Instruction selection used fallback path for stofp_v3i8_v3f16
-; CHECK-GI-NOFP16-NEXT:  warning: Instruction selection used fallback path for utofp_v3i8_v3f16
-
 define double @stofp_i64_f64(i64 %a) {
 ; CHECK-LABEL: stofp_i64_f64:
 ; CHECK:       // %bb.0: // %entry
@@ -1754,47 +1747,109 @@ entry:
 }
 
 define <3 x double> @stofp_v3i8_v3f64(<3 x i8> %a) {
-; CHECK-LABEL: stofp_v3i8_v3f64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmov s0, w0
-; CHECK-NEXT:    fmov s1, w2
-; CHECK-NEXT:    mov v0.s[1], w1
-; CHECK-NEXT:    shl v1.2s, v1.2s, #24
-; CHECK-NEXT:    sshr v1.2s, v1.2s, #24
-; CHECK-NEXT:    shl v0.2s, v0.2s, #24
-; CHECK-NEXT:    sshll v1.2d, v1.2s, #0
-; CHECK-NEXT:    sshr v0.2s, v0.2s, #24
-; CHECK-NEXT:    scvtf v2.2d, v1.2d
-; CHECK-NEXT:    sshll v0.2d, v0.2s, #0
-; CHECK-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: stofp_v3i8_v3f64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    fmov s1, w2
+; CHECK-SD-NEXT:    mov v0.s[1], w1
+; CHECK-SD-NEXT:    shl v1.2s, v1.2s, #24
+; CHECK-SD-NEXT:    sshr v1.2s, v1.2s, #24
+; CHECK-SD-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-SD-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-SD-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-SD-NEXT:    scvtf v2.2d, v1.2d
+; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: stofp_v3i8_v3f64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov s0, w0
+; CHECK-GI-NEXT:    fmov s1, w1
+; CHECK-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NEXT:    fmov s1, w2
+; CHECK-GI-NEXT:    mov v0.h[2], v1.h[0]
+; CHECK-GI-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-GI-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-GI-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov v0.h[1], v2.h[0]
+; CHECK-GI-NEXT:    mov h2, v1.h[1]
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov v1.h[1], v2.h[0]
+; CHECK-GI-NEXT:    smov x8, v0.s[0]
+; CHECK-GI-NEXT:    smov x9, v0.s[1]
+; CHECK-GI-NEXT:    sshll v0.4s, v1.4h, #0
+; CHECK-GI-NEXT:    fmov d1, x8
+; CHECK-GI-NEXT:    smov x8, v0.s[0]
+; CHECK-GI-NEXT:    mov v1.d[1], x9
+; CHECK-GI-NEXT:    smov x9, v0.s[1]
+; CHECK-GI-NEXT:    fmov d2, x8
+; CHECK-GI-NEXT:    scvtf v0.2d, v1.2d
+; CHECK-GI-NEXT:    mov v2.d[1], x9
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-GI-NEXT:    ret
 entry:
   %c = sitofp <3 x i8> %a to <3 x double>
   ret <3 x double> %c
 }
 
 define <3 x double> @utofp_v3i8_v3f64(<3 x i8> %a) {
-; CHECK-LABEL: utofp_v3i8_v3f64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmov s0, w0
-; CHECK-NEXT:    movi d1, #0x0000ff000000ff
-; CHECK-NEXT:    fmov s2, w2
-; CHECK-NEXT:    mov v0.s[1], w1
-; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    and v1.8b, v2.8b, v1.8b
-; CHECK-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-NEXT:    ushll v1.2d, v1.2s, #0
-; CHECK-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-NEXT:    ucvtf v2.2d, v1.2d
-; CHECK-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: utofp_v3i8_v3f64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-SD-NEXT:    fmov s2, w2
+; CHECK-SD-NEXT:    mov v0.s[1], w1
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    and v1.8b, v2.8b, v1.8b
+; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-SD-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-SD-NEXT:    ucvtf v2.2d, v1.2d
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: utofp_v3i8_v3f64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov s0, w0
+; CHECK-GI-NEXT:    fmov s1, w1
+; CHECK-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NEXT:    fmov s1, w2
+; CHECK-GI-NEXT:    mov v0.h[2], v1.h[0]
+; CHECK-GI-NEXT:    movi d1, #0xff00ff00ff00ff
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov v0.h[1], v2.h[0]
+; CHECK-GI-NEXT:    mov h2, v1.h[1]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov v1.h[1], v2.h[0]
+; CHECK-GI-NEXT:    mov w8, v0.s[0]
+; CHECK-GI-NEXT:    mov w9, v0.s[1]
+; CHECK-GI-NEXT:    ushll v0.4s, v1.4h, #0
+; CHECK-GI-NEXT:    fmov d1, x8
+; CHECK-GI-NEXT:    mov w8, v0.s[0]
+; CHECK-GI-NEXT:    mov v1.d[1], x9
+; CHECK-GI-NEXT:    mov w9, v0.s[1]
+; CHECK-GI-NEXT:    fmov d2, x8
+; CHECK-GI-NEXT:    ucvtf v0.2d, v1.2d
+; CHECK-GI-NEXT:    mov v2.d[1], x9
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-GI-NEXT:    ret
 entry:
   %c = uitofp <3 x i8> %a to <3 x double>
   ret <3 x double> %c
@@ -3372,31 +3427,71 @@ entry:
 }
 
 define <3 x float> @stofp_v3i8_v3f32(<3 x i8> %a) {
-; CHECK-LABEL: stofp_v3i8_v3f32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmov s0, w0
-; CHECK-NEXT:    mov v0.h[1], w1
-; CHECK-NEXT:    mov v0.h[2], w2
-; CHECK-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-NEXT:    sshr v0.4h, v0.4h, #8
-; CHECK-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: stofp_v3i8_v3f32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    mov v0.h[1], w1
+; CHECK-SD-NEXT:    mov v0.h[2], w2
+; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-SD-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: stofp_v3i8_v3f32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov s0, w0
+; CHECK-GI-NEXT:    fmov s1, w1
+; CHECK-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NEXT:    fmov s1, w2
+; CHECK-GI-NEXT:    mov v0.h[2], v1.h[0]
+; CHECK-GI-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-GI-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-NEXT:    mov h3, v1.h[1]
+; CHECK-GI-NEXT:    mov v0.h[1], v2.h[0]
+; CHECK-GI-NEXT:    mov v1.h[1], v3.h[0]
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %c = sitofp <3 x i8> %a to <3 x float>
   ret <3 x float> %c
 }
 
 define <3 x float> @utofp_v3i8_v3f32(<3 x i8> %a) {
-; CHECK-LABEL: utofp_v3i8_v3f32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fmov s0, w0
-; CHECK-NEXT:    mov v0.h[1], w1
-; CHECK-NEXT:    mov v0.h[2], w2
-; CHECK-NEXT:    bic v0.4h, #255, lsl #8
-; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: utofp_v3i8_v3f32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    mov v0.h[1], w1
+; CHECK-SD-NEXT:    mov v0.h[2], w2
+; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-SD-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: utofp_v3i8_v3f32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov s0, w0
+; CHECK-GI-NEXT:    fmov s1, w1
+; CHECK-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NEXT:    fmov s1, w2
+; CHECK-GI-NEXT:    mov v0.h[2], v1.h[0]
+; CHECK-GI-NEXT:    movi d1, #0xff00ff00ff00ff
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-NEXT:    mov h3, v1.h[1]
+; CHECK-GI-NEXT:    mov v0.h[1], v2.h[0]
+; CHECK-GI-NEXT:    mov v1.h[1], v3.h[0]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %c = uitofp <3 x i8> %a to <3 x float>
   ret <3 x float> %c
@@ -5582,12 +5677,18 @@ define <2 x half> @utofp_v2i8_v2f16(<2 x i8> %a) {
 ; CHECK-GI-FP16-LABEL: utofp_v2i8_v2f16:
 ; CHECK-GI-FP16:       // %bb.0: // %entry
 ; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    mov w8, v0.s[1]
-; CHECK-GI-FP16-NEXT:    fmov w9, s0
-; CHECK-GI-FP16-NEXT:    and w9, w9, #0xff
-; CHECK-GI-FP16-NEXT:    and w8, w8, #0xff
-; CHECK-GI-FP16-NEXT:    ucvtf h0, w9
-; CHECK-GI-FP16-NEXT:    ucvtf h1, w8
+; CHECK-GI-FP16-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-FP16-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-FP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-FP16-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-GI-FP16-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-FP16-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-GI-FP16-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-FP16-NEXT:    ucvtf v0.4h, v0.4h
+; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
 ; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
 ; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-FP16-NEXT:    ret
@@ -5622,11 +5723,20 @@ define <3 x half> @stofp_v3i8_v3f16(<3 x i8> %a) {
 ; CHECK-GI-NOFP16-LABEL: stofp_v3i8_v3f16:
 ; CHECK-GI-NOFP16:       // %bb.0: // %entry
 ; CHECK-GI-NOFP16-NEXT:    fmov s0, w0
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[1], w1
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[2], w2
+; CHECK-GI-NOFP16-NEXT:    fmov s1, w1
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NOFP16-NEXT:    fmov s1, w2
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[2], v1.h[0]
 ; CHECK-GI-NOFP16-NEXT:    shl v0.4h, v0.4h, #8
 ; CHECK-GI-NOFP16-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-GI-NOFP16-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NOFP16-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h3, v1.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[1], v2.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
 ; CHECK-GI-NOFP16-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NOFP16-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-GI-NOFP16-NEXT:    scvtf v0.4s, v0.4s
 ; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-GI-NOFP16-NEXT:    ret
@@ -5635,11 +5745,10 @@ define <3 x half> @stofp_v3i8_v3f16(<3 x i8> %a) {
 ; CHECK-GI-FP16:       // %bb.0: // %entry
 ; CHECK-GI-FP16-NEXT:    fmov s0, w0
 ; CHECK-GI-FP16-NEXT:    fmov s1, w1
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-FP16-NEXT:    mov v0.b[1], v1.b[0]
 ; CHECK-GI-FP16-NEXT:    fmov s1, w2
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-GI-FP16-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-GI-FP16-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-GI-FP16-NEXT:    mov v0.b[2], v1.b[0]
+; CHECK-GI-FP16-NEXT:    sshll v0.8h, v0.8b, #0
 ; CHECK-GI-FP16-NEXT:    scvtf v0.4h, v0.4h
 ; CHECK-GI-FP16-NEXT:    ret
 entry:
@@ -5671,10 +5780,20 @@ define <3 x half> @utofp_v3i8_v3f16(<3 x i8> %a) {
 ; CHECK-GI-NOFP16-LABEL: utofp_v3i8_v3f16:
 ; CHECK-GI-NOFP16:       // %bb.0: // %entry
 ; CHECK-GI-NOFP16-NEXT:    fmov s0, w0
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[1], w1
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[2], w2
-; CHECK-GI-NOFP16-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-GI-NOFP16-NEXT:    fmov s1, w1
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NOFP16-NEXT:    fmov s1, w2
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[2], v1.h[0]
+; CHECK-GI-NOFP16-NEXT:    movi d1, #0xff00ff00ff00ff
+; CHECK-GI-NOFP16-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-GI-NOFP16-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NOFP16-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h3, v1.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[1], v2.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
 ; CHECK-GI-NOFP16-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NOFP16-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-GI-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
 ; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-GI-NOFP16-NEXT:    ret
@@ -5683,11 +5802,10 @@ define <3 x half> @utofp_v3i8_v3f16(<3 x i8> %a) {
 ; CHECK-GI-FP16:       // %bb.0: // %entry
 ; CHECK-GI-FP16-NEXT:    fmov s0, w0
 ; CHECK-GI-FP16-NEXT:    fmov s1, w1
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-FP16-NEXT:    mov v0.b[1], v1.b[0]
 ; CHECK-GI-FP16-NEXT:    fmov s1, w2
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-GI-FP16-NEXT:    movi d1, #0xff00ff00ff00ff
-; CHECK-GI-FP16-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-GI-FP16-NEXT:    mov v0.b[2], v1.b[0]
+; CHECK-GI-FP16-NEXT:    ushll v0.8h, v0.8b, #0
 ; CHECK-GI-FP16-NEXT:    ucvtf v0.4h, v0.4h
 ; CHECK-GI-FP16-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/AArch64/load.ll b/llvm/test/CodeGen/AArch64/load.ll
index 39143e5c53ff..c3c0ec5e3d9d 100644
--- a/llvm/test/CodeGen/AArch64/load.ll
+++ b/llvm/test/CodeGen/AArch64/load.ll
@@ -159,7 +159,8 @@ define <2 x i16> @load_v2i16(ptr %ptr){
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    ldr h0, [x0]
 ; CHECK-GI-NEXT:    ldr h1, [x0, #2]
-; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    ret
     %a = load <2 x i16>, ptr %ptr
diff --git a/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll b/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll
index 01620652301e..57f220f621cf 100644
--- a/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll
@@ -1117,8 +1117,15 @@ define <4 x i16> @vselect_constant_cond_zero_v4i16(<4 x i16> %a) {
 ;
 ; CHECK-GI-LABEL: vselect_constant_cond_zero_v4i16:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    adrp x8, .LCPI84_0
-; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI84_0]
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    mov w9, #0 // =0x0
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    mov v3.16b, v1.16b
+; CHECK-GI-NEXT:    mov v3.b[1], v2.b[0]
+; CHECK-GI-NEXT:    mov v3.b[2], v2.b[0]
+; CHECK-GI-NEXT:    mov v3.b[3], v1.b[0]
+; CHECK-GI-NEXT:    ushll v1.8h, v3.8b, #0
 ; CHECK-GI-NEXT:    shl v1.4h, v1.4h, #15
 ; CHECK-GI-NEXT:    sshr v1.4h, v1.4h, #15
 ; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
@@ -1137,8 +1144,16 @@ define <4 x i32> @vselect_constant_cond_zero_v4i32(<4 x i32> %a) {
 ;
 ; CHECK-GI-LABEL: vselect_constant_cond_zero_v4i32:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    adrp x8, .LCPI85_0
-; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI85_0]
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    mov w9, #0 // =0x0
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    mov v3.16b, v1.16b
+; CHECK-GI-NEXT:    mov v3.h[1], v2.h[0]
+; CHECK-GI-NEXT:    mov v2.h[1], v1.h[0]
+; CHECK-GI-NEXT:    ushll v1.4s, v3.4h, #0
+; CHECK-GI-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT:    mov v1.d[1], v2.d[0]
 ; CHECK-GI-NEXT:    shl v1.4s, v1.4s, #31
 ; CHECK-GI-NEXT:    sshr v1.4s, v1.4s, #31
 ; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
@@ -1181,8 +1196,15 @@ define <4 x i16> @vselect_constant_cond_v4i16(<4 x i16> %a, <4 x i16> %b) {
 ;
 ; CHECK-GI-LABEL: vselect_constant_cond_v4i16:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    adrp x8, .LCPI87_0
-; CHECK-GI-NEXT:    ldr d2, [x8, :lo12:.LCPI87_0]
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    mov w9, #0 // =0x0
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    fmov s3, w9
+; CHECK-GI-NEXT:    mov v4.16b, v2.16b
+; CHECK-GI-NEXT:    mov v4.b[1], v3.b[0]
+; CHECK-GI-NEXT:    mov v4.b[2], v3.b[0]
+; CHECK-GI-NEXT:    mov v4.b[3], v2.b[0]
+; CHECK-GI-NEXT:    ushll v2.8h, v4.8b, #0
 ; CHECK-GI-NEXT:    shl v2.4h, v2.4h, #15
 ; CHECK-GI-NEXT:    sshr v2.4h, v2.4h, #15
 ; CHECK-GI-NEXT:    bif v0.8b, v1.8b, v2.8b
@@ -1201,8 +1223,16 @@ define <4 x i32> @vselect_constant_cond_v4i32(<4 x i32> %a, <4 x i32> %b) {
 ;
 ; CHECK-GI-LABEL: vselect_constant_cond_v4i32:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    adrp x8, .LCPI88_0
-; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI88_0]
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    mov w9, #0 // =0x0
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    fmov s3, w9
+; CHECK-GI-NEXT:    mov v4.16b, v2.16b
+; CHECK-GI-NEXT:    mov v4.h[1], v3.h[0]
+; CHECK-GI-NEXT:    mov v3.h[1], v2.h[0]
+; CHECK-GI-NEXT:    ushll v2.4s, v4.4h, #0
+; CHECK-GI-NEXT:    ushll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT:    mov v2.d[1], v3.d[0]
 ; CHECK-GI-NEXT:    shl v2.4s, v2.4s, #31
 ; CHECK-GI-NEXT:    sshr v2.4s, v2.4s, #31
 ; CHECK-GI-NEXT:    bif v0.16b, v1.16b, v2.16b
diff --git a/llvm/test/CodeGen/AArch64/shift.ll b/llvm/test/CodeGen/AArch64/shift.ll
index 5287839ee7b7..9c8d3e0f07de 100644
--- a/llvm/test/CodeGen/AArch64/shift.ll
+++ b/llvm/test/CodeGen/AArch64/shift.ll
@@ -1,13 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-
-; CHECK-GI:         warning: Instruction selection used fallback path for shl_v4i8
-; CHECK-GI-NEXT:    warning: Instruction selection used fallback path for shl_v2i16
-; CHECK-GI-NEXT:    warning: Instruction selection used fallback path for ashr_v4i8
-; CHECK-GI-NEXT:    warning: Instruction selection used fallback path for ashr_v2i16
-; CHECK-GI-NEXT:    warning: Instruction selection used fallback path for lshr_v4i8
-; CHECK-GI-NEXT:    warning: Instruction selection used fallback path for lshr_v2i16
+; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 define i1 @shl_i1(i1 %0, i1 %1){
 ; CHECK-SD-LABEL: shl_i1:
@@ -530,11 +523,38 @@ define <2 x i64> @lshr_v2i64(<2 x i64> %0, <2 x i64> %1){
 ; ===== Vector Larger/Smaller than Legal =====
 
 define <4 x i8> @shl_v4i8(<4 x i8> %0, <4 x i8> %1){
-; CHECK-LABEL: shl_v4i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    bic v1.4h, #255, lsl #8
-; CHECK-NEXT:    ushl v0.4h, v0.4h, v1.4h
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: shl_v4i8:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    bic v1.4h, #255, lsl #8
+; CHECK-SD-NEXT:    ushl v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: shl_v4i8:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-NEXT:    mov h3, v1.h[1]
+; CHECK-GI-NEXT:    mov h4, v0.h[2]
+; CHECK-GI-NEXT:    mov h5, v0.h[3]
+; CHECK-GI-NEXT:    mov h6, v1.h[3]
+; CHECK-GI-NEXT:    mov v0.b[1], v2.b[0]
+; CHECK-GI-NEXT:    mov h2, v1.h[2]
+; CHECK-GI-NEXT:    mov v1.b[1], v3.b[0]
+; CHECK-GI-NEXT:    mov v0.b[2], v4.b[0]
+; CHECK-GI-NEXT:    mov v1.b[2], v2.b[0]
+; CHECK-GI-NEXT:    mov v0.b[3], v5.b[0]
+; CHECK-GI-NEXT:    mov v1.b[3], v6.b[0]
+; CHECK-GI-NEXT:    ushl v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov b2, v0.b[2]
+; CHECK-GI-NEXT:    mov b3, v0.b[3]
+; CHECK-GI-NEXT:    mov v0.b[1], v1.b[0]
+; CHECK-GI-NEXT:    mov v0.b[2], v2.b[0]
+; CHECK-GI-NEXT:    mov v0.b[3], v3.b[0]
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
     %3 = shl <4 x i8> %0, %1
     ret <4 x i8> %3
 }
@@ -556,12 +576,27 @@ define <32 x i8> @shl_v32i8(<32 x i8> %0, <32 x i8> %1){
 }
 
 define <2 x i16> @shl_v2i16(<2 x i16> %0, <2 x i16> %1){
-; CHECK-LABEL: shl_v2i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi d2, #0x00ffff0000ffff
-; CHECK-NEXT:    and v1.8b, v1.8b, v2.8b
-; CHECK-NEXT:    ushl v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: shl_v2i16:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    movi d2, #0x00ffff0000ffff
+; CHECK-SD-NEXT:    and v1.8b, v1.8b, v2.8b
+; CHECK-SD-NEXT:    ushl v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: shl_v2i16:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT:    mov s2, v0.s[1]
+; CHECK-GI-NEXT:    mov s3, v1.s[1]
+; CHECK-GI-NEXT:    mov v0.h[1], v2.h[0]
+; CHECK-GI-NEXT:    mov v1.h[1], v3.h[0]
+; CHECK-GI-NEXT:    ushl v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
     %3 = shl <2 x i16> %0, %1
     ret <2 x i16> %3
 }
@@ -633,14 +668,42 @@ define <4 x i64> @shl_v4i64(<4 x i64> %0, <4 x i64> %1){
 }
 
 define <4 x i8> @ashr_v4i8(<4 x i8> %0, <4 x i8> %1){
-; CHECK-LABEL: ashr_v4i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-NEXT:    bic v1.4h, #255, lsl #8
-; CHECK-NEXT:    sshr v0.4h, v0.4h, #8
-; CHECK-NEXT:    neg v1.4h, v1.4h
-; CHECK-NEXT:    sshl v0.4h, v0.4h, v1.4h
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: ashr_v4i8:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-SD-NEXT:    bic v1.4h, #255, lsl #8
+; CHECK-SD-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-SD-NEXT:    neg v1.4h, v1.4h
+; CHECK-SD-NEXT:    sshl v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ashr_v4i8:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT:    mov h2, v1.h[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h3, v0.h[1]
+; CHECK-GI-NEXT:    mov h4, v1.h[2]
+; CHECK-GI-NEXT:    mov h5, v1.h[3]
+; CHECK-GI-NEXT:    mov h6, v0.h[3]
+; CHECK-GI-NEXT:    mov v1.b[1], v2.b[0]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    mov v0.b[1], v3.b[0]
+; CHECK-GI-NEXT:    mov v1.b[2], v4.b[0]
+; CHECK-GI-NEXT:    mov v0.b[2], v2.b[0]
+; CHECK-GI-NEXT:    mov v1.b[3], v5.b[0]
+; CHECK-GI-NEXT:    mov v0.b[3], v6.b[0]
+; CHECK-GI-NEXT:    neg v1.8b, v1.8b
+; CHECK-GI-NEXT:    sshl v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov b2, v0.b[2]
+; CHECK-GI-NEXT:    mov b3, v0.b[3]
+; CHECK-GI-NEXT:    mov v0.b[1], v1.b[0]
+; CHECK-GI-NEXT:    mov v0.b[2], v2.b[0]
+; CHECK-GI-NEXT:    mov v0.b[3], v3.b[0]
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
     %3 = ashr <4 x i8> %0, %1
     ret <4 x i8> %3
 }
@@ -658,15 +721,31 @@ define <32 x i8> @ashr_v32i8(<32 x i8> %0, <32 x i8> %1){
 }
 
 define <2 x i16> @ashr_v2i16(<2 x i16> %0, <2 x i16> %1){
-; CHECK-LABEL: ashr_v2i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi d2, #0x00ffff0000ffff
-; CHECK-NEXT:    shl v0.2s, v0.2s, #16
-; CHECK-NEXT:    sshr v0.2s, v0.2s, #16
-; CHECK-NEXT:    and v1.8b, v1.8b, v2.8b
-; CHECK-NEXT:    neg v1.2s, v1.2s
-; CHECK-NEXT:    sshl v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: ashr_v2i16:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    movi d2, #0x00ffff0000ffff
+; CHECK-SD-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-SD-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-SD-NEXT:    and v1.8b, v1.8b, v2.8b
+; CHECK-SD-NEXT:    neg v1.2s, v1.2s
+; CHECK-SD-NEXT:    sshl v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ashr_v2i16:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT:    mov s2, v1.s[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov s3, v0.s[1]
+; CHECK-GI-NEXT:    mov v1.h[1], v2.h[0]
+; CHECK-GI-NEXT:    mov v0.h[1], v3.h[0]
+; CHECK-GI-NEXT:    neg v1.4h, v1.4h
+; CHECK-GI-NEXT:    sshl v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
     %3 = ashr <2 x i16> %0, %1
     ret <2 x i16> %3
 }
@@ -727,13 +806,41 @@ define <4 x i64> @ashr_v4i64(<4 x i64> %0, <4 x i64> %1){
 }
 
 define <4 x i8> @lshr_v4i8(<4 x i8> %0, <4 x i8> %1){
-; CHECK-LABEL: lshr_v4i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    bic v1.4h, #255, lsl #8
-; CHECK-NEXT:    bic v0.4h, #255, lsl #8
-; CHECK-NEXT:    neg v1.4h, v1.4h
-; CHECK-NEXT:    ushl v0.4h, v0.4h, v1.4h
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: lshr_v4i8:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    bic v1.4h, #255, lsl #8
+; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-SD-NEXT:    neg v1.4h, v1.4h
+; CHECK-SD-NEXT:    ushl v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: lshr_v4i8:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT:    mov h2, v1.h[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h3, v0.h[1]
+; CHECK-GI-NEXT:    mov h4, v1.h[2]
+; CHECK-GI-NEXT:    mov h5, v1.h[3]
+; CHECK-GI-NEXT:    mov h6, v0.h[3]
+; CHECK-GI-NEXT:    mov v1.b[1], v2.b[0]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    mov v0.b[1], v3.b[0]
+; CHECK-GI-NEXT:    mov v1.b[2], v4.b[0]
+; CHECK-GI-NEXT:    mov v0.b[2], v2.b[0]
+; CHECK-GI-NEXT:    mov v1.b[3], v5.b[0]
+; CHECK-GI-NEXT:    mov v0.b[3], v6.b[0]
+; CHECK-GI-NEXT:    neg v1.8b, v1.8b
+; CHECK-GI-NEXT:    ushl v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov b2, v0.b[2]
+; CHECK-GI-NEXT:    mov b3, v0.b[3]
+; CHECK-GI-NEXT:    mov v0.b[1], v1.b[0]
+; CHECK-GI-NEXT:    mov v0.b[2], v2.b[0]
+; CHECK-GI-NEXT:    mov v0.b[3], v3.b[0]
+; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
     %3 = lshr <4 x i8> %0, %1
     ret <4 x i8> %3
 }
@@ -751,14 +858,30 @@ define <32 x i8> @lshr_v32i8(<32 x i8> %0, <32 x i8> %1){
 }
 
 define <2 x i16> @lshr_v2i16(<2 x i16> %0, <2 x i16> %1){
-; CHECK-LABEL: lshr_v2i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi d2, #0x00ffff0000ffff
-; CHECK-NEXT:    and v1.8b, v1.8b, v2.8b
-; CHECK-NEXT:    and v0.8b, v0.8b, v2.8b
-; CHECK-NEXT:    neg v1.2s, v1.2s
-; CHECK-NEXT:    ushl v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: lshr_v2i16:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    movi d2, #0x00ffff0000ffff
+; CHECK-SD-NEXT:    and v1.8b, v1.8b, v2.8b
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v2.8b
+; CHECK-SD-NEXT:    neg v1.2s, v1.2s
+; CHECK-SD-NEXT:    ushl v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: lshr_v2i16:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT:    mov s2, v1.s[1]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov s3, v0.s[1]
+; CHECK-GI-NEXT:    mov v1.h[1], v2.h[0]
+; CHECK-GI-NEXT:    mov v0.h[1], v3.h[0]
+; CHECK-GI-NEXT:    neg v1.4h, v1.4h
+; CHECK-GI-NEXT:    ushl v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
     %3 = lshr <2 x i16> %0, %1
     ret <2 x i16> %3
 }
-- 
GitLab


From 256343a0e919bc09f65a8ee26751b561fa2dbfc1 Mon Sep 17 00:00:00 2001
From: Thomas Symalla <5754458+tsymalla@users.noreply.github.com>
Date: Tue, 26 Mar 2024 11:01:08 +0100
Subject: [PATCH 254/404] Revert "Update amdgpu_gfx functions to use s0-s3 for
 inreg SGPR arguments on targets using scratch instructions for stack #78226"
 (#86273)

Reverts llvm/llvm-project#81394

This reverts commit 3ac243bc0d7922d083af2cf025247b5698556062.
It is not handling RSrc registers s0-s3 correctly. This leads to a
broken test, where it expects s0-s3 as function argument and uses it as
RSrc register as well.
We need to re-visit the patch, but apparently we only want to have s0-s3
as
argument registers if we don't need them as RSrc registers.
---
 llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp |    5 +-
 llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td   |    1 -
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |    7 +-
 .../GlobalISel/irtranslator-call-non-fixed.ll |   10 +-
 .../AMDGPU/GlobalISel/irtranslator-call.ll    |   10 +-
 llvm/test/CodeGen/AMDGPU/bf16.ll              |    2 +-
 .../CodeGen/AMDGPU/combine_andor_with_cmps.ll |   24 +-
 .../CodeGen/AMDGPU/function-args-inreg.ll     |  133 -
 .../AMDGPU/gfx-callable-argument-types.ll     | 4377 ++++++++++-------
 llvm/test/CodeGen/AMDGPU/indirect-call.ll     |   60 +-
 .../CodeGen/AMDGPU/schedule-addrspaces.ll     |    2 +-
 .../CodeGen/AMDGPU/scratch-pointer-sink.ll    |    4 +-
 12 files changed, 2563 insertions(+), 2072 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
index 8969b3b5b6ce..7e1f041fa109 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
@@ -715,13 +715,10 @@ bool AMDGPUCallLowering::lowerFormalArguments(
   if (!IsEntryFunc && !IsGraphics) {
     // For the fixed ABI, pass workitem IDs in the last argument register.
     TLI.allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info);
-  }
 
-  if (!IsEntryFunc) {
     if (!Subtarget.enableFlatScratch())
       CCInfo.AllocateReg(Info->getScratchRSrcReg());
-    if (!IsGraphics)
-      TLI.allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info);
+    TLI.allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info);
   }
 
   IncomingValueAssigner Assigner(AssignFn);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td b/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td
index 9bd0fd7eca76..4be64629ddac 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td
@@ -23,7 +23,6 @@ def CC_SI_Gfx : CallingConv<[
   // 33 is reserved for the frame pointer
   // 34 is reserved for the base pointer
   CCIfInReg<CCIfType<[f32, i32, f16, i16, v2i16, v2f16, bf16, v2bf16] , CCAssignToReg<[
-    SGPR0, SGPR1, SGPR2, SGPR3,
     SGPR4, SGPR5, SGPR6, SGPR7,
     SGPR8, SGPR9, SGPR10, SGPR11, SGPR12, SGPR13, SGPR14, SGPR15,
     SGPR16, SGPR17, SGPR18, SGPR19, SGPR20, SGPR21, SGPR22, SGPR23,
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 375ddb24674b..0a4370de0613 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -2859,13 +2859,12 @@ SDValue SITargetLowering::LowerFormalArguments(
   } else if (!IsGraphics) {
     // For the fixed ABI, pass workitem IDs in the last argument register.
     allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info);
-  }
 
-  if (!IsEntryFunc) {
+    // FIXME: Sink this into allocateSpecialInputSGPRs
     if (!Subtarget->enableFlatScratch())
       CCInfo.AllocateReg(Info->getScratchRSrcReg());
-    if (!IsGraphics)
-      allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info);
+
+    allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info);
   }
 
   if (!IsKernel) {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll
index fad833c0a6ad..5effd24a7520 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll
@@ -50,10 +50,10 @@ define amdgpu_gfx void @test_gfx_call_external_void_func_i32_imm_inreg(i32 inreg
   ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
   ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $scc
   ; CHECK-NEXT:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_gfx_void_func_i32_inreg
-  ; CHECK-NEXT:   $sgpr0 = COPY [[C]](s32)
+  ; CHECK-NEXT:   $sgpr4 = COPY [[C]](s32)
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY1]](<4 x s32>)
-  ; CHECK-NEXT:   $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3
+  ; CHECK-NEXT:   $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr4, implicit $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
   ; CHECK-NEXT:   SI_RETURN
   call amdgpu_gfx void @external_gfx_void_func_i32_inreg(i32 inreg 42)
@@ -99,11 +99,11 @@ define amdgpu_gfx void @test_gfx_call_external_void_func_struct_i8_i32_inreg() #
   ; CHECK-NEXT:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_gfx_void_func_struct_i8_i32_inreg
   ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[LOAD1]](s8)
   ; CHECK-NEXT:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT]](s16)
-  ; CHECK-NEXT:   $sgpr0 = COPY [[ANYEXT1]](s32)
-  ; CHECK-NEXT:   $sgpr1 = COPY [[LOAD2]](s32)
+  ; CHECK-NEXT:   $sgpr4 = COPY [[ANYEXT1]](s32)
+  ; CHECK-NEXT:   $sgpr5 = COPY [[LOAD2]](s32)
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY]](<4 x s32>)
-  ; CHECK-NEXT:   $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_struct_i8_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr0, implicit $sgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3
+  ; CHECK-NEXT:   $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_struct_i8_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr4, implicit $sgpr5, implicit $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
   ; CHECK-NEXT:   SI_RETURN
   %ptr0 = load ptr addrspace(1), ptr addrspace(4) undef
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
index 75670604baa1..392b0ae6823e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll
@@ -942,10 +942,10 @@ define amdgpu_gfx void @test_gfx_call_external_void_func_i32_imm_inreg(i32 inreg
   ; CHECK-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
   ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $scc
   ; CHECK-NEXT:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_gfx_void_func_i32_inreg
-  ; CHECK-NEXT:   $sgpr0 = COPY [[C]](s32)
+  ; CHECK-NEXT:   $sgpr4 = COPY [[C]](s32)
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY1]](<4 x s32>)
-  ; CHECK-NEXT:   $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3
+  ; CHECK-NEXT:   $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr4, implicit $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
   ; CHECK-NEXT:   SI_RETURN
   call amdgpu_gfx void @external_gfx_void_func_i32_inreg(i32 inreg 42)
@@ -3984,11 +3984,11 @@ define amdgpu_gfx void @test_gfx_call_external_void_func_struct_i8_i32_inreg() #
   ; CHECK-NEXT:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_gfx_void_func_struct_i8_i32_inreg
   ; CHECK-NEXT:   [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[LOAD1]](s8)
   ; CHECK-NEXT:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT]](s16)
-  ; CHECK-NEXT:   $sgpr0 = COPY [[ANYEXT1]](s32)
-  ; CHECK-NEXT:   $sgpr1 = COPY [[LOAD2]](s32)
+  ; CHECK-NEXT:   $sgpr4 = COPY [[ANYEXT1]](s32)
+  ; CHECK-NEXT:   $sgpr5 = COPY [[LOAD2]](s32)
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY]](<4 x s32>)
-  ; CHECK-NEXT:   $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_struct_i8_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr0, implicit $sgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3
+  ; CHECK-NEXT:   $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_struct_i8_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr4, implicit $sgpr5, implicit $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
   ; CHECK-NEXT:   SI_RETURN
   %ptr0 = load ptr addrspace(1), ptr addrspace(4) undef
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index e369f7e3b9a5..98658834e897 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -3337,7 +3337,7 @@ define amdgpu_gfx void @test_inreg_arg_store(bfloat inreg %in, ptr addrspace(1)
 ; GFX11-LABEL: test_inreg_arg_store:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_mov_b32_e32 v2, s0
+; GFX11-NEXT:    v_mov_b32_e32 v2, s4
 ; GFX11-NEXT:    global_store_b16 v[0:1], v2, off
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   store bfloat %in, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll
index e1e3220cc275..10d71a315fbf 100644
--- a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll
@@ -472,7 +472,7 @@ define amdgpu_gfx void @test34(i32 inreg %arg1, i32 inreg %arg2) {
 ; GCN-LABEL: test34:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_min_i32 s0, s0, s1
+; GCN-NEXT:    s_min_i32 s0, s4, s5
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0
 ; GCN-NEXT:    s_cmpk_lt_i32 s0, 0x3e9
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0
@@ -492,7 +492,7 @@ define amdgpu_gfx void @test35(i32 inreg %arg1, i32 inreg %arg2) {
 ; GCN-LABEL: test35:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_max_i32 s0, s0, s1
+; GCN-NEXT:    s_max_i32 s0, s4, s5
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0
 ; GCN-NEXT:    s_cmpk_gt_i32 s0, 0x3e8
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0
@@ -512,9 +512,9 @@ define amdgpu_gfx void @test36(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3
 ; GCN-LABEL: test36:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_min_u32 s0, s0, s1
+; GCN-NEXT:    s_min_u32 s0, s4, s5
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0
-; GCN-NEXT:    s_cmp_lt_u32 s0, s2
+; GCN-NEXT:    s_cmp_lt_u32 s0, s6
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0
 ; GCN-NEXT:    s_cselect_b32 s0, -1, 0
 ; GCN-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s0
@@ -532,9 +532,9 @@ define amdgpu_gfx void @test37(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3
 ; GCN-LABEL: test37:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_max_i32 s0, s0, s1
+; GCN-NEXT:    s_max_i32 s0, s4, s5
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0
-; GCN-NEXT:    s_cmp_ge_i32 s0, s2
+; GCN-NEXT:    s_cmp_ge_i32 s0, s6
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0
 ; GCN-NEXT:    s_cselect_b32 s0, -1, 0
 ; GCN-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s0
@@ -552,7 +552,7 @@ define amdgpu_gfx void @test38(i32 inreg %arg1, i32 inreg %arg2) {
 ; GCN-LABEL: test38:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_max_u32 s0, s0, s1
+; GCN-NEXT:    s_max_u32 s0, s4, s5
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0
 ; GCN-NEXT:    s_cmpk_lt_u32 s0, 0x3e9
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0
@@ -572,7 +572,7 @@ define amdgpu_gfx void @test39(i32 inreg %arg1, i32 inreg %arg2) {
 ; GCN-LABEL: test39:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_min_i32 s0, s0, s1
+; GCN-NEXT:    s_min_i32 s0, s4, s5
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0
 ; GCN-NEXT:    s_cmpk_gt_i32 s0, 0x3e7
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0
@@ -592,9 +592,9 @@ define amdgpu_gfx void @test40(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3
 ; GCN-LABEL: test40:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_max_i32 s0, s0, s1
+; GCN-NEXT:    s_max_i32 s0, s4, s5
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0
-; GCN-NEXT:    s_cmp_le_i32 s0, s2
+; GCN-NEXT:    s_cmp_le_i32 s0, s6
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0
 ; GCN-NEXT:    s_cselect_b32 s0, -1, 0
 ; GCN-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s0
@@ -612,9 +612,9 @@ define amdgpu_gfx void @test41(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3
 ; GCN-LABEL: test41:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_min_u32 s0, s0, s1
+; GCN-NEXT:    s_min_u32 s0, s4, s5
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0
-; GCN-NEXT:    s_cmp_ge_u32 s0, s2
+; GCN-NEXT:    s_cmp_ge_u32 s0, s6
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0
 ; GCN-NEXT:    s_cselect_b32 s0, -1, 0
 ; GCN-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s0
diff --git a/llvm/test/CodeGen/AMDGPU/function-args-inreg.ll b/llvm/test/CodeGen/AMDGPU/function-args-inreg.ll
index 27845b6b5b2f..44a9127b4bd0 100644
--- a/llvm/test/CodeGen/AMDGPU/function-args-inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-args-inreg.ll
@@ -2176,93 +2176,6 @@ define void @void_func_a5i32_inreg([5 x i32] inreg %arg0, ptr addrspace(1) %ptr)
 declare void @extern()
 
 define void @void_func_a13i32_inreg([13  x i32] inreg %arg0, ptr addrspace(1) %ptr) {
-; GFX9-LABEL: void_func_a13i32_inreg:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s27, s33
-; GFX9-NEXT:    s_mov_b32 s33, s32
-; GFX9-NEXT:    s_or_saveexec_b64 s[28:29], -1
-; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
-; GFX9-NEXT:    s_mov_b64 exec, s[28:29]
-; GFX9-NEXT:    v_mov_b32_e32 v2, s26
-; GFX9-NEXT:    global_store_dword v[0:1], v2, off offset:48
-; GFX9-NEXT:    v_mov_b32_e32 v5, s25
-; GFX9-NEXT:    v_mov_b32_e32 v4, s24
-; GFX9-NEXT:    v_mov_b32_e32 v3, s23
-; GFX9-NEXT:    v_mov_b32_e32 v2, s22
-; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:32
-; GFX9-NEXT:    v_writelane_b32 v40, s27, 2
-; GFX9-NEXT:    v_mov_b32_e32 v5, s21
-; GFX9-NEXT:    v_mov_b32_e32 v4, s20
-; GFX9-NEXT:    v_mov_b32_e32 v3, s19
-; GFX9-NEXT:    v_mov_b32_e32 v2, s18
-; GFX9-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:16
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, s17
-; GFX9-NEXT:    v_mov_b32_e32 v4, s16
-; GFX9-NEXT:    s_getpc_b64 s[16:17]
-; GFX9-NEXT:    s_add_u32 s16, s16, extern@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s17, s17, extern@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[16:17], 0x0
-; GFX9-NEXT:    v_mov_b32_e32 v3, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, s6
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX9-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s4, v40, 2
-; GFX9-NEXT:    s_or_saveexec_b64 s[6:7], -1
-; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
-; GFX9-NEXT:    s_mov_b64 exec, s[6:7]
-; GFX9-NEXT:    s_addk_i32 s32, 0xfc00
-; GFX9-NEXT:    s_mov_b32 s33, s4
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: void_func_a13i32_inreg:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s23, s33
-; GFX11-NEXT:    s_mov_b32 s33, s32
-; GFX11-NEXT:    s_or_saveexec_b32 s24, -1
-; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
-; GFX11-NEXT:    s_mov_b32 exec_lo, s24
-; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_dual_mov_b32 v4, s20 :: v_dual_mov_b32 v3, s19
-; GFX11-NEXT:    v_dual_mov_b32 v2, s18 :: v_dual_mov_b32 v9, s17
-; GFX11-NEXT:    s_getpc_b64 s[18:19]
-; GFX11-NEXT:    s_add_u32 s18, s18, extern@gotpcrel32@lo+4
-; GFX11-NEXT:    s_addc_u32 s19, s19, extern@gotpcrel32@hi+12
-; GFX11-NEXT:    v_dual_mov_b32 v8, s16 :: v_dual_mov_b32 v7, s7
-; GFX11-NEXT:    s_load_b64 s[16:17], s[18:19], 0x0
-; GFX11-NEXT:    v_writelane_b32 v40, s23, 2
-; GFX11-NEXT:    v_dual_mov_b32 v14, s22 :: v_dual_mov_b32 v5, s21
-; GFX11-NEXT:    v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v13, s3
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX11-NEXT:    v_dual_mov_b32 v12, s2 :: v_dual_mov_b32 v11, s1
-; GFX11-NEXT:    v_mov_b32_e32 v10, s0
-; GFX11-NEXT:    s_clause 0x3
-; GFX11-NEXT:    global_store_b32 v[0:1], v14, off offset:48
-; GFX11-NEXT:    global_store_b128 v[0:1], v[2:5], off offset:32
-; GFX11-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:16
-; GFX11-NEXT:    global_store_b128 v[0:1], v[10:13], off
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
-; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
-; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
-; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    s_add_i32 s32, s32, -16
-; GFX11-NEXT:    s_mov_b32 s33, s0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
   store [13 x i32] %arg0, ptr addrspace(1) %ptr
   call void @extern()
   ret void
@@ -2290,52 +2203,6 @@ define void @void_func_a13i32_inreg([13  x i32] inreg %arg0, ptr addrspace(1) %p
 
 ; FIXME: Should still fail
 define void @void_func_a16i32_inreg__noimplicit([16 x i32] inreg %arg0, ptr addrspace(1) %ptr) {
-; GFX9-LABEL: void_func_a16i32_inreg__noimplicit:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v5, s19
-; GFX9-NEXT:    v_mov_b32_e32 v4, s18
-; GFX9-NEXT:    v_mov_b32_e32 v3, s17
-; GFX9-NEXT:    v_mov_b32_e32 v2, s16
-; GFX9-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:48
-; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, s15
-; GFX9-NEXT:    v_mov_b32_e32 v4, s14
-; GFX9-NEXT:    v_mov_b32_e32 v3, s13
-; GFX9-NEXT:    v_mov_b32_e32 v2, s12
-; GFX9-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:32
-; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, s11
-; GFX9-NEXT:    v_mov_b32_e32 v4, s10
-; GFX9-NEXT:    v_mov_b32_e32 v3, s9
-; GFX9-NEXT:    v_mov_b32_e32 v2, s8
-; GFX9-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off offset:16
-; GFX9-NEXT:    s_nop 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, s7
-; GFX9-NEXT:    v_mov_b32_e32 v4, s6
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: void_func_a16i32_inreg__noimplicit:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_dual_mov_b32 v5, s15 :: v_dual_mov_b32 v4, s14
-; GFX11-NEXT:    v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v2, s12
-; GFX11-NEXT:    v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v8, s10
-; GFX11-NEXT:    v_dual_mov_b32 v7, s9 :: v_dual_mov_b32 v6, s8
-; GFX11-NEXT:    v_dual_mov_b32 v13, s7 :: v_dual_mov_b32 v12, s6
-; GFX11-NEXT:    v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v10, s4
-; GFX11-NEXT:    v_dual_mov_b32 v17, s3 :: v_dual_mov_b32 v16, s2
-; GFX11-NEXT:    v_dual_mov_b32 v15, s1 :: v_dual_mov_b32 v14, s0
-; GFX11-NEXT:    s_clause 0x3
-; GFX11-NEXT:    global_store_b128 v[0:1], v[2:5], off offset:48
-; GFX11-NEXT:    global_store_b128 v[0:1], v[6:9], off offset:32
-; GFX11-NEXT:    global_store_b128 v[0:1], v[10:13], off offset:16
-; GFX11-NEXT:    global_store_b128 v[0:1], v[14:17], off
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
   store [16 x i32] %arg0, ptr addrspace(1) %ptr
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
index 3e1db5fb4e1d..a118fa388f86 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
@@ -9567,17 +9567,19 @@ define amdgpu_gfx void @test_call_external_void_func_i8_imm_inreg(i32) #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 1
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_i8_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_i8_inreg@abs32@lo
-; GFX9-NEXT:    s_movk_i32 s0, 0x7b
+; GFX9-NEXT:    s_movk_i32 s4, 0x7b
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 2
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 3
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -9595,17 +9597,19 @@ define amdgpu_gfx void @test_call_external_void_func_i8_imm_inreg(i32) #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 3
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_i8_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_i8_inreg@abs32@lo
-; GFX10-NEXT:    s_movk_i32 s0, 0x7b
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_movk_i32 s4, 0x7b
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 2
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 3
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -9623,18 +9627,20 @@ define amdgpu_gfx void @test_call_external_void_func_i8_imm_inreg(i32) #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_i8_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_i8_inreg@abs32@lo
-; GFX11-NEXT:    s_movk_i32 s0, 0x7b
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 3
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_i8_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_i8_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_movk_i32 s4, 0x7b
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 2
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 3
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -9652,17 +9658,19 @@ define amdgpu_gfx void @test_call_external_void_func_i8_imm_inreg(i32) #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_i8_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_i8_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_movk_i32 s0, 0x7b
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 3
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_i8_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_i8_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_movk_i32 s4, 0x7b
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 2
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 3
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -9684,17 +9692,19 @@ define amdgpu_gfx void @test_call_external_void_func_i16_imm_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 1
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_i16_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_i16_inreg@abs32@lo
-; GFX9-NEXT:    s_movk_i32 s0, 0x7b
+; GFX9-NEXT:    s_movk_i32 s4, 0x7b
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 2
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 3
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -9712,17 +9722,19 @@ define amdgpu_gfx void @test_call_external_void_func_i16_imm_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 3
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_i16_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_i16_inreg@abs32@lo
-; GFX10-NEXT:    s_movk_i32 s0, 0x7b
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_movk_i32 s4, 0x7b
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 2
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 3
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -9740,18 +9752,20 @@ define amdgpu_gfx void @test_call_external_void_func_i16_imm_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_i16_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_i16_inreg@abs32@lo
-; GFX11-NEXT:    s_movk_i32 s0, 0x7b
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 3
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_i16_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_i16_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_movk_i32 s4, 0x7b
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 2
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 3
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -9769,17 +9783,19 @@ define amdgpu_gfx void @test_call_external_void_func_i16_imm_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_i16_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_i16_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_movk_i32 s0, 0x7b
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 3
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_i16_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_i16_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_movk_i32 s4, 0x7b
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 2
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 3
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -9801,17 +9817,19 @@ define amdgpu_gfx void @test_call_external_void_func_i32_imm_inreg(i32) #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 1
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_i32_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_i32_inreg@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, 42
+; GFX9-NEXT:    s_mov_b32 s4, 42
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 2
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 3
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -9829,17 +9847,19 @@ define amdgpu_gfx void @test_call_external_void_func_i32_imm_inreg(i32) #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 3
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_i32_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_i32_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, 42
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_mov_b32 s4, 42
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 2
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 3
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -9857,18 +9877,20 @@ define amdgpu_gfx void @test_call_external_void_func_i32_imm_inreg(i32) #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_i32_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_i32_inreg@abs32@lo
-; GFX11-NEXT:    s_mov_b32 s0, 42
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 3
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_i32_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_i32_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_mov_b32 s4, 42
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 2
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 3
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -9886,17 +9908,19 @@ define amdgpu_gfx void @test_call_external_void_func_i32_imm_inreg(i32) #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_i32_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_i32_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 42
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 3
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_i32_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_i32_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 42
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 2
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 3
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -9918,18 +9942,22 @@ define amdgpu_gfx void @test_call_external_void_func_i64_imm_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 2
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_i64_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_i64_inreg@abs32@lo
-; GFX9-NEXT:    s_movk_i32 s0, 0x7b
-; GFX9-NEXT:    s_mov_b32 s1, 0
+; GFX9-NEXT:    s_movk_i32 s4, 0x7b
+; GFX9-NEXT:    s_mov_b32 s5, 0
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 3
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -9947,18 +9975,22 @@ define amdgpu_gfx void @test_call_external_void_func_i64_imm_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 4
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_i64_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_i64_inreg@abs32@lo
-; GFX10-NEXT:    s_movk_i32 s0, 0x7b
-; GFX10-NEXT:    s_mov_b32 s1, 0
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_movk_i32 s4, 0x7b
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    s_mov_b32 s5, 0
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 3
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -9976,19 +10008,23 @@ define amdgpu_gfx void @test_call_external_void_func_i64_imm_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_i64_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_i64_inreg@abs32@lo
-; GFX11-NEXT:    s_movk_i32 s0, 0x7b
-; GFX11-NEXT:    s_mov_b32 s1, 0
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 4
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_i64_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_i64_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_movk_i32 s4, 0x7b
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    s_mov_b32 s5, 0
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -10006,18 +10042,22 @@ define amdgpu_gfx void @test_call_external_void_func_i64_imm_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_i64_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_i64_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_movk_i32 s0, 0x7b
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 0
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 4
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_i64_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_i64_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_movk_i32 s4, 0x7b
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -10039,25 +10079,28 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 6
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s6, 2
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:    s_load_dwordx4 s[36:39], s[34:35], 0x0
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[34:35], 0x0
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 4
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s0, s36
-; GFX9-NEXT:    s_mov_b32 s1, s37
-; GFX9-NEXT:    s_mov_b32 s2, s38
-; GFX9-NEXT:    s_mov_b32 s3, s39
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 5
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
-; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
-; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX9-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 6
+; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
+; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
 ; GFX9-NEXT:    s_addk_i32 s32, 0xfc00
 ; GFX9-NEXT:    s_mov_b32 s33, s34
@@ -10073,23 +10116,26 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 6
 ; GFX10-NEXT:    s_mov_b64 s[34:35], 0
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    s_load_dwordx4 s[36:39], s[34:35], 0x0
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[34:35], 0x0
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s0, s36
-; GFX10-NEXT:    s_mov_b32 s1, s37
-; GFX10-NEXT:    s_mov_b32 s2, s38
-; GFX10-NEXT:    s_mov_b32 s3, s39
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 5
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX10-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 6
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -10107,19 +10153,27 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 6
 ; GFX11-NEXT:    s_mov_b64 s[0:1], 0
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi
-; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX11-NEXT:    s_load_b128 s[4:7], s[0:1], 0x0
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v2i64_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v2i64_inreg@abs32@lo
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 5
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX11-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 6
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -10137,18 +10191,26 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 6
 ; GFX10-SCRATCH-NEXT:    s_mov_b64 s[0:1], 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-SCRATCH-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v2i64_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v2i64_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 5
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 6
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -10171,20 +10233,28 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_imm_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 6
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 4
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, 1
-; GFX9-NEXT:    s_mov_b32 s1, 2
-; GFX9-NEXT:    s_mov_b32 s2, 3
-; GFX9-NEXT:    s_mov_b32 s3, 4
+; GFX9-NEXT:    s_mov_b32 s4, 1
+; GFX9-NEXT:    s_mov_b32 s5, 2
+; GFX9-NEXT:    s_mov_b32 s6, 3
+; GFX9-NEXT:    s_mov_b32 s7, 4
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 5
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX9-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 6
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -10202,20 +10272,28 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_imm_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 6
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, 1
-; GFX10-NEXT:    s_mov_b32 s1, 2
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    s_mov_b32 s2, 3
-; GFX10-NEXT:    s_mov_b32 s3, 4
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_mov_b32 s4, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    s_mov_b32 s5, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-NEXT:    s_mov_b32 s6, 3
+; GFX10-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-NEXT:    s_mov_b32 s7, 4
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 5
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX10-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 6
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -10233,21 +10311,29 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_imm_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo
-; GFX11-NEXT:    s_mov_b32 s0, 1
-; GFX11-NEXT:    s_mov_b32 s1, 2
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX11-NEXT:    s_mov_b32 s2, 3
-; GFX11-NEXT:    s_mov_b32 s3, 4
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 6
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v2i64_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v2i64_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_mov_b32 s4, 1
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    s_mov_b32 s5, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX11-NEXT:    s_mov_b32 s6, 3
+; GFX11-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX11-NEXT:    s_mov_b32 s7, 4
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 5
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX11-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 6
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -10265,20 +10351,28 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_imm_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 1
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 2
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, 3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, 4
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 6
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v2i64_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v2i64_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s6, 3
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s7, 4
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 5
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 6
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -10300,29 +10394,32 @@ define amdgpu_gfx void @test_call_external_void_func_v3i64_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 4
-; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:    s_load_dwordx4 s[36:39], s[34:35], 0x0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 8
 ; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[34:35], 0x0
+; GFX9-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 6
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v3i64_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v3i64_inreg@abs32@lo
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s0, s36
-; GFX9-NEXT:    s_mov_b32 s1, s37
-; GFX9-NEXT:    s_mov_b32 s2, s38
-; GFX9-NEXT:    s_mov_b32 s3, s39
-; GFX9-NEXT:    s_mov_b32 s4, 1
-; GFX9-NEXT:    s_mov_b32 s5, 2
+; GFX9-NEXT:    s_mov_b32 s8, 1
+; GFX9-NEXT:    s_mov_b32 s9, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 7
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 3
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 7
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 6
+; GFX9-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX9-NEXT:    v_readlane_b32 s8, v40, 4
+; GFX9-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 4
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 8
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -10340,29 +10437,32 @@ define amdgpu_gfx void @test_call_external_void_func_v3i64_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 4
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 8
 ; GFX10-NEXT:    s_mov_b64 s[34:35], 0
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    s_load_dwordx4 s[36:39], s[34:35], 0x0
-; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v3i64_inreg@abs32@hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v3i64_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s4, 1
 ; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
-; GFX10-NEXT:    s_mov_b32 s5, 2
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 2
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 3
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s0, s36
-; GFX10-NEXT:    s_mov_b32 s1, s37
-; GFX10-NEXT:    s_mov_b32 s2, s38
-; GFX10-NEXT:    s_mov_b32 s3, s39
+; GFX10-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[34:35], 0x0
+; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v3i64_inreg@abs32@hi
+; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v3i64_inreg@abs32@lo
+; GFX10-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX10-NEXT:    s_mov_b32 s8, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX10-NEXT:    s_mov_b32 s9, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 6
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 7
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 3
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 7
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 6
+; GFX10-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX10-NEXT:    v_readlane_b32 s8, v40, 4
+; GFX10-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 4
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 8
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -10380,25 +10480,33 @@ define amdgpu_gfx void @test_call_external_void_func_v3i64_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 4
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 8
 ; GFX11-NEXT:    s_mov_b64 s[0:1], 0
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v3i64_inreg@abs32@hi
-; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v3i64_inreg@abs32@lo
-; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX11-NEXT:    s_mov_b32 s4, 1
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
-; GFX11-NEXT:    s_mov_b32 s5, 2
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 2
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 3
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX11-NEXT:    s_load_b128 s[4:7], s[0:1], 0x0
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v3i64_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v3i64_inreg@abs32@lo
+; GFX11-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX11-NEXT:    s_mov_b32 s8, 1
+; GFX11-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX11-NEXT:    s_mov_b32 s9, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 6
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 7
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 3
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 7
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 6
+; GFX11-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX11-NEXT:    v_readlane_b32 s8, v40, 4
+; GFX11-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 4
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 8
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -10416,24 +10524,32 @@ define amdgpu_gfx void @test_call_external_void_func_v3i64_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 4
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 8
 ; GFX10-SCRATCH-NEXT:    s_mov_b64 s[0:1], 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v3i64_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v3i64_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 1
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 2
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 2
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 3
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 3
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-SCRATCH-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v3i64_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v3i64_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s8, 1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s9, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 6
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 7
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 7
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 6
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s8, v40, 4
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 4
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 8
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -10458,35 +10574,38 @@ define amdgpu_gfx void @test_call_external_void_func_v4i64_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 6
-; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:    s_load_dwordx4 s[36:39], s[34:35], 0x0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 10
 ; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
 ; GFX9-NEXT:    v_writelane_b32 v40, s6, 2
 ; GFX9-NEXT:    v_writelane_b32 v40, s7, 3
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[34:35], 0x0
+; GFX9-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX9-NEXT:    v_writelane_b32 v40, s10, 6
+; GFX9-NEXT:    v_writelane_b32 v40, s11, 7
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 8
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v4i64_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v4i64_inreg@abs32@lo
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s0, s36
-; GFX9-NEXT:    s_mov_b32 s1, s37
-; GFX9-NEXT:    s_mov_b32 s2, s38
-; GFX9-NEXT:    s_mov_b32 s3, s39
-; GFX9-NEXT:    s_mov_b32 s4, 1
-; GFX9-NEXT:    s_mov_b32 s5, 2
-; GFX9-NEXT:    s_mov_b32 s6, 3
-; GFX9-NEXT:    s_mov_b32 s7, 4
+; GFX9-NEXT:    s_mov_b32 s8, 1
+; GFX9-NEXT:    s_mov_b32 s9, 2
+; GFX9-NEXT:    s_mov_b32 s10, 3
+; GFX9-NEXT:    s_mov_b32 s11, 4
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 5
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 9
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 5
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 9
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 8
+; GFX9-NEXT:    v_readlane_b32 s11, v40, 7
+; GFX9-NEXT:    v_readlane_b32 s10, v40, 6
+; GFX9-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX9-NEXT:    v_readlane_b32 s8, v40, 4
 ; GFX9-NEXT:    v_readlane_b32 s7, v40, 3
 ; GFX9-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 6
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 10
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -10504,35 +10623,38 @@ define amdgpu_gfx void @test_call_external_void_func_v4i64_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 6
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 10
 ; GFX10-NEXT:    s_mov_b64 s[34:35], 0
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    s_load_dwordx4 s[36:39], s[34:35], 0x0
-; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v4i64_inreg@abs32@hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v4i64_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s4, 1
 ; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
-; GFX10-NEXT:    s_mov_b32 s5, 2
 ; GFX10-NEXT:    v_writelane_b32 v40, s6, 2
-; GFX10-NEXT:    s_mov_b32 s6, 3
 ; GFX10-NEXT:    v_writelane_b32 v40, s7, 3
-; GFX10-NEXT:    s_mov_b32 s7, 4
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s0, s36
-; GFX10-NEXT:    s_mov_b32 s1, s37
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 4
-; GFX10-NEXT:    s_mov_b32 s2, s38
-; GFX10-NEXT:    s_mov_b32 s3, s39
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 5
+; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[34:35], 0x0
+; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v4i64_inreg@abs32@hi
+; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v4i64_inreg@abs32@lo
+; GFX10-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX10-NEXT:    s_mov_b32 s8, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX10-NEXT:    s_mov_b32 s9, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s10, 6
+; GFX10-NEXT:    s_mov_b32 s10, 3
+; GFX10-NEXT:    v_writelane_b32 v40, s11, 7
+; GFX10-NEXT:    s_mov_b32 s11, 4
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 8
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 9
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 5
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 9
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 8
+; GFX10-NEXT:    v_readlane_b32 s11, v40, 7
+; GFX10-NEXT:    v_readlane_b32 s10, v40, 6
+; GFX10-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX10-NEXT:    v_readlane_b32 s8, v40, 4
 ; GFX10-NEXT:    v_readlane_b32 s7, v40, 3
 ; GFX10-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 6
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 10
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -10550,31 +10672,39 @@ define amdgpu_gfx void @test_call_external_void_func_v4i64_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 6
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 10
 ; GFX11-NEXT:    s_mov_b64 s[0:1], 0
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v4i64_inreg@abs32@hi
-; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v4i64_inreg@abs32@lo
-; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX11-NEXT:    s_mov_b32 s4, 1
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
-; GFX11-NEXT:    s_mov_b32 s5, 2
 ; GFX11-NEXT:    v_writelane_b32 v40, s6, 2
-; GFX11-NEXT:    s_mov_b32 s6, 3
 ; GFX11-NEXT:    v_writelane_b32 v40, s7, 3
-; GFX11-NEXT:    s_mov_b32 s7, 4
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 4
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 5
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    s_load_b128 s[4:7], s[0:1], 0x0
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v4i64_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v4i64_inreg@abs32@lo
+; GFX11-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX11-NEXT:    s_mov_b32 s8, 1
+; GFX11-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX11-NEXT:    s_mov_b32 s9, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s10, 6
+; GFX11-NEXT:    s_mov_b32 s10, 3
+; GFX11-NEXT:    v_writelane_b32 v40, s11, 7
+; GFX11-NEXT:    s_mov_b32 s11, 4
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 8
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 9
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 5
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 9
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 8
+; GFX11-NEXT:    v_readlane_b32 s11, v40, 7
+; GFX11-NEXT:    v_readlane_b32 s10, v40, 6
+; GFX11-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX11-NEXT:    v_readlane_b32 s8, v40, 4
 ; GFX11-NEXT:    v_readlane_b32 s7, v40, 3
 ; GFX11-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 6
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 10
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -10592,30 +10722,38 @@ define amdgpu_gfx void @test_call_external_void_func_v4i64_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 6
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 10
 ; GFX10-SCRATCH-NEXT:    s_mov_b64 s[0:1], 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v4i64_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v4i64_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 1
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 2
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s6, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s6, 3
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s7, 3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s7, 4
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 4
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 5
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 5
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX10-SCRATCH-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v4i64_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v4i64_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s8, 1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s9, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s10, 6
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s10, 3
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s11, 7
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s11, 4
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 8
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 9
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 9
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 8
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s11, v40, 7
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s10, v40, 6
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s8, v40, 4
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s7, v40, 3
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 6
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 10
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -10639,17 +10777,19 @@ define amdgpu_gfx void @test_call_external_void_func_f16_imm_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 1
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_f16_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_f16_inreg@abs32@lo
-; GFX9-NEXT:    s_movk_i32 s0, 0x4400
+; GFX9-NEXT:    s_movk_i32 s4, 0x4400
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 2
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 3
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -10667,17 +10807,19 @@ define amdgpu_gfx void @test_call_external_void_func_f16_imm_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 3
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_f16_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_f16_inreg@abs32@lo
-; GFX10-NEXT:    s_movk_i32 s0, 0x4400
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_movk_i32 s4, 0x4400
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 2
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 3
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -10695,18 +10837,20 @@ define amdgpu_gfx void @test_call_external_void_func_f16_imm_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_f16_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_f16_inreg@abs32@lo
-; GFX11-NEXT:    s_movk_i32 s0, 0x4400
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 3
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_f16_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_f16_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_movk_i32 s4, 0x4400
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 2
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 3
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -10724,17 +10868,19 @@ define amdgpu_gfx void @test_call_external_void_func_f16_imm_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_f16_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_f16_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_movk_i32 s0, 0x4400
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 3
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_f16_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_f16_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_movk_i32 s4, 0x4400
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 2
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 3
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -10756,17 +10902,19 @@ define amdgpu_gfx void @test_call_external_void_func_f32_imm_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 1
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_f32_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_f32_inreg@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, 4.0
+; GFX9-NEXT:    s_mov_b32 s4, 4.0
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 2
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 3
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -10784,17 +10932,19 @@ define amdgpu_gfx void @test_call_external_void_func_f32_imm_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 3
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_f32_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_f32_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, 4.0
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_mov_b32 s4, 4.0
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 2
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 3
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -10812,18 +10962,20 @@ define amdgpu_gfx void @test_call_external_void_func_f32_imm_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_f32_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_f32_inreg@abs32@lo
-; GFX11-NEXT:    s_mov_b32 s0, 4.0
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 3
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_f32_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_f32_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_mov_b32 s4, 4.0
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 2
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 3
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -10841,17 +10993,19 @@ define amdgpu_gfx void @test_call_external_void_func_f32_imm_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_f32_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_f32_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 4.0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 3
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_f32_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_f32_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 4.0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 2
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 3
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -10873,18 +11027,22 @@ define amdgpu_gfx void @test_call_external_void_func_v2f32_imm_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 2
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v2f32_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v2f32_inreg@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, 1.0
-; GFX9-NEXT:    s_mov_b32 s1, 2.0
+; GFX9-NEXT:    s_mov_b32 s4, 1.0
+; GFX9-NEXT:    s_mov_b32 s5, 2.0
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 3
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -10902,18 +11060,22 @@ define amdgpu_gfx void @test_call_external_void_func_v2f32_imm_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 4
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v2f32_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v2f32_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, 1.0
-; GFX10-NEXT:    s_mov_b32 s1, 2.0
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_mov_b32 s4, 1.0
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    s_mov_b32 s5, 2.0
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 3
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -10931,19 +11093,23 @@ define amdgpu_gfx void @test_call_external_void_func_v2f32_imm_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_v2f32_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_v2f32_inreg@abs32@lo
-; GFX11-NEXT:    s_mov_b32 s0, 1.0
-; GFX11-NEXT:    s_mov_b32 s1, 2.0
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 4
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v2f32_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v2f32_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_mov_b32 s4, 1.0
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    s_mov_b32 s5, 2.0
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -10961,24 +11127,28 @@ define amdgpu_gfx void @test_call_external_void_func_v2f32_imm_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_v2f32_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_v2f32_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 1.0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 2.0
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 4
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v2f32_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v2f32_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
-; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
-; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
-; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, -16
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 1.0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 2.0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 4
+; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
+; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
+; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
+; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, -16
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s0
 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-SCRATCH-NEXT:    s_setpc_b64 s[30:31]
   call amdgpu_gfx void @external_void_func_v2f32_inreg(<2 x float> inreg <float 1.0, float 2.0>)
@@ -10994,19 +11164,25 @@ define amdgpu_gfx void @test_call_external_void_func_v3f32_imm_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 5
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 3
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v3f32_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v3f32_inreg@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, 1.0
-; GFX9-NEXT:    s_mov_b32 s1, 2.0
-; GFX9-NEXT:    s_mov_b32 s2, 4.0
+; GFX9-NEXT:    s_mov_b32 s4, 1.0
+; GFX9-NEXT:    s_mov_b32 s5, 2.0
+; GFX9-NEXT:    s_mov_b32 s6, 4.0
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 4
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 4
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 5
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -11024,19 +11200,25 @@ define amdgpu_gfx void @test_call_external_void_func_v3f32_imm_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 5
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v3f32_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v3f32_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, 1.0
-; GFX10-NEXT:    s_mov_b32 s1, 2.0
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    s_mov_b32 s2, 4.0
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_mov_b32 s4, 1.0
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    s_mov_b32 s5, 2.0
+; GFX10-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-NEXT:    s_mov_b32 s6, 4.0
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 3
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 4
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 4
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 5
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -11054,20 +11236,26 @@ define amdgpu_gfx void @test_call_external_void_func_v3f32_imm_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v3f32_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v3f32_inreg@abs32@lo
-; GFX11-NEXT:    s_mov_b32 s0, 1.0
-; GFX11-NEXT:    s_mov_b32 s1, 2.0
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX11-NEXT:    s_mov_b32 s2, 4.0
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 5
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v3f32_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v3f32_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_mov_b32 s4, 1.0
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    s_mov_b32 s5, 2.0
+; GFX11-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX11-NEXT:    s_mov_b32 s6, 4.0
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 3
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 4
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 4
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 5
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -11085,19 +11273,25 @@ define amdgpu_gfx void @test_call_external_void_func_v3f32_imm_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v3f32_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v3f32_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 1.0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 2.0
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, 4.0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 5
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v3f32_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v3f32_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 1.0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 2.0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s6, 4.0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 3
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 4
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 4
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 5
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -11119,23 +11313,31 @@ define amdgpu_gfx void @test_call_external_void_func_v5f32_imm_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 7
 ; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 5
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v5f32_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v5f32_inreg@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, 1.0
-; GFX9-NEXT:    s_mov_b32 s1, 2.0
-; GFX9-NEXT:    s_mov_b32 s2, 4.0
-; GFX9-NEXT:    s_mov_b32 s3, -1.0
-; GFX9-NEXT:    s_mov_b32 s4, 0.5
+; GFX9-NEXT:    s_mov_b32 s4, 1.0
+; GFX9-NEXT:    s_mov_b32 s5, 2.0
+; GFX9-NEXT:    s_mov_b32 s6, 4.0
+; GFX9-NEXT:    s_mov_b32 s7, -1.0
+; GFX9-NEXT:    s_mov_b32 s8, 0.5
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 6
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 2
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 6
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 5
+; GFX9-NEXT:    v_readlane_b32 s8, v40, 4
+; GFX9-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 7
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -11153,23 +11355,31 @@ define amdgpu_gfx void @test_call_external_void_func_v5f32_imm_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 3
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 7
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v5f32_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v5f32_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, 1.0
-; GFX10-NEXT:    s_mov_b32 s1, 2.0
-; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX10-NEXT:    s_mov_b32 s2, 4.0
-; GFX10-NEXT:    s_mov_b32 s3, -1.0
-; GFX10-NEXT:    s_mov_b32 s4, 0.5
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 1
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_mov_b32 s4, 1.0
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    s_mov_b32 s5, 2.0
+; GFX10-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-NEXT:    s_mov_b32 s6, 4.0
+; GFX10-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-NEXT:    s_mov_b32 s7, -1.0
+; GFX10-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX10-NEXT:    s_mov_b32 s8, 0.5
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 5
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 6
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 2
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 6
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 5
+; GFX10-NEXT:    v_readlane_b32 s8, v40, 4
+; GFX10-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 7
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -11187,24 +11397,32 @@ define amdgpu_gfx void @test_call_external_void_func_v5f32_imm_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 3
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v5f32_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v5f32_inreg@abs32@lo
-; GFX11-NEXT:    s_mov_b32 s0, 1.0
-; GFX11-NEXT:    s_mov_b32 s1, 2.0
-; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX11-NEXT:    s_mov_b32 s2, 4.0
-; GFX11-NEXT:    s_mov_b32 s3, -1.0
-; GFX11-NEXT:    s_mov_b32 s4, 0.5
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 7
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v5f32_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v5f32_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 1
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 2
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_mov_b32 s4, 1.0
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    s_mov_b32 s5, 2.0
+; GFX11-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX11-NEXT:    s_mov_b32 s6, 4.0
+; GFX11-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX11-NEXT:    s_mov_b32 s7, -1.0
+; GFX11-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX11-NEXT:    s_mov_b32 s8, 0.5
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 5
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 6
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 2
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 6
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 5
+; GFX11-NEXT:    v_readlane_b32 s8, v40, 4
+; GFX11-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 7
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -11222,23 +11440,31 @@ define amdgpu_gfx void @test_call_external_void_func_v5f32_imm_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v5f32_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v5f32_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 1.0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 2.0
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, 4.0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, -1.0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 0.5
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 7
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v5f32_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v5f32_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 2
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 2
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 1.0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 2.0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s6, 4.0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s7, -1.0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s8, 0.5
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 5
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 6
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 6
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 5
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s8, v40, 4
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 7
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -11260,18 +11486,22 @@ define amdgpu_gfx void @test_call_external_void_func_f64_imm_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 2
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_f64_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_f64_inreg@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, 0
-; GFX9-NEXT:    s_mov_b32 s1, 0x40100000
+; GFX9-NEXT:    s_mov_b32 s4, 0
+; GFX9-NEXT:    s_mov_b32 s5, 0x40100000
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 3
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -11289,18 +11519,22 @@ define amdgpu_gfx void @test_call_external_void_func_f64_imm_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 4
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_f64_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_f64_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, 0
-; GFX10-NEXT:    s_mov_b32 s1, 0x40100000
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    s_mov_b32 s5, 0x40100000
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 3
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -11318,19 +11552,23 @@ define amdgpu_gfx void @test_call_external_void_func_f64_imm_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_f64_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_f64_inreg@abs32@lo
-; GFX11-NEXT:    s_mov_b32 s0, 0
-; GFX11-NEXT:    s_mov_b32 s1, 0x40100000
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 4
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_f64_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_f64_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    s_mov_b32 s5, 0x40100000
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -11348,18 +11586,22 @@ define amdgpu_gfx void @test_call_external_void_func_f64_imm_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_f64_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_f64_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 0x40100000
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 4
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_f64_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_f64_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 0x40100000
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -11381,20 +11623,28 @@ define amdgpu_gfx void @test_call_external_void_func_v2f64_imm_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 6
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 4
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v2f64_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v2f64_inreg@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s2, 0
-; GFX9-NEXT:    s_mov_b32 s0, 0
-; GFX9-NEXT:    s_mov_b32 s1, 2.0
-; GFX9-NEXT:    s_mov_b32 s3, 0x40100000
+; GFX9-NEXT:    s_mov_b32 s4, 0
+; GFX9-NEXT:    s_mov_b32 s5, 2.0
+; GFX9-NEXT:    s_mov_b32 s6, 0
+; GFX9-NEXT:    s_mov_b32 s7, 0x40100000
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 5
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX9-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 6
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -11412,20 +11662,28 @@ define amdgpu_gfx void @test_call_external_void_func_v2f64_imm_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 6
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v2f64_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v2f64_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s2, 0
-; GFX10-NEXT:    s_mov_b32 s0, 0
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    s_mov_b32 s1, 2.0
-; GFX10-NEXT:    s_mov_b32 s3, 0x40100000
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    s_mov_b32 s5, 2.0
+; GFX10-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-NEXT:    s_mov_b32 s6, 0
+; GFX10-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-NEXT:    s_mov_b32 s7, 0x40100000
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 5
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX10-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 6
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -11443,21 +11701,29 @@ define amdgpu_gfx void @test_call_external_void_func_v2f64_imm_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v2f64_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v2f64_inreg@abs32@lo
-; GFX11-NEXT:    s_mov_b32 s0, 0
-; GFX11-NEXT:    s_mov_b32 s1, 2.0
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX11-NEXT:    s_mov_b32 s2, 0
-; GFX11-NEXT:    s_mov_b32 s3, 0x40100000
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 6
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v2f64_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v2f64_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    s_mov_b32 s5, 2.0
+; GFX11-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX11-NEXT:    s_mov_b32 s6, 0
+; GFX11-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX11-NEXT:    s_mov_b32 s7, 0x40100000
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 5
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX11-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 6
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -11475,20 +11741,28 @@ define amdgpu_gfx void @test_call_external_void_func_v2f64_imm_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v2f64_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v2f64_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 2.0
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, 0x40100000
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 6
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v2f64_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v2f64_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 2.0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s6, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s7, 0x40100000
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 5
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 6
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -11510,26 +11784,34 @@ define amdgpu_gfx void @test_call_external_void_func_v3f64_imm_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 8
 ; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 6
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v3f64_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v3f64_inreg@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, 0
-; GFX9-NEXT:    s_mov_b32 s1, 2.0
-; GFX9-NEXT:    s_mov_b32 s2, 0
-; GFX9-NEXT:    s_mov_b32 s3, 0x40100000
 ; GFX9-NEXT:    s_mov_b32 s4, 0
-; GFX9-NEXT:    s_mov_b32 s5, 0x40200000
+; GFX9-NEXT:    s_mov_b32 s5, 2.0
+; GFX9-NEXT:    s_mov_b32 s6, 0
+; GFX9-NEXT:    s_mov_b32 s7, 0x40100000
+; GFX9-NEXT:    s_mov_b32 s8, 0
+; GFX9-NEXT:    s_mov_b32 s9, 0x40200000
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 7
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 3
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 7
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 6
+; GFX9-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX9-NEXT:    v_readlane_b32 s8, v40, 4
+; GFX9-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 4
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 8
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -11547,26 +11829,34 @@ define amdgpu_gfx void @test_call_external_void_func_v3f64_imm_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 4
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 8
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v3f64_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v3f64_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, 0
-; GFX10-NEXT:    s_mov_b32 s1, 2.0
+; GFX10-NEXT:    s_addk_i32 s32, 0x200
 ; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX10-NEXT:    s_mov_b32 s2, 0
-; GFX10-NEXT:    s_mov_b32 s3, 0x40100000
 ; GFX10-NEXT:    s_mov_b32 s4, 0
-; GFX10-NEXT:    s_addk_i32 s32, 0x200
 ; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
-; GFX10-NEXT:    s_mov_b32 s5, 0x40200000
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 2
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX10-NEXT:    s_mov_b32 s5, 2.0
+; GFX10-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-NEXT:    s_mov_b32 s6, 0
+; GFX10-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-NEXT:    s_mov_b32 s7, 0x40100000
+; GFX10-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX10-NEXT:    s_mov_b32 s8, 0
+; GFX10-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX10-NEXT:    s_mov_b32 s9, 0x40200000
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 6
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 7
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 3
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 7
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 6
+; GFX10-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX10-NEXT:    v_readlane_b32 s8, v40, 4
+; GFX10-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 4
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 8
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -11584,27 +11874,35 @@ define amdgpu_gfx void @test_call_external_void_func_v3f64_imm_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 4
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v3f64_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v3f64_inreg@abs32@lo
-; GFX11-NEXT:    s_mov_b32 s0, 0
-; GFX11-NEXT:    s_mov_b32 s1, 2.0
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 8
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v3f64_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v3f64_inreg@abs32@lo
+; GFX11-NEXT:    s_add_i32 s32, s32, 16
 ; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX11-NEXT:    s_mov_b32 s2, 0
-; GFX11-NEXT:    s_mov_b32 s3, 0x40100000
 ; GFX11-NEXT:    s_mov_b32 s4, 0
-; GFX11-NEXT:    s_add_i32 s32, s32, 16
 ; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
-; GFX11-NEXT:    s_mov_b32 s5, 0x40200000
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 2
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 3
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    s_mov_b32 s5, 2.0
+; GFX11-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX11-NEXT:    s_mov_b32 s6, 0
+; GFX11-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX11-NEXT:    s_mov_b32 s7, 0x40100000
+; GFX11-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX11-NEXT:    s_mov_b32 s8, 0
+; GFX11-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX11-NEXT:    s_mov_b32 s9, 0x40200000
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 6
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 7
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 3
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 7
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 6
+; GFX11-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX11-NEXT:    v_readlane_b32 s8, v40, 4
+; GFX11-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 4
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 8
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -11622,26 +11920,34 @@ define amdgpu_gfx void @test_call_external_void_func_v3f64_imm_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 4
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v3f64_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v3f64_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 2.0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 8
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v3f64_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v3f64_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, 0x40100000
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 0
-; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 0x40200000
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 2
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 3
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 3
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 2.0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s6, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s7, 0x40100000
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s8, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s9, 0x40200000
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 6
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 7
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 7
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 6
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s8, v40, 4
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 4
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 8
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -11663,17 +11969,19 @@ define amdgpu_gfx void @test_call_external_void_func_v2i16_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    s_load_dword s0, s[34:35], 0x0
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    s_load_dword s4, s[34:35], 0x0
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 1
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v2i16_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v2i16_inreg@abs32@lo
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 2
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 3
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -11691,17 +11999,19 @@ define amdgpu_gfx void @test_call_external_void_func_v2i16_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    s_load_dword s0, s[34:35], 0x0
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 3
+; GFX10-NEXT:    s_addk_i32 s32, 0x200
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_load_dword s4, s[34:35], 0x0
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v2i16_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v2i16_inreg@abs32@lo
-; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 2
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 3
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -11719,18 +12029,20 @@ define amdgpu_gfx void @test_call_external_void_func_v2i16_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_load_b32 s0, s[0:1], 0x0
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_v2i16_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_v2i16_inreg@abs32@lo
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 3
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x0
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v2i16_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v2i16_inreg@abs32@lo
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 2
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 3
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -11748,17 +12060,19 @@ define amdgpu_gfx void @test_call_external_void_func_v2i16_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_load_dword s0, s[0:1], 0x0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_v2i16_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_v2i16_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 3
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v2i16_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v2i16_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 2
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 3
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -11781,20 +12095,21 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    s_load_dwordx2 s[34:35], s[34:35], 0x0
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s37, external_void_func_v3i16_inreg@abs32@hi
-; GFX9-NEXT:    s_mov_b32 s36, external_void_func_v3i16_inreg@abs32@lo
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[34:35], 0x0
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v3i16_inreg@abs32@hi
+; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v3i16_inreg@abs32@lo
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s1, s35
-; GFX9-NEXT:    s_mov_b32 s0, s34
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[36:37]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -11812,20 +12127,21 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX10-NEXT:    s_load_dwordx2 s[34:35], s[34:35], 0x0
-; GFX10-NEXT:    s_mov_b32 s37, external_void_func_v3i16_inreg@abs32@hi
-; GFX10-NEXT:    s_mov_b32 s36, external_void_func_v3i16_inreg@abs32@lo
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 4
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s1, s35
-; GFX10-NEXT:    s_mov_b32 s0, s34
-; GFX10-NEXT:    s_swappc_b64 s[30:31], s[36:37]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[34:35], 0x0
+; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v3i16_inreg@abs32@hi
+; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v3i16_inreg@abs32@lo
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -11843,18 +12159,22 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_v3i16_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_v3i16_inreg@abs32@lo
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 4
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v3i16_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v3i16_inreg@abs32@lo
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -11872,17 +12192,21 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_v3i16_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_v3i16_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 4
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v3i16_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v3i16_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -11905,20 +12229,21 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    s_load_dwordx2 s[34:35], s[34:35], 0x0
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s37, external_void_func_v3f16_inreg@abs32@hi
-; GFX9-NEXT:    s_mov_b32 s36, external_void_func_v3f16_inreg@abs32@lo
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[34:35], 0x0
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v3f16_inreg@abs32@hi
+; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v3f16_inreg@abs32@lo
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s1, s35
-; GFX9-NEXT:    s_mov_b32 s0, s34
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[36:37]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -11936,20 +12261,21 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX10-NEXT:    s_load_dwordx2 s[34:35], s[34:35], 0x0
-; GFX10-NEXT:    s_mov_b32 s37, external_void_func_v3f16_inreg@abs32@hi
-; GFX10-NEXT:    s_mov_b32 s36, external_void_func_v3f16_inreg@abs32@lo
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 4
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s1, s35
-; GFX10-NEXT:    s_mov_b32 s0, s34
-; GFX10-NEXT:    s_swappc_b64 s[30:31], s[36:37]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[34:35], 0x0
+; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v3f16_inreg@abs32@hi
+; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v3f16_inreg@abs32@lo
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -11967,18 +12293,22 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_v3f16_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_v3f16_inreg@abs32@lo
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 4
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v3f16_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v3f16_inreg@abs32@lo
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -11996,17 +12326,21 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_v3f16_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_v3f16_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 4
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v3f16_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v3f16_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -12029,18 +12363,22 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_imm_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 2
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v3i16_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v3i16_inreg@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, 0x20001
-; GFX9-NEXT:    s_mov_b32 s1, 3
+; GFX9-NEXT:    s_mov_b32 s4, 0x20001
+; GFX9-NEXT:    s_mov_b32 s5, 3
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 3
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -12058,18 +12396,22 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_imm_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 4
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v3i16_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v3i16_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, 0x20001
-; GFX10-NEXT:    s_mov_b32 s1, 3
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_mov_b32 s4, 0x20001
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    s_mov_b32 s5, 3
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 3
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -12087,19 +12429,23 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_imm_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_v3i16_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_v3i16_inreg@abs32@lo
-; GFX11-NEXT:    s_mov_b32 s0, 0x20001
-; GFX11-NEXT:    s_mov_b32 s1, 3
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 4
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v3i16_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v3i16_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_mov_b32 s4, 0x20001
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    s_mov_b32 s5, 3
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -12117,18 +12463,22 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_imm_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_v3i16_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_v3i16_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 0x20001
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 3
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 4
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v3i16_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v3i16_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 0x20001
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 3
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -12150,18 +12500,22 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_imm_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 2
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v3f16_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v3f16_inreg@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, 0x40003c00
-; GFX9-NEXT:    s_movk_i32 s1, 0x4400
+; GFX9-NEXT:    s_mov_b32 s4, 0x40003c00
+; GFX9-NEXT:    s_movk_i32 s5, 0x4400
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 3
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -12179,18 +12533,22 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_imm_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 4
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v3f16_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v3f16_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, 0x40003c00
-; GFX10-NEXT:    s_movk_i32 s1, 0x4400
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_mov_b32 s4, 0x40003c00
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    s_movk_i32 s5, 0x4400
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 3
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -12208,19 +12566,23 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_imm_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_v3f16_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_v3f16_inreg@abs32@lo
-; GFX11-NEXT:    s_mov_b32 s0, 0x40003c00
-; GFX11-NEXT:    s_movk_i32 s1, 0x4400
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 4
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v3f16_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v3f16_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_mov_b32 s4, 0x40003c00
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    s_movk_i32 s5, 0x4400
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -12238,18 +12600,22 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_imm_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_v3f16_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_v3f16_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 0x40003c00
-; GFX10-SCRATCH-NEXT:    s_movk_i32 s1, 0x4400
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 4
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v3f16_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v3f16_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 0x40003c00
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    s_movk_i32 s5, 0x4400
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -12271,20 +12637,21 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    s_load_dwordx2 s[34:35], s[34:35], 0x0
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s37, external_void_func_v4i16_inreg@abs32@hi
-; GFX9-NEXT:    s_mov_b32 s36, external_void_func_v4i16_inreg@abs32@lo
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[34:35], 0x0
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v4i16_inreg@abs32@hi
+; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v4i16_inreg@abs32@lo
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s0, s34
-; GFX9-NEXT:    s_mov_b32 s1, s35
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[36:37]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -12302,20 +12669,21 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX10-NEXT:    s_load_dwordx2 s[34:35], s[34:35], 0x0
-; GFX10-NEXT:    s_mov_b32 s37, external_void_func_v4i16_inreg@abs32@hi
-; GFX10-NEXT:    s_mov_b32 s36, external_void_func_v4i16_inreg@abs32@lo
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 4
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s0, s34
-; GFX10-NEXT:    s_mov_b32 s1, s35
-; GFX10-NEXT:    s_swappc_b64 s[30:31], s[36:37]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[34:35], 0x0
+; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v4i16_inreg@abs32@hi
+; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v4i16_inreg@abs32@lo
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -12333,18 +12701,22 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_v4i16_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_v4i16_inreg@abs32@lo
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 4
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v4i16_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v4i16_inreg@abs32@lo
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -12362,19 +12734,23 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_v4i16_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_v4i16_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 4
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
-; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
-; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v4i16_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v4i16_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 4
+; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
+; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, -16
@@ -12395,18 +12771,22 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_imm_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 2
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v4i16_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v4i16_inreg@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, 0x20001
-; GFX9-NEXT:    s_mov_b32 s1, 0x40003
+; GFX9-NEXT:    s_mov_b32 s4, 0x20001
+; GFX9-NEXT:    s_mov_b32 s5, 0x40003
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 3
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -12424,18 +12804,22 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_imm_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 4
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v4i16_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v4i16_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, 0x20001
-; GFX10-NEXT:    s_mov_b32 s1, 0x40003
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_mov_b32 s4, 0x20001
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    s_mov_b32 s5, 0x40003
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 3
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -12453,19 +12837,23 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_imm_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_v4i16_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_v4i16_inreg@abs32@lo
-; GFX11-NEXT:    s_mov_b32 s0, 0x20001
-; GFX11-NEXT:    s_mov_b32 s1, 0x40003
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 4
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v4i16_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v4i16_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_mov_b32 s4, 0x20001
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    s_mov_b32 s5, 0x40003
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -12483,18 +12871,22 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_imm_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_v4i16_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_v4i16_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 0x20001
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 0x40003
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 4
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v4i16_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v4i16_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 0x20001
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 0x40003
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -12516,17 +12908,19 @@ define amdgpu_gfx void @test_call_external_void_func_v2f16_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    s_load_dword s0, s[34:35], 0x0
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    s_load_dword s4, s[34:35], 0x0
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 1
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v2f16_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v2f16_inreg@abs32@lo
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 2
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 3
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -12544,17 +12938,19 @@ define amdgpu_gfx void @test_call_external_void_func_v2f16_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    s_load_dword s0, s[34:35], 0x0
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 3
+; GFX10-NEXT:    s_addk_i32 s32, 0x200
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_load_dword s4, s[34:35], 0x0
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v2f16_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v2f16_inreg@abs32@lo
-; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 2
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 3
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -12572,18 +12968,20 @@ define amdgpu_gfx void @test_call_external_void_func_v2f16_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_load_b32 s0, s[0:1], 0x0
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_v2f16_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_v2f16_inreg@abs32@lo
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 3
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x0
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v2f16_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v2f16_inreg@abs32@lo
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 2
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 3
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -12601,17 +12999,19 @@ define amdgpu_gfx void @test_call_external_void_func_v2f16_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_load_dword s0, s[0:1], 0x0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_v2f16_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_v2f16_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 3
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v2f16_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v2f16_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 2
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 3
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -12634,20 +13034,21 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    s_load_dwordx2 s[34:35], s[34:35], 0x0
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX9-NEXT:    s_mov_b32 s37, external_void_func_v2i32_inreg@abs32@hi
-; GFX9-NEXT:    s_mov_b32 s36, external_void_func_v2i32_inreg@abs32@lo
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[34:35], 0x0
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v2i32_inreg@abs32@hi
+; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v2i32_inreg@abs32@lo
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s0, s34
-; GFX9-NEXT:    s_mov_b32 s1, s35
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[36:37]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -12665,20 +13066,21 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX10-NEXT:    s_load_dwordx2 s[34:35], s[34:35], 0x0
-; GFX10-NEXT:    s_mov_b32 s37, external_void_func_v2i32_inreg@abs32@hi
-; GFX10-NEXT:    s_mov_b32 s36, external_void_func_v2i32_inreg@abs32@lo
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 4
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s0, s34
-; GFX10-NEXT:    s_mov_b32 s1, s35
-; GFX10-NEXT:    s_swappc_b64 s[30:31], s[36:37]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[34:35], 0x0
+; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v2i32_inreg@abs32@hi
+; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v2i32_inreg@abs32@lo
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -12696,18 +13098,22 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_v2i32_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_v2i32_inreg@abs32@lo
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 4
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v2i32_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v2i32_inreg@abs32@lo
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -12725,17 +13131,21 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_v2i32_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_v2i32_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 4
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v2i32_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v2i32_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -12758,18 +13168,22 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_imm_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 2
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v2i32_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v2i32_inreg@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, 1
-; GFX9-NEXT:    s_mov_b32 s1, 2
+; GFX9-NEXT:    s_mov_b32 s4, 1
+; GFX9-NEXT:    s_mov_b32 s5, 2
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 3
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -12787,18 +13201,22 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_imm_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 4
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v2i32_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v2i32_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, 1
-; GFX10-NEXT:    s_mov_b32 s1, 2
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_mov_b32 s4, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    s_mov_b32 s5, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 3
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 4
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -12816,19 +13234,23 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_imm_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_v2i32_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_v2i32_inreg@abs32@lo
-; GFX11-NEXT:    s_mov_b32 s0, 1
-; GFX11-NEXT:    s_mov_b32 s1, 2
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 4
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v2i32_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v2i32_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_mov_b32 s4, 1
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    s_mov_b32 s5, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -12846,18 +13268,22 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_imm_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_v2i32_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_v2i32_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 1
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 2
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 4
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v2i32_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v2i32_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 3
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 4
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -12879,19 +13305,25 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_imm_inreg(i32) #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 5
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 3
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v3i32_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v3i32_inreg@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, 3
-; GFX9-NEXT:    s_mov_b32 s1, 4
-; GFX9-NEXT:    s_mov_b32 s2, 5
+; GFX9-NEXT:    s_mov_b32 s4, 3
+; GFX9-NEXT:    s_mov_b32 s5, 4
+; GFX9-NEXT:    s_mov_b32 s6, 5
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 4
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 4
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 5
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -12909,19 +13341,25 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_imm_inreg(i32) #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 5
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v3i32_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v3i32_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, 3
-; GFX10-NEXT:    s_mov_b32 s1, 4
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    s_mov_b32 s2, 5
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_mov_b32 s4, 3
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    s_mov_b32 s5, 4
+; GFX10-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-NEXT:    s_mov_b32 s6, 5
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 3
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 4
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 4
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 5
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -12939,20 +13377,26 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_imm_inreg(i32) #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v3i32_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v3i32_inreg@abs32@lo
-; GFX11-NEXT:    s_mov_b32 s0, 3
-; GFX11-NEXT:    s_mov_b32 s1, 4
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX11-NEXT:    s_mov_b32 s2, 5
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 5
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v3i32_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v3i32_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_mov_b32 s4, 3
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    s_mov_b32 s5, 4
+; GFX11-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX11-NEXT:    s_mov_b32 s6, 5
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 3
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 4
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 4
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 5
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -12970,19 +13414,25 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_imm_inreg(i32) #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v3i32_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v3i32_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 4
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, 5
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 5
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v3i32_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v3i32_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 3
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 4
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s6, 5
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 3
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 4
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 4
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 5
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -13004,20 +13454,28 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_i32_inreg(i32) #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 6
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 4
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v3i32_i32_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v3i32_i32_inreg@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, 3
-; GFX9-NEXT:    s_mov_b32 s1, 4
-; GFX9-NEXT:    s_mov_b32 s2, 5
-; GFX9-NEXT:    s_mov_b32 s3, 6
+; GFX9-NEXT:    s_mov_b32 s4, 3
+; GFX9-NEXT:    s_mov_b32 s5, 4
+; GFX9-NEXT:    s_mov_b32 s6, 5
+; GFX9-NEXT:    s_mov_b32 s7, 6
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 5
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX9-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 6
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -13035,20 +13493,28 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_i32_inreg(i32) #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 6
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v3i32_i32_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v3i32_i32_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, 3
-; GFX10-NEXT:    s_mov_b32 s1, 4
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    s_mov_b32 s2, 5
-; GFX10-NEXT:    s_mov_b32 s3, 6
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_mov_b32 s4, 3
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    s_mov_b32 s5, 4
+; GFX10-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-NEXT:    s_mov_b32 s6, 5
+; GFX10-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-NEXT:    s_mov_b32 s7, 6
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 5
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX10-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 6
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -13066,21 +13532,29 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_i32_inreg(i32) #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v3i32_i32_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v3i32_i32_inreg@abs32@lo
-; GFX11-NEXT:    s_mov_b32 s0, 3
-; GFX11-NEXT:    s_mov_b32 s1, 4
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX11-NEXT:    s_mov_b32 s2, 5
-; GFX11-NEXT:    s_mov_b32 s3, 6
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 6
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v3i32_i32_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v3i32_i32_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_mov_b32 s4, 3
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    s_mov_b32 s5, 4
+; GFX11-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX11-NEXT:    s_mov_b32 s6, 5
+; GFX11-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX11-NEXT:    s_mov_b32 s7, 6
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 5
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX11-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 6
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -13098,20 +13572,28 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_i32_inreg(i32) #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v3i32_i32_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v3i32_i32_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 4
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, 5
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, 6
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 6
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v3i32_i32_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v3i32_i32_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 3
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 4
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s6, 5
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s7, 6
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 5
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 6
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -13133,22 +13615,25 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    s_load_dwordx4 s[36:39], s[34:35], 0x0
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 6
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[34:35], 0x0
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 4
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s0, s36
-; GFX9-NEXT:    s_mov_b32 s1, s37
-; GFX9-NEXT:    s_mov_b32 s2, s38
-; GFX9-NEXT:    s_mov_b32 s3, s39
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 5
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX9-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 6
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -13166,22 +13651,25 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    s_load_dwordx4 s[36:39], s[34:35], 0x0
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 6
+; GFX10-NEXT:    s_addk_i32 s32, 0x200
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[34:35], 0x0
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo
-; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s0, s36
-; GFX10-NEXT:    s_mov_b32 s1, s37
-; GFX10-NEXT:    s_mov_b32 s2, s38
-; GFX10-NEXT:    s_mov_b32 s3, s39
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 5
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX10-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 6
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -13199,18 +13687,26 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 6
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX11-NEXT:    s_load_b128 s[4:7], s[0:1], 0x0
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v4i32_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v4i32_inreg@abs32@lo
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 5
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX11-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 6
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -13228,17 +13724,25 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 6
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-SCRATCH-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v4i32_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v4i32_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 5
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 6
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -13261,20 +13765,28 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_imm_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 6
+; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 4
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, 1
-; GFX9-NEXT:    s_mov_b32 s1, 2
-; GFX9-NEXT:    s_mov_b32 s2, 3
-; GFX9-NEXT:    s_mov_b32 s3, 4
+; GFX9-NEXT:    s_mov_b32 s4, 1
+; GFX9-NEXT:    s_mov_b32 s5, 2
+; GFX9-NEXT:    s_mov_b32 s6, 3
+; GFX9-NEXT:    s_mov_b32 s7, 4
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 5
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX9-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 6
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -13292,20 +13804,28 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_imm_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 6
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, 1
-; GFX10-NEXT:    s_mov_b32 s1, 2
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    s_mov_b32 s2, 3
-; GFX10-NEXT:    s_mov_b32 s3, 4
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_mov_b32 s4, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    s_mov_b32 s5, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-NEXT:    s_mov_b32 s6, 3
+; GFX10-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-NEXT:    s_mov_b32 s7, 4
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 5
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX10-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 6
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -13323,21 +13843,29 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_imm_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo
-; GFX11-NEXT:    s_mov_b32 s0, 1
-; GFX11-NEXT:    s_mov_b32 s1, 2
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX11-NEXT:    s_mov_b32 s2, 3
-; GFX11-NEXT:    s_mov_b32 s3, 4
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 6
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v4i32_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v4i32_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_mov_b32 s4, 1
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    s_mov_b32 s5, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX11-NEXT:    s_mov_b32 s6, 3
+; GFX11-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX11-NEXT:    s_mov_b32 s7, 4
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 5
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX11-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 6
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -13355,20 +13883,28 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_imm_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 1
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 2
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, 3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, 4
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 6
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v4i32_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v4i32_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s6, 3
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s7, 4
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 5
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 5
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 6
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -13390,23 +13926,31 @@ define amdgpu_gfx void @test_call_external_void_func_v5i32_imm_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 7
 ; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX9-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 5
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v5i32_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v5i32_inreg@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, 1
-; GFX9-NEXT:    s_mov_b32 s1, 2
-; GFX9-NEXT:    s_mov_b32 s2, 3
-; GFX9-NEXT:    s_mov_b32 s3, 4
-; GFX9-NEXT:    s_mov_b32 s4, 5
+; GFX9-NEXT:    s_mov_b32 s4, 1
+; GFX9-NEXT:    s_mov_b32 s5, 2
+; GFX9-NEXT:    s_mov_b32 s6, 3
+; GFX9-NEXT:    s_mov_b32 s7, 4
+; GFX9-NEXT:    s_mov_b32 s8, 5
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 6
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 2
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 6
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 5
+; GFX9-NEXT:    v_readlane_b32 s8, v40, 4
+; GFX9-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 3
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 7
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -13424,23 +13968,31 @@ define amdgpu_gfx void @test_call_external_void_func_v5i32_imm_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 3
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 7
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v5i32_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v5i32_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, 1
-; GFX10-NEXT:    s_mov_b32 s1, 2
-; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX10-NEXT:    s_mov_b32 s2, 3
-; GFX10-NEXT:    s_mov_b32 s3, 4
-; GFX10-NEXT:    s_mov_b32 s4, 5
-; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 1
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 2
+; GFX10-NEXT:    s_addk_i32 s32, 0x200
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_mov_b32 s4, 1
+; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-NEXT:    s_mov_b32 s5, 2
+; GFX10-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-NEXT:    s_mov_b32 s6, 3
+; GFX10-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-NEXT:    s_mov_b32 s7, 4
+; GFX10-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX10-NEXT:    s_mov_b32 s8, 5
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 5
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 6
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 2
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 6
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 5
+; GFX10-NEXT:    v_readlane_b32 s8, v40, 4
+; GFX10-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 3
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 7
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -13458,24 +14010,32 @@ define amdgpu_gfx void @test_call_external_void_func_v5i32_imm_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 3
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v5i32_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v5i32_inreg@abs32@lo
-; GFX11-NEXT:    s_mov_b32 s0, 1
-; GFX11-NEXT:    s_mov_b32 s1, 2
-; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX11-NEXT:    s_mov_b32 s2, 3
-; GFX11-NEXT:    s_mov_b32 s3, 4
-; GFX11-NEXT:    s_mov_b32 s4, 5
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 7
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v5i32_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v5i32_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 1
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 2
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_mov_b32 s4, 1
+; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX11-NEXT:    s_mov_b32 s5, 2
+; GFX11-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX11-NEXT:    s_mov_b32 s6, 3
+; GFX11-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX11-NEXT:    s_mov_b32 s7, 4
+; GFX11-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX11-NEXT:    s_mov_b32 s8, 5
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 5
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 6
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 2
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 6
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 5
+; GFX11-NEXT:    v_readlane_b32 s8, v40, 4
+; GFX11-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 3
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 7
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -13493,23 +14053,31 @@ define amdgpu_gfx void @test_call_external_void_func_v5i32_imm_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v5i32_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v5i32_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 1
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 2
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, 3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, 4
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 5
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 7
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v5i32_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v5i32_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 2
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 2
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 2
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s6, 3
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s7, 4
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s8, 5
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 5
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 6
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 6
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 5
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s8, v40, 4
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s7, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s6, v40, 2
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 3
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 7
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -13531,36 +14099,35 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 6
-; GFX9-NEXT:    s_load_dwordx2 s[34:35], s[34:35], 0x0
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 10
 ; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
 ; GFX9-NEXT:    v_writelane_b32 v40, s6, 2
+; GFX9-NEXT:    s_load_dwordx2 s[34:35], s[34:35], 0x0
 ; GFX9-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX9-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX9-NEXT:    v_writelane_b32 v40, s10, 6
+; GFX9-NEXT:    v_writelane_b32 v40, s11, 7
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx8 s[36:43], s[34:35], 0x0
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX9-NEXT:    s_load_dwordx8 s[4:11], s[34:35], 0x0
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 8
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s0, s36
-; GFX9-NEXT:    s_mov_b32 s1, s37
-; GFX9-NEXT:    s_mov_b32 s2, s38
-; GFX9-NEXT:    s_mov_b32 s3, s39
-; GFX9-NEXT:    s_mov_b32 s4, s40
-; GFX9-NEXT:    s_mov_b32 s5, s41
-; GFX9-NEXT:    s_mov_b32 s6, s42
-; GFX9-NEXT:    s_mov_b32 s7, s43
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 5
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 9
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 5
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 9
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 8
+; GFX9-NEXT:    v_readlane_b32 s11, v40, 7
+; GFX9-NEXT:    v_readlane_b32 s10, v40, 6
+; GFX9-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX9-NEXT:    v_readlane_b32 s8, v40, 4
 ; GFX9-NEXT:    v_readlane_b32 s7, v40, 3
 ; GFX9-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 6
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 10
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -13578,36 +14145,35 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 6
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 10
 ; GFX10-NEXT:    s_load_dwordx2 s[34:35], s[34:35], 0x0
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
 ; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
 ; GFX10-NEXT:    v_writelane_b32 v40, s6, 2
 ; GFX10-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX10-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX10-NEXT:    v_writelane_b32 v40, s10, 6
+; GFX10-NEXT:    v_writelane_b32 v40, s11, 7
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_load_dwordx8 s[36:43], s[34:35], 0x0
+; GFX10-NEXT:    s_load_dwordx8 s[4:11], s[34:35], 0x0
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 4
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 5
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s0, s36
-; GFX10-NEXT:    s_mov_b32 s1, s37
-; GFX10-NEXT:    s_mov_b32 s2, s38
-; GFX10-NEXT:    s_mov_b32 s3, s39
-; GFX10-NEXT:    s_mov_b32 s4, s40
-; GFX10-NEXT:    s_mov_b32 s5, s41
-; GFX10-NEXT:    s_mov_b32 s6, s42
-; GFX10-NEXT:    s_mov_b32 s7, s43
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 8
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 9
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 5
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 9
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 8
+; GFX10-NEXT:    v_readlane_b32 s11, v40, 7
+; GFX10-NEXT:    v_readlane_b32 s10, v40, 6
+; GFX10-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX10-NEXT:    v_readlane_b32 s8, v40, 4
 ; GFX10-NEXT:    v_readlane_b32 s7, v40, 3
 ; GFX10-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 6
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 10
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -13625,28 +14191,36 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 6
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 10
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
 ; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
 ; GFX11-NEXT:    v_writelane_b32 v40, s6, 2
 ; GFX11-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX11-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX11-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX11-NEXT:    v_writelane_b32 v40, s10, 6
+; GFX11-NEXT:    v_writelane_b32 v40, s11, 7
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_load_b256 s[0:7], s[0:1], 0x0
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 4
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 5
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    s_load_b256 s[4:11], s[0:1], 0x0
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v8i32_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v8i32_inreg@abs32@lo
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 8
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 9
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 5
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 9
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 8
+; GFX11-NEXT:    v_readlane_b32 s11, v40, 7
+; GFX11-NEXT:    v_readlane_b32 s10, v40, 6
+; GFX11-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX11-NEXT:    v_readlane_b32 s8, v40, 4
 ; GFX11-NEXT:    v_readlane_b32 s7, v40, 3
 ; GFX11-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 6
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 10
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -13664,27 +14238,35 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 6
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 10
 ; GFX10-SCRATCH-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s6, 2
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s7, 3
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s10, 6
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s11, 7
 ; GFX10-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SCRATCH-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x0
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 4
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 5
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 5
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX10-SCRATCH-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v8i32_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v8i32_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 8
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 9
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 9
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 8
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s11, v40, 7
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s10, v40, 6
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s8, v40, 4
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s7, v40, 3
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 6
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 10
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -13708,32 +14290,40 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_imm_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 6
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 10
 ; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
 ; GFX9-NEXT:    v_writelane_b32 v40, s6, 2
 ; GFX9-NEXT:    v_writelane_b32 v40, s7, 3
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX9-NEXT:    v_writelane_b32 v40, s10, 6
+; GFX9-NEXT:    v_writelane_b32 v40, s11, 7
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 8
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, 1
-; GFX9-NEXT:    s_mov_b32 s1, 2
-; GFX9-NEXT:    s_mov_b32 s2, 3
-; GFX9-NEXT:    s_mov_b32 s3, 4
-; GFX9-NEXT:    s_mov_b32 s4, 5
-; GFX9-NEXT:    s_mov_b32 s5, 6
-; GFX9-NEXT:    s_mov_b32 s6, 7
-; GFX9-NEXT:    s_mov_b32 s7, 8
+; GFX9-NEXT:    s_mov_b32 s4, 1
+; GFX9-NEXT:    s_mov_b32 s5, 2
+; GFX9-NEXT:    s_mov_b32 s6, 3
+; GFX9-NEXT:    s_mov_b32 s7, 4
+; GFX9-NEXT:    s_mov_b32 s8, 5
+; GFX9-NEXT:    s_mov_b32 s9, 6
+; GFX9-NEXT:    s_mov_b32 s10, 7
+; GFX9-NEXT:    s_mov_b32 s11, 8
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 5
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 9
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 5
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 9
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 8
+; GFX9-NEXT:    v_readlane_b32 s11, v40, 7
+; GFX9-NEXT:    v_readlane_b32 s10, v40, 6
+; GFX9-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX9-NEXT:    v_readlane_b32 s8, v40, 4
 ; GFX9-NEXT:    v_readlane_b32 s7, v40, 3
 ; GFX9-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 6
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 10
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -13751,32 +14341,40 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_imm_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 6
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 10
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, 1
-; GFX10-NEXT:    s_mov_b32 s1, 2
-; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX10-NEXT:    s_mov_b32 s2, 3
-; GFX10-NEXT:    s_mov_b32 s3, 4
-; GFX10-NEXT:    s_mov_b32 s4, 5
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
+; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-NEXT:    s_mov_b32 s4, 1
 ; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
-; GFX10-NEXT:    s_mov_b32 s5, 6
+; GFX10-NEXT:    s_mov_b32 s5, 2
 ; GFX10-NEXT:    v_writelane_b32 v40, s6, 2
-; GFX10-NEXT:    s_mov_b32 s6, 7
+; GFX10-NEXT:    s_mov_b32 s6, 3
 ; GFX10-NEXT:    v_writelane_b32 v40, s7, 3
-; GFX10-NEXT:    s_mov_b32 s7, 8
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 4
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 5
+; GFX10-NEXT:    s_mov_b32 s7, 4
+; GFX10-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX10-NEXT:    s_mov_b32 s8, 5
+; GFX10-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX10-NEXT:    s_mov_b32 s9, 6
+; GFX10-NEXT:    v_writelane_b32 v40, s10, 6
+; GFX10-NEXT:    s_mov_b32 s10, 7
+; GFX10-NEXT:    v_writelane_b32 v40, s11, 7
+; GFX10-NEXT:    s_mov_b32 s11, 8
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 8
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 9
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 5
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 9
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 8
+; GFX10-NEXT:    v_readlane_b32 s11, v40, 7
+; GFX10-NEXT:    v_readlane_b32 s10, v40, 6
+; GFX10-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX10-NEXT:    v_readlane_b32 s8, v40, 4
 ; GFX10-NEXT:    v_readlane_b32 s7, v40, 3
 ; GFX10-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 6
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 10
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -13794,33 +14392,41 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_imm_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 6
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo
-; GFX11-NEXT:    s_mov_b32 s0, 1
-; GFX11-NEXT:    s_mov_b32 s1, 2
-; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX11-NEXT:    s_mov_b32 s2, 3
-; GFX11-NEXT:    s_mov_b32 s3, 4
-; GFX11-NEXT:    s_mov_b32 s4, 5
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 10
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v8i32_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v8i32_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
+; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX11-NEXT:    s_mov_b32 s4, 1
 ; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
-; GFX11-NEXT:    s_mov_b32 s5, 6
+; GFX11-NEXT:    s_mov_b32 s5, 2
 ; GFX11-NEXT:    v_writelane_b32 v40, s6, 2
-; GFX11-NEXT:    s_mov_b32 s6, 7
+; GFX11-NEXT:    s_mov_b32 s6, 3
 ; GFX11-NEXT:    v_writelane_b32 v40, s7, 3
-; GFX11-NEXT:    s_mov_b32 s7, 8
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 4
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 5
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    s_mov_b32 s7, 4
+; GFX11-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX11-NEXT:    s_mov_b32 s8, 5
+; GFX11-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX11-NEXT:    s_mov_b32 s9, 6
+; GFX11-NEXT:    v_writelane_b32 v40, s10, 6
+; GFX11-NEXT:    s_mov_b32 s10, 7
+; GFX11-NEXT:    v_writelane_b32 v40, s11, 7
+; GFX11-NEXT:    s_mov_b32 s11, 8
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 8
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 9
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 5
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 9
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 8
+; GFX11-NEXT:    v_readlane_b32 s11, v40, 7
+; GFX11-NEXT:    v_readlane_b32 s10, v40, 6
+; GFX11-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX11-NEXT:    v_readlane_b32 s8, v40, 4
 ; GFX11-NEXT:    v_readlane_b32 s7, v40, 3
 ; GFX11-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 6
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 10
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -13838,32 +14444,40 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_imm_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 6
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, 1
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, 2
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, 3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, 4
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 5
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 10
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v8i32_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v8i32_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s4, 1
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 6
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s5, 2
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s6, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s6, 7
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s6, 3
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s7, 3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s7, 8
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 4
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 5
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 5
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 4
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s7, 4
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s8, 4
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s8, 5
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s9, 5
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s9, 6
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s10, 6
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s10, 7
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s11, 7
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s11, 8
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 8
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 9
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 9
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 8
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s11, v40, 7
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s10, v40, 6
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s9, v40, 5
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s8, v40, 4
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s7, v40, 3
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 6
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 10
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -13885,47 +14499,38 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 {
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 14
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 18
 ; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
 ; GFX9-NEXT:    v_writelane_b32 v40, s6, 2
-; GFX9-NEXT:    s_load_dwordx2 s[34:35], s[34:35], 0x0
 ; GFX9-NEXT:    v_writelane_b32 v40, s7, 3
 ; GFX9-NEXT:    v_writelane_b32 v40, s8, 4
 ; GFX9-NEXT:    v_writelane_b32 v40, s9, 5
 ; GFX9-NEXT:    v_writelane_b32 v40, s10, 6
 ; GFX9-NEXT:    v_writelane_b32 v40, s11, 7
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx16 s[36:51], s[34:35], 0x0
 ; GFX9-NEXT:    v_writelane_b32 v40, s12, 8
 ; GFX9-NEXT:    v_writelane_b32 v40, s13, 9
 ; GFX9-NEXT:    v_writelane_b32 v40, s14, 10
+; GFX9-NEXT:    s_load_dwordx2 s[34:35], s[34:35], 0x0
 ; GFX9-NEXT:    v_writelane_b32 v40, s15, 11
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 12
+; GFX9-NEXT:    v_writelane_b32 v40, s16, 12
+; GFX9-NEXT:    v_writelane_b32 v40, s17, 13
+; GFX9-NEXT:    v_writelane_b32 v40, s18, 14
+; GFX9-NEXT:    v_writelane_b32 v40, s19, 15
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dwordx16 s[4:19], s[34:35], 0x0
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 16
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v16i32_inreg@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v16i32_inreg@abs32@lo
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_mov_b32 s0, s36
-; GFX9-NEXT:    s_mov_b32 s1, s37
-; GFX9-NEXT:    s_mov_b32 s2, s38
-; GFX9-NEXT:    s_mov_b32 s3, s39
-; GFX9-NEXT:    s_mov_b32 s4, s40
-; GFX9-NEXT:    s_mov_b32 s5, s41
-; GFX9-NEXT:    s_mov_b32 s6, s42
-; GFX9-NEXT:    s_mov_b32 s7, s43
-; GFX9-NEXT:    s_mov_b32 s8, s44
-; GFX9-NEXT:    s_mov_b32 s9, s45
-; GFX9-NEXT:    s_mov_b32 s10, s46
-; GFX9-NEXT:    s_mov_b32 s11, s47
-; GFX9-NEXT:    s_mov_b32 s12, s48
-; GFX9-NEXT:    s_mov_b32 s13, s49
-; GFX9-NEXT:    s_mov_b32 s14, s50
-; GFX9-NEXT:    s_mov_b32 s15, s51
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 13
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 17
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 13
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 12
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 17
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 16
+; GFX9-NEXT:    v_readlane_b32 s19, v40, 15
+; GFX9-NEXT:    v_readlane_b32 s18, v40, 14
+; GFX9-NEXT:    v_readlane_b32 s17, v40, 13
+; GFX9-NEXT:    v_readlane_b32 s16, v40, 12
 ; GFX9-NEXT:    v_readlane_b32 s15, v40, 11
 ; GFX9-NEXT:    v_readlane_b32 s14, v40, 10
 ; GFX9-NEXT:    v_readlane_b32 s13, v40, 9
@@ -13938,7 +14543,7 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 {
 ; GFX9-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 14
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 18
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -13956,47 +14561,38 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 {
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 14
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 18
 ; GFX10-NEXT:    s_load_dwordx2 s[34:35], s[34:35], 0x0
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
 ; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
 ; GFX10-NEXT:    v_writelane_b32 v40, s6, 2
 ; GFX10-NEXT:    v_writelane_b32 v40, s7, 3
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_load_dwordx16 s[36:51], s[34:35], 0x0
-; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v16i32_inreg@abs32@hi
-; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v16i32_inreg@abs32@lo
 ; GFX10-NEXT:    v_writelane_b32 v40, s8, 4
 ; GFX10-NEXT:    v_writelane_b32 v40, s9, 5
 ; GFX10-NEXT:    v_writelane_b32 v40, s10, 6
 ; GFX10-NEXT:    v_writelane_b32 v40, s11, 7
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s0, s36
-; GFX10-NEXT:    s_mov_b32 s1, s37
-; GFX10-NEXT:    s_mov_b32 s2, s38
-; GFX10-NEXT:    s_mov_b32 s3, s39
 ; GFX10-NEXT:    v_writelane_b32 v40, s12, 8
-; GFX10-NEXT:    s_mov_b32 s4, s40
-; GFX10-NEXT:    s_mov_b32 s5, s41
-; GFX10-NEXT:    s_mov_b32 s6, s42
-; GFX10-NEXT:    s_mov_b32 s7, s43
 ; GFX10-NEXT:    v_writelane_b32 v40, s13, 9
-; GFX10-NEXT:    s_mov_b32 s8, s44
-; GFX10-NEXT:    s_mov_b32 s9, s45
-; GFX10-NEXT:    s_mov_b32 s10, s46
-; GFX10-NEXT:    s_mov_b32 s11, s47
 ; GFX10-NEXT:    v_writelane_b32 v40, s14, 10
-; GFX10-NEXT:    s_mov_b32 s12, s48
-; GFX10-NEXT:    s_mov_b32 s13, s49
-; GFX10-NEXT:    s_mov_b32 s14, s50
 ; GFX10-NEXT:    v_writelane_b32 v40, s15, 11
-; GFX10-NEXT:    s_mov_b32 s15, s51
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 12
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 13
+; GFX10-NEXT:    v_writelane_b32 v40, s16, 12
+; GFX10-NEXT:    v_writelane_b32 v40, s17, 13
+; GFX10-NEXT:    v_writelane_b32 v40, s18, 14
+; GFX10-NEXT:    v_writelane_b32 v40, s19, 15
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_load_dwordx16 s[4:19], s[34:35], 0x0
+; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v16i32_inreg@abs32@hi
+; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v16i32_inreg@abs32@lo
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 16
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 17
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 13
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 12
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 17
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 16
+; GFX10-NEXT:    v_readlane_b32 s19, v40, 15
+; GFX10-NEXT:    v_readlane_b32 s18, v40, 14
+; GFX10-NEXT:    v_readlane_b32 s17, v40, 13
+; GFX10-NEXT:    v_readlane_b32 s16, v40, 12
 ; GFX10-NEXT:    v_readlane_b32 s15, v40, 11
 ; GFX10-NEXT:    v_readlane_b32 s14, v40, 10
 ; GFX10-NEXT:    v_readlane_b32 s13, v40, 9
@@ -14009,7 +14605,7 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 {
 ; GFX10-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 14
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 18
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -14027,10 +14623,8 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 {
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
-; GFX11-NEXT:    v_writelane_b32 v40, s0, 14
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 18
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v16i32_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v16i32_inreg@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
 ; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
@@ -14044,14 +14638,24 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 {
 ; GFX11-NEXT:    v_writelane_b32 v40, s13, 9
 ; GFX11-NEXT:    v_writelane_b32 v40, s14, 10
 ; GFX11-NEXT:    v_writelane_b32 v40, s15, 11
+; GFX11-NEXT:    v_writelane_b32 v40, s16, 12
+; GFX11-NEXT:    v_writelane_b32 v40, s17, 13
+; GFX11-NEXT:    v_writelane_b32 v40, s18, 14
+; GFX11-NEXT:    v_writelane_b32 v40, s19, 15
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_load_b512 s[0:15], s[0:1], 0x0
-; GFX11-NEXT:    v_writelane_b32 v40, s30, 12
-; GFX11-NEXT:    v_writelane_b32 v40, s31, 13
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    s_load_b512 s[4:19], s[0:1], 0x0
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v16i32_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v16i32_inreg@abs32@lo
+; GFX11-NEXT:    v_writelane_b32 v40, s30, 16
+; GFX11-NEXT:    v_writelane_b32 v40, s31, 17
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_readlane_b32 s31, v40, 13
-; GFX11-NEXT:    v_readlane_b32 s30, v40, 12
+; GFX11-NEXT:    v_readlane_b32 s31, v40, 17
+; GFX11-NEXT:    v_readlane_b32 s30, v40, 16
+; GFX11-NEXT:    v_readlane_b32 s19, v40, 15
+; GFX11-NEXT:    v_readlane_b32 s18, v40, 14
+; GFX11-NEXT:    v_readlane_b32 s17, v40, 13
+; GFX11-NEXT:    v_readlane_b32 s16, v40, 12
 ; GFX11-NEXT:    v_readlane_b32 s15, v40, 11
 ; GFX11-NEXT:    v_readlane_b32 s14, v40, 10
 ; GFX11-NEXT:    v_readlane_b32 s13, v40, 9
@@ -14064,7 +14668,7 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 {
 ; GFX11-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX11-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX11-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX11-NEXT:    v_readlane_b32 s0, v40, 14
+; GFX11-NEXT:    v_readlane_b32 s0, v40, 18
 ; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
 ; GFX11-NEXT:    s_mov_b32 exec_lo, s1
@@ -14082,10 +14686,8 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 14
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 18
 ; GFX10-SCRATCH-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v16i32_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v16i32_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
@@ -14099,13 +14701,23 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s13, 9
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s14, 10
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s15, 11
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s16, 12
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s17, 13
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s18, 14
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s19, 15
 ; GFX10-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SCRATCH-NEXT:    s_load_dwordx16 s[0:15], s[0:1], 0x0
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 12
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 13
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 13
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 12
+; GFX10-SCRATCH-NEXT:    s_load_dwordx16 s[4:19], s[0:1], 0x0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v16i32_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v16i32_inreg@abs32@lo
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 16
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 17
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 17
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 16
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s19, v40, 15
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s18, v40, 14
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s17, v40, 13
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s16, v40, 12
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s15, v40, 11
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s14, v40, 10
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s13, v40, 9
@@ -14118,7 +14730,7 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s6, v40, 2
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s5, v40, 1
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 14
+; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 18
 ; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_load_dword v40, off, s33 ; 4-byte Folded Reload
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
@@ -14159,47 +14771,49 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_inreg() #0 {
 ; GFX9-NEXT:    v_writelane_b32 v40, s17, 13
 ; GFX9-NEXT:    v_writelane_b32 v40, s18, 14
 ; GFX9-NEXT:    v_writelane_b32 v40, s19, 15
+; GFX9-NEXT:    s_load_dwordx2 s[34:35], s[34:35], 0x0
 ; GFX9-NEXT:    v_writelane_b32 v40, s20, 16
 ; GFX9-NEXT:    v_writelane_b32 v40, s21, 17
 ; GFX9-NEXT:    v_writelane_b32 v40, s22, 18
 ; GFX9-NEXT:    v_writelane_b32 v40, s23, 19
 ; GFX9-NEXT:    v_writelane_b32 v40, s24, 20
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dwordx16 s[36:51], s[34:35], 0x40
+; GFX9-NEXT:    s_load_dwordx16 s[4:19], s[34:35], 0x0
 ; GFX9-NEXT:    v_writelane_b32 v40, s25, 21
 ; GFX9-NEXT:    v_writelane_b32 v40, s26, 22
-; GFX9-NEXT:    s_load_dwordx2 s[34:35], s[34:35], 0x0
 ; GFX9-NEXT:    v_writelane_b32 v40, s27, 23
-; GFX9-NEXT:    v_writelane_b32 v40, s28, 24
-; GFX9-NEXT:    v_writelane_b32 v40, s29, 25
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 26
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 27
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx16 s[16:31], s[34:35], 0x40
-; GFX9-NEXT:    s_load_dwordx16 s[36:51], s[34:35], 0x0
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    s_mov_b32 s53, external_void_func_v32i32_inreg@abs32@hi
-; GFX9-NEXT:    s_mov_b32 s52, external_void_func_v32i32_inreg@abs32@lo
+; GFX9-NEXT:    v_writelane_b32 v40, s28, 24
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s30
-; GFX9-NEXT:    v_mov_b32_e32 v1, s31
+; GFX9-NEXT:    v_mov_b32_e32 v0, s46
+; GFX9-NEXT:    v_writelane_b32 v40, s29, 25
+; GFX9-NEXT:    v_mov_b32_e32 v1, s47
+; GFX9-NEXT:    v_mov_b32_e32 v2, s48
+; GFX9-NEXT:    v_mov_b32_e32 v3, s49
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32
 ; GFX9-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:4
-; GFX9-NEXT:    s_mov_b32 s0, s36
-; GFX9-NEXT:    s_mov_b32 s1, s37
-; GFX9-NEXT:    s_mov_b32 s2, s38
-; GFX9-NEXT:    s_mov_b32 s3, s39
-; GFX9-NEXT:    s_mov_b32 s4, s40
-; GFX9-NEXT:    s_mov_b32 s5, s41
-; GFX9-NEXT:    s_mov_b32 s6, s42
-; GFX9-NEXT:    s_mov_b32 s7, s43
-; GFX9-NEXT:    s_mov_b32 s8, s44
-; GFX9-NEXT:    s_mov_b32 s9, s45
-; GFX9-NEXT:    s_mov_b32 s10, s46
-; GFX9-NEXT:    s_mov_b32 s11, s47
-; GFX9-NEXT:    s_mov_b32 s12, s48
-; GFX9-NEXT:    s_mov_b32 s13, s49
-; GFX9-NEXT:    s_mov_b32 s14, s50
-; GFX9-NEXT:    s_mov_b32 s15, s51
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[52:53]
+; GFX9-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    v_mov_b32_e32 v0, s50
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:16
+; GFX9-NEXT:    v_mov_b32_e32 v0, s51
+; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v32i32_inreg@abs32@hi
+; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v32i32_inreg@abs32@lo
+; GFX9-NEXT:    s_mov_b32 s20, s36
+; GFX9-NEXT:    s_mov_b32 s21, s37
+; GFX9-NEXT:    s_mov_b32 s22, s38
+; GFX9-NEXT:    s_mov_b32 s23, s39
+; GFX9-NEXT:    s_mov_b32 s24, s40
+; GFX9-NEXT:    s_mov_b32 s25, s41
+; GFX9-NEXT:    s_mov_b32 s26, s42
+; GFX9-NEXT:    s_mov_b32 s27, s43
+; GFX9-NEXT:    s_mov_b32 s28, s44
+; GFX9-NEXT:    s_mov_b32 s29, s45
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 27
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 27
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 26
 ; GFX9-NEXT:    v_readlane_b32 s29, v40, 25
@@ -14265,46 +14879,47 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_inreg() #0 {
 ; GFX10-NEXT:    v_writelane_b32 v40, s17, 13
 ; GFX10-NEXT:    v_writelane_b32 v40, s18, 14
 ; GFX10-NEXT:    v_writelane_b32 v40, s19, 15
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_clause 0x1
+; GFX10-NEXT:    s_load_dwordx16 s[36:51], s[34:35], 0x40
+; GFX10-NEXT:    s_load_dwordx16 s[4:19], s[34:35], 0x0
+; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v32i32_inreg@abs32@hi
+; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v32i32_inreg@abs32@lo
 ; GFX10-NEXT:    v_writelane_b32 v40, s20, 16
 ; GFX10-NEXT:    v_writelane_b32 v40, s21, 17
 ; GFX10-NEXT:    v_writelane_b32 v40, s22, 18
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, s46
 ; GFX10-NEXT:    v_writelane_b32 v40, s23, 19
+; GFX10-NEXT:    v_mov_b32_e32 v1, s47
+; GFX10-NEXT:    v_mov_b32_e32 v2, s48
+; GFX10-NEXT:    v_mov_b32_e32 v3, s49
+; GFX10-NEXT:    s_mov_b32 s20, s36
 ; GFX10-NEXT:    v_writelane_b32 v40, s24, 20
+; GFX10-NEXT:    s_mov_b32 s21, s37
+; GFX10-NEXT:    s_mov_b32 s22, s38
+; GFX10-NEXT:    s_mov_b32 s23, s39
+; GFX10-NEXT:    s_mov_b32 s24, s40
 ; GFX10-NEXT:    v_writelane_b32 v40, s25, 21
+; GFX10-NEXT:    s_mov_b32 s25, s41
+; GFX10-NEXT:    v_mov_b32_e32 v4, s50
+; GFX10-NEXT:    v_mov_b32_e32 v5, s51
+; GFX10-NEXT:    buffer_store_dword v0, off, s[0:3], s32
+; GFX10-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:4
+; GFX10-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:8
+; GFX10-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:12
+; GFX10-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:16
+; GFX10-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:20
 ; GFX10-NEXT:    v_writelane_b32 v40, s26, 22
+; GFX10-NEXT:    s_mov_b32 s26, s42
 ; GFX10-NEXT:    v_writelane_b32 v40, s27, 23
+; GFX10-NEXT:    s_mov_b32 s27, s43
 ; GFX10-NEXT:    v_writelane_b32 v40, s28, 24
+; GFX10-NEXT:    s_mov_b32 s28, s44
 ; GFX10-NEXT:    v_writelane_b32 v40, s29, 25
+; GFX10-NEXT:    s_mov_b32 s29, s45
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 26
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 27
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx16 s[16:31], s[34:35], 0x40
-; GFX10-NEXT:    s_load_dwordx16 s[36:51], s[34:35], 0x0
-; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v32i32_inreg@abs32@hi
-; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v32i32_inreg@abs32@lo
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_mov_b32_e32 v0, s30
-; GFX10-NEXT:    v_mov_b32_e32 v1, s31
-; GFX10-NEXT:    s_mov_b32 s4, s40
-; GFX10-NEXT:    s_mov_b32 s5, s41
-; GFX10-NEXT:    buffer_store_dword v0, off, s[0:3], s32
-; GFX10-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:4
-; GFX10-NEXT:    s_mov_b32 s6, s42
-; GFX10-NEXT:    s_mov_b32 s7, s43
-; GFX10-NEXT:    s_mov_b32 s8, s44
-; GFX10-NEXT:    s_mov_b32 s9, s45
-; GFX10-NEXT:    s_mov_b32 s10, s46
-; GFX10-NEXT:    s_mov_b32 s11, s47
-; GFX10-NEXT:    s_mov_b32 s12, s48
-; GFX10-NEXT:    s_mov_b32 s13, s49
-; GFX10-NEXT:    s_mov_b32 s14, s50
-; GFX10-NEXT:    s_mov_b32 s15, s51
-; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_mov_b32 s0, s36
-; GFX10-NEXT:    s_mov_b32 s1, s37
-; GFX10-NEXT:    s_mov_b32 s2, s38
-; GFX10-NEXT:    s_mov_b32 s3, s39
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
 ; GFX10-NEXT:    v_readlane_b32 s31, v40, 27
 ; GFX10-NEXT:    v_readlane_b32 s30, v40, 26
@@ -14355,8 +14970,8 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_inreg() #0 {
 ; GFX11-NEXT:    v_writelane_b32 v40, s0, 28
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v32i32_inreg@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v32i32_inreg@abs32@lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_i32 s2, s32, 16
 ; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
 ; GFX11-NEXT:    v_writelane_b32 v40, s6, 2
@@ -14373,26 +14988,42 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_inreg() #0 {
 ; GFX11-NEXT:    v_writelane_b32 v40, s17, 13
 ; GFX11-NEXT:    v_writelane_b32 v40, s18, 14
 ; GFX11-NEXT:    v_writelane_b32 v40, s19, 15
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b512 s[36:51], s[0:1], 0x40
+; GFX11-NEXT:    s_load_b512 s[4:19], s[0:1], 0x0
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v32i32_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v32i32_inreg@abs32@lo
 ; GFX11-NEXT:    v_writelane_b32 v40, s20, 16
 ; GFX11-NEXT:    v_writelane_b32 v40, s21, 17
 ; GFX11-NEXT:    v_writelane_b32 v40, s22, 18
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v4, s50 :: v_dual_mov_b32 v5, s51
 ; GFX11-NEXT:    v_writelane_b32 v40, s23, 19
+; GFX11-NEXT:    v_dual_mov_b32 v0, s46 :: v_dual_mov_b32 v1, s47
+; GFX11-NEXT:    v_dual_mov_b32 v2, s48 :: v_dual_mov_b32 v3, s49
 ; GFX11-NEXT:    v_writelane_b32 v40, s24, 20
+; GFX11-NEXT:    s_mov_b32 s20, s36
+; GFX11-NEXT:    s_mov_b32 s21, s37
+; GFX11-NEXT:    s_mov_b32 s22, s38
+; GFX11-NEXT:    s_mov_b32 s23, s39
 ; GFX11-NEXT:    v_writelane_b32 v40, s25, 21
+; GFX11-NEXT:    s_mov_b32 s24, s40
+; GFX11-NEXT:    s_mov_b32 s25, s41
+; GFX11-NEXT:    scratch_store_b64 off, v[4:5], s2
+; GFX11-NEXT:    scratch_store_b128 off, v[0:3], s32
 ; GFX11-NEXT:    v_writelane_b32 v40, s26, 22
+; GFX11-NEXT:    s_mov_b32 s26, s42
 ; GFX11-NEXT:    v_writelane_b32 v40, s27, 23
+; GFX11-NEXT:    s_mov_b32 s27, s43
 ; GFX11-NEXT:    v_writelane_b32 v40, s28, 24
+; GFX11-NEXT:    s_mov_b32 s28, s44
 ; GFX11-NEXT:    v_writelane_b32 v40, s29, 25
+; GFX11-NEXT:    s_mov_b32 s29, s45
 ; GFX11-NEXT:    v_writelane_b32 v40, s30, 26
 ; GFX11-NEXT:    v_writelane_b32 v40, s31, 27
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b512 s[16:31], s[0:1], 0x40
-; GFX11-NEXT:    s_load_b512 s[0:15], s[0:1], 0x0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_dual_mov_b32 v0, s30 :: v_dual_mov_b32 v1, s31
-; GFX11-NEXT:    scratch_store_b64 off, v[0:1], s32
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readlane_b32 s31, v40, 27
 ; GFX11-NEXT:    v_readlane_b32 s30, v40, 26
 ; GFX11-NEXT:    v_readlane_b32 s29, v40, 25
@@ -14440,8 +15071,9 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 28
-; GFX10-SCRATCH-NEXT:    s_load_dwordx2 s[34:35], s[0:1], 0x0
+; GFX10-SCRATCH-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
+; GFX10-SCRATCH-NEXT:    s_add_i32 s2, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s6, 2
@@ -14458,29 +15090,44 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_inreg() #0 {
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s17, 13
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s18, 14
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s19, 15
+; GFX10-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SCRATCH-NEXT:    s_clause 0x1
+; GFX10-SCRATCH-NEXT:    s_load_dwordx16 s[36:51], s[0:1], 0x40
+; GFX10-SCRATCH-NEXT:    s_load_dwordx16 s[4:19], s[0:1], 0x0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v32i32_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v32i32_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s20, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s21, 17
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s22, 18
+; GFX10-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v4, s50
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s23, 19
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v5, s51
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, s46
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, s47
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v2, s48
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s24, 20
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v3, s49
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s20, s36
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s21, s37
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s22, s38
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s25, 21
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s23, s39
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s24, s40
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s25, s41
+; GFX10-SCRATCH-NEXT:    scratch_store_dwordx2 off, v[4:5], s2
+; GFX10-SCRATCH-NEXT:    scratch_store_dwordx4 off, v[0:3], s32
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s26, 22
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s26, s42
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s27, 23
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s27, s43
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s28, 24
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s28, s44
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s29, 25
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s29, s45
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 26
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 27
-; GFX10-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SCRATCH-NEXT:    s_clause 0x1
-; GFX10-SCRATCH-NEXT:    s_load_dwordx16 s[16:31], s[34:35], 0x40
-; GFX10-SCRATCH-NEXT:    s_load_dwordx16 s[0:15], s[34:35], 0x0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v32i32_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v32i32_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, s30
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, s31
-; GFX10-SCRATCH-NEXT:    scratch_store_dwordx2 off, v[0:1], s32
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 27
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 26
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s29, v40, 25
@@ -14549,53 +15196,55 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 {
 ; GFX9-NEXT:    v_writelane_b32 v40, s16, 12
 ; GFX9-NEXT:    v_writelane_b32 v40, s17, 13
 ; GFX9-NEXT:    v_writelane_b32 v40, s18, 14
+; GFX9-NEXT:    s_load_dwordx2 s[34:35], s[34:35], 0x0
 ; GFX9-NEXT:    v_writelane_b32 v40, s19, 15
 ; GFX9-NEXT:    v_writelane_b32 v40, s20, 16
 ; GFX9-NEXT:    v_writelane_b32 v40, s21, 17
 ; GFX9-NEXT:    v_writelane_b32 v40, s22, 18
 ; GFX9-NEXT:    v_writelane_b32 v40, s23, 19
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dword s52, s[34:35], 0x0
+; GFX9-NEXT:    ; kill: killed $sgpr34_sgpr35
+; GFX9-NEXT:    ; kill: killed $sgpr34_sgpr35
+; GFX9-NEXT:    s_load_dwordx16 s[36:51], s[34:35], 0x40
+; GFX9-NEXT:    s_load_dwordx16 s[4:19], s[34:35], 0x0
 ; GFX9-NEXT:    v_writelane_b32 v40, s24, 20
-; GFX9-NEXT:    s_load_dwordx2 s[34:35], s[34:35], 0x0
 ; GFX9-NEXT:    v_writelane_b32 v40, s25, 21
+; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s26, 22
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s52
 ; GFX9-NEXT:    v_writelane_b32 v40, s27, 23
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:24
+; GFX9-NEXT:    v_mov_b32_e32 v0, s46
 ; GFX9-NEXT:    v_writelane_b32 v40, s28, 24
+; GFX9-NEXT:    v_mov_b32_e32 v1, s47
+; GFX9-NEXT:    v_mov_b32_e32 v2, s48
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32
+; GFX9-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:4
+; GFX9-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:8
+; GFX9-NEXT:    v_mov_b32_e32 v0, s49
 ; GFX9-NEXT:    v_writelane_b32 v40, s29, 25
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dword s52, s[34:35], 0x0
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:12
+; GFX9-NEXT:    v_mov_b32_e32 v0, s50
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 26
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:16
+; GFX9-NEXT:    v_mov_b32_e32 v0, s51
+; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v32i32_i32_inreg@abs32@hi
+; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v32i32_i32_inreg@abs32@lo
+; GFX9-NEXT:    s_mov_b32 s20, s36
+; GFX9-NEXT:    s_mov_b32 s21, s37
+; GFX9-NEXT:    s_mov_b32 s22, s38
+; GFX9-NEXT:    s_mov_b32 s23, s39
+; GFX9-NEXT:    s_mov_b32 s24, s40
+; GFX9-NEXT:    s_mov_b32 s25, s41
+; GFX9-NEXT:    s_mov_b32 s26, s42
+; GFX9-NEXT:    s_mov_b32 s27, s43
+; GFX9-NEXT:    s_mov_b32 s28, s44
+; GFX9-NEXT:    s_mov_b32 s29, s45
 ; GFX9-NEXT:    v_writelane_b32 v40, s31, 27
-; GFX9-NEXT:    s_load_dwordx16 s[16:31], s[34:35], 0x40
-; GFX9-NEXT:    s_load_dwordx16 s[36:51], s[34:35], 0x0
-; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s52
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:8
-; GFX9-NEXT:    v_mov_b32_e32 v0, s30
-; GFX9-NEXT:    s_mov_b32 s53, external_void_func_v32i32_i32_inreg@abs32@hi
-; GFX9-NEXT:    v_mov_b32_e32 v1, s31
-; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32
-; GFX9-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:4
-; GFX9-NEXT:    s_mov_b32 s52, external_void_func_v32i32_i32_inreg@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, s36
-; GFX9-NEXT:    s_mov_b32 s1, s37
-; GFX9-NEXT:    s_mov_b32 s2, s38
-; GFX9-NEXT:    s_mov_b32 s3, s39
-; GFX9-NEXT:    s_mov_b32 s4, s40
-; GFX9-NEXT:    s_mov_b32 s5, s41
-; GFX9-NEXT:    s_mov_b32 s6, s42
-; GFX9-NEXT:    s_mov_b32 s7, s43
-; GFX9-NEXT:    s_mov_b32 s8, s44
-; GFX9-NEXT:    s_mov_b32 s9, s45
-; GFX9-NEXT:    s_mov_b32 s10, s46
-; GFX9-NEXT:    s_mov_b32 s11, s47
-; GFX9-NEXT:    s_mov_b32 s12, s48
-; GFX9-NEXT:    s_mov_b32 s13, s49
-; GFX9-NEXT:    s_mov_b32 s14, s50
-; GFX9-NEXT:    s_mov_b32 s15, s51
-; GFX9-NEXT:    ; kill: killed $sgpr34_sgpr35
-; GFX9-NEXT:    ; kill: killed $sgpr34_sgpr35
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[52:53]
+; GFX9-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:20
+; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
 ; GFX9-NEXT:    v_readlane_b32 s31, v40, 27
 ; GFX9-NEXT:    v_readlane_b32 s30, v40, 26
 ; GFX9-NEXT:    v_readlane_b32 s29, v40, 25
@@ -14661,51 +15310,52 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 {
 ; GFX10-NEXT:    v_writelane_b32 v40, s17, 13
 ; GFX10-NEXT:    v_writelane_b32 v40, s18, 14
 ; GFX10-NEXT:    v_writelane_b32 v40, s19, 15
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_clause 0x2
+; GFX10-NEXT:    s_load_dword s52, s[34:35], 0x0
+; GFX10-NEXT:    ; meta instruction
+; GFX10-NEXT:    ; meta instruction
+; GFX10-NEXT:    s_load_dwordx16 s[36:51], s[34:35], 0x40
+; GFX10-NEXT:    s_load_dwordx16 s[4:19], s[34:35], 0x0
+; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v32i32_i32_inreg@abs32@hi
+; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v32i32_i32_inreg@abs32@lo
 ; GFX10-NEXT:    v_writelane_b32 v40, s20, 16
 ; GFX10-NEXT:    v_writelane_b32 v40, s21, 17
 ; GFX10-NEXT:    v_writelane_b32 v40, s22, 18
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mov_b32_e32 v0, s52
+; GFX10-NEXT:    v_mov_b32_e32 v1, s47
 ; GFX10-NEXT:    v_writelane_b32 v40, s23, 19
+; GFX10-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:24
+; GFX10-NEXT:    v_mov_b32_e32 v0, s46
+; GFX10-NEXT:    v_mov_b32_e32 v2, s48
+; GFX10-NEXT:    v_mov_b32_e32 v3, s49
 ; GFX10-NEXT:    v_writelane_b32 v40, s24, 20
+; GFX10-NEXT:    s_mov_b32 s20, s36
+; GFX10-NEXT:    s_mov_b32 s21, s37
+; GFX10-NEXT:    s_mov_b32 s22, s38
+; GFX10-NEXT:    s_mov_b32 s23, s39
 ; GFX10-NEXT:    v_writelane_b32 v40, s25, 21
+; GFX10-NEXT:    s_mov_b32 s24, s40
+; GFX10-NEXT:    s_mov_b32 s25, s41
+; GFX10-NEXT:    v_mov_b32_e32 v4, s50
+; GFX10-NEXT:    v_mov_b32_e32 v5, s51
 ; GFX10-NEXT:    v_writelane_b32 v40, s26, 22
+; GFX10-NEXT:    s_mov_b32 s26, s42
+; GFX10-NEXT:    buffer_store_dword v0, off, s[0:3], s32
+; GFX10-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:4
+; GFX10-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:8
+; GFX10-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:12
+; GFX10-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:16
+; GFX10-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:20
 ; GFX10-NEXT:    v_writelane_b32 v40, s27, 23
+; GFX10-NEXT:    s_mov_b32 s27, s43
 ; GFX10-NEXT:    v_writelane_b32 v40, s28, 24
+; GFX10-NEXT:    s_mov_b32 s28, s44
 ; GFX10-NEXT:    v_writelane_b32 v40, s29, 25
+; GFX10-NEXT:    s_mov_b32 s29, s45
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 26
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 27
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_clause 0x2
-; GFX10-NEXT:    s_load_dword s52, s[34:35], 0x0
-; GFX10-NEXT:    ; meta instruction
-; GFX10-NEXT:    ; meta instruction
-; GFX10-NEXT:    s_load_dwordx16 s[16:31], s[34:35], 0x40
-; GFX10-NEXT:    s_load_dwordx16 s[36:51], s[34:35], 0x0
-; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v32i32_i32_inreg@abs32@hi
-; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v32i32_i32_inreg@abs32@lo
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_mov_b32_e32 v0, s52
-; GFX10-NEXT:    v_mov_b32_e32 v1, s30
-; GFX10-NEXT:    v_mov_b32_e32 v2, s31
-; GFX10-NEXT:    s_mov_b32 s4, s40
-; GFX10-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:8
-; GFX10-NEXT:    buffer_store_dword v1, off, s[0:3], s32
-; GFX10-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:4
-; GFX10-NEXT:    s_mov_b32 s5, s41
-; GFX10-NEXT:    s_mov_b32 s6, s42
-; GFX10-NEXT:    s_mov_b32 s7, s43
-; GFX10-NEXT:    s_mov_b32 s8, s44
-; GFX10-NEXT:    s_mov_b32 s9, s45
-; GFX10-NEXT:    s_mov_b32 s10, s46
-; GFX10-NEXT:    s_mov_b32 s11, s47
-; GFX10-NEXT:    s_mov_b32 s12, s48
-; GFX10-NEXT:    s_mov_b32 s13, s49
-; GFX10-NEXT:    s_mov_b32 s14, s50
-; GFX10-NEXT:    s_mov_b32 s15, s51
-; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_mov_b32 s0, s36
-; GFX10-NEXT:    s_mov_b32 s1, s37
-; GFX10-NEXT:    s_mov_b32 s2, s38
-; GFX10-NEXT:    s_mov_b32 s3, s39
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
 ; GFX10-NEXT:    v_readlane_b32 s31, v40, 27
 ; GFX10-NEXT:    v_readlane_b32 s30, v40, 26
@@ -14756,8 +15406,8 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 {
 ; GFX11-NEXT:    v_writelane_b32 v40, s0, 28
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v32i32_i32_inreg@abs32@hi
-; GFX11-NEXT:    s_add_i32 s36, s32, 8
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_i32 s3, s32, 16
 ; GFX11-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX11-NEXT:    v_writelane_b32 v40, s5, 1
 ; GFX11-NEXT:    v_writelane_b32 v40, s6, 2
@@ -14774,30 +15424,46 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 {
 ; GFX11-NEXT:    v_writelane_b32 v40, s17, 13
 ; GFX11-NEXT:    v_writelane_b32 v40, s18, 14
 ; GFX11-NEXT:    v_writelane_b32 v40, s19, 15
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x0
+; GFX11-NEXT:    s_load_b512 s[36:51], s[0:1], 0x40
+; GFX11-NEXT:    s_load_b512 s[4:19], s[0:1], 0x0
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v32i32_i32_inreg@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v32i32_i32_inreg@abs32@lo
 ; GFX11-NEXT:    v_writelane_b32 v40, s20, 16
 ; GFX11-NEXT:    v_writelane_b32 v40, s21, 17
 ; GFX11-NEXT:    v_writelane_b32 v40, s22, 18
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v6, s2 :: v_dual_mov_b32 v5, s51
 ; GFX11-NEXT:    v_writelane_b32 v40, s23, 19
+; GFX11-NEXT:    v_dual_mov_b32 v4, s50 :: v_dual_mov_b32 v1, s47
+; GFX11-NEXT:    v_dual_mov_b32 v0, s46 :: v_dual_mov_b32 v3, s49
 ; GFX11-NEXT:    v_writelane_b32 v40, s24, 20
+; GFX11-NEXT:    v_mov_b32_e32 v2, s48
+; GFX11-NEXT:    s_add_i32 s2, s32, 24
+; GFX11-NEXT:    s_mov_b32 s20, s36
+; GFX11-NEXT:    s_mov_b32 s21, s37
 ; GFX11-NEXT:    v_writelane_b32 v40, s25, 21
+; GFX11-NEXT:    s_mov_b32 s22, s38
+; GFX11-NEXT:    s_mov_b32 s23, s39
+; GFX11-NEXT:    s_mov_b32 s24, s40
+; GFX11-NEXT:    s_mov_b32 s25, s41
 ; GFX11-NEXT:    v_writelane_b32 v40, s26, 22
+; GFX11-NEXT:    s_mov_b32 s26, s42
+; GFX11-NEXT:    scratch_store_b32 off, v6, s2
+; GFX11-NEXT:    scratch_store_b64 off, v[4:5], s3
+; GFX11-NEXT:    scratch_store_b128 off, v[0:3], s32
 ; GFX11-NEXT:    v_writelane_b32 v40, s27, 23
+; GFX11-NEXT:    s_mov_b32 s27, s43
 ; GFX11-NEXT:    v_writelane_b32 v40, s28, 24
+; GFX11-NEXT:    s_mov_b32 s28, s44
 ; GFX11-NEXT:    v_writelane_b32 v40, s29, 25
+; GFX11-NEXT:    s_mov_b32 s29, s45
 ; GFX11-NEXT:    v_writelane_b32 v40, s30, 26
 ; GFX11-NEXT:    v_writelane_b32 v40, s31, 27
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_clause 0x2
-; GFX11-NEXT:    s_load_b32 s34, s[0:1], 0x0
-; GFX11-NEXT:    s_load_b512 s[16:31], s[0:1], 0x40
-; GFX11-NEXT:    s_load_b512 s[0:15], s[0:1], 0x0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_dual_mov_b32 v2, s34 :: v_dual_mov_b32 v1, s31
-; GFX11-NEXT:    v_mov_b32_e32 v0, s30
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v32i32_i32_inreg@abs32@lo
-; GFX11-NEXT:    scratch_store_b32 off, v2, s36
-; GFX11-NEXT:    scratch_store_b64 off, v[0:1], s32
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readlane_b32 s31, v40, 27
 ; GFX11-NEXT:    v_readlane_b32 s30, v40, 26
 ; GFX11-NEXT:    v_readlane_b32 s29, v40, 25
@@ -14845,17 +15511,13 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 {
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 28
-; GFX10-SCRATCH-NEXT:    s_clause 0x1
-; GFX10-SCRATCH-NEXT:    s_load_dwordx2 s[34:35], s[0:1], 0x0
-; GFX10-SCRATCH-NEXT:    s_load_dword s36, s[0:1], 0x0
+; GFX10-SCRATCH-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
+; GFX10-SCRATCH-NEXT:    s_add_i32 s3, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s4, 0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s5, 1
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s6, 2
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s7, 3
-; GFX10-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v2, s36
-; GFX10-SCRATCH-NEXT:    s_add_i32 s36, s32, 8
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s8, 4
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s9, 5
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s10, 6
@@ -14868,29 +15530,50 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 {
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s17, 13
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s18, 14
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s19, 15
+; GFX10-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SCRATCH-NEXT:    s_clause 0x2
+; GFX10-SCRATCH-NEXT:    s_load_dword s2, s[0:1], 0x0
+; GFX10-SCRATCH-NEXT:    ; meta instruction
+; GFX10-SCRATCH-NEXT:    ; meta instruction
+; GFX10-SCRATCH-NEXT:    s_load_dwordx16 s[36:51], s[0:1], 0x40
+; GFX10-SCRATCH-NEXT:    s_load_dwordx16 s[4:19], s[0:1], 0x0
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v32i32_i32_inreg@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v32i32_i32_inreg@abs32@lo
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s20, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s21, 17
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s22, 18
+; GFX10-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v6, s2
+; GFX10-SCRATCH-NEXT:    s_add_i32 s2, s32, 24
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v4, s50
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s23, 19
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v5, s51
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, s46
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, s47
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v2, s48
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s24, 20
+; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v3, s49
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s20, s36
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s21, s37
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s22, s38
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s25, 21
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s23, s39
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s24, s40
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s25, s41
+; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v6, s2
+; GFX10-SCRATCH-NEXT:    scratch_store_dwordx2 off, v[4:5], s3
+; GFX10-SCRATCH-NEXT:    scratch_store_dwordx4 off, v[0:3], s32
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s26, 22
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s26, s42
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s27, 23
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s27, s43
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s28, 24
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s28, s44
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s29, 25
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s29, s45
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 26
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 27
-; GFX10-SCRATCH-NEXT:    s_clause 0x1
-; GFX10-SCRATCH-NEXT:    s_load_dwordx16 s[16:31], s[34:35], 0x40
-; GFX10-SCRATCH-NEXT:    s_load_dwordx16 s[0:15], s[34:35], 0x0
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v32i32_i32_inreg@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v32i32_i32_inreg@abs32@lo
-; GFX10-SCRATCH-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v0, s30
-; GFX10-SCRATCH-NEXT:    v_mov_b32_e32 v1, s31
-; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v2, s36
-; GFX10-SCRATCH-NEXT:    scratch_store_dwordx2 off, v[0:1], s32
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 27
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 26
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s29, v40, 25
@@ -16714,7 +17397,6 @@ define amdgpu_gfx void @test_call_external_void_func_bf16_inreg(i16 inreg %arg)
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_bf16@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_bf16@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, s4
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
@@ -16741,7 +17423,6 @@ define amdgpu_gfx void @test_call_external_void_func_bf16_inreg(i16 inreg %arg)
 ; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_bf16@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_bf16@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, s4
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
@@ -16761,18 +17442,18 @@ define amdgpu_gfx void @test_call_external_void_func_bf16_inreg(i16 inreg %arg)
 ; GFX11-LABEL: test_call_external_void_func_bf16_inreg:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s1, s33
+; GFX11-NEXT:    s_mov_b32 s0, s33
 ; GFX11-NEXT:    s_mov_b32 s33, s32
-; GFX11-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
-; GFX11-NEXT:    s_mov_b32 exec_lo, s2
-; GFX11-NEXT:    v_writelane_b32 v40, s1, 2
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_bf16@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_bf16@abs32@lo
+; GFX11-NEXT:    s_mov_b32 exec_lo, s1
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_bf16@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_bf16@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
 ; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
@@ -16788,19 +17469,19 @@ define amdgpu_gfx void @test_call_external_void_func_bf16_inreg(i16 inreg %arg)
 ; GFX10-SCRATCH-LABEL: test_call_external_void_func_bf16_inreg:
 ; GFX10-SCRATCH:       ; %bb.0:
 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, s33
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, s33
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
-; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s2
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s1, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_bf16@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_bf16@abs32@lo
+; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_bf16@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_bf16@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
@@ -16830,7 +17511,6 @@ define amdgpu_gfx void @test_call_external_void_func_v1bf16_inreg(i16 inreg %arg
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v1bf16@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v1bf16@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, s4
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
@@ -16857,7 +17537,6 @@ define amdgpu_gfx void @test_call_external_void_func_v1bf16_inreg(i16 inreg %arg
 ; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v1bf16@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v1bf16@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, s4
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
@@ -16877,18 +17556,18 @@ define amdgpu_gfx void @test_call_external_void_func_v1bf16_inreg(i16 inreg %arg
 ; GFX11-LABEL: test_call_external_void_func_v1bf16_inreg:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s1, s33
+; GFX11-NEXT:    s_mov_b32 s0, s33
 ; GFX11-NEXT:    s_mov_b32 s33, s32
-; GFX11-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
-; GFX11-NEXT:    s_mov_b32 exec_lo, s2
-; GFX11-NEXT:    v_writelane_b32 v40, s1, 2
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_v1bf16@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_v1bf16@abs32@lo
+; GFX11-NEXT:    s_mov_b32 exec_lo, s1
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v1bf16@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v1bf16@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
 ; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
@@ -16904,19 +17583,19 @@ define amdgpu_gfx void @test_call_external_void_func_v1bf16_inreg(i16 inreg %arg
 ; GFX10-SCRATCH-LABEL: test_call_external_void_func_v1bf16_inreg:
 ; GFX10-SCRATCH:       ; %bb.0:
 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, s33
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, s33
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
-; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s2
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s1, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_v1bf16@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_v1bf16@abs32@lo
+; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v1bf16@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v1bf16@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
@@ -16946,7 +17625,6 @@ define amdgpu_gfx void @test_call_external_void_func_v2bf16_inreg(i32 inreg %arg
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v2bf16@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v2bf16@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s0, s4
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
@@ -16973,7 +17651,6 @@ define amdgpu_gfx void @test_call_external_void_func_v2bf16_inreg(i32 inreg %arg
 ; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v2bf16@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v2bf16@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s0, s4
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
@@ -16993,18 +17670,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2bf16_inreg(i32 inreg %arg
 ; GFX11-LABEL: test_call_external_void_func_v2bf16_inreg:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s1, s33
+; GFX11-NEXT:    s_mov_b32 s0, s33
 ; GFX11-NEXT:    s_mov_b32 s33, s32
-; GFX11-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
-; GFX11-NEXT:    s_mov_b32 exec_lo, s2
-; GFX11-NEXT:    v_writelane_b32 v40, s1, 2
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_v2bf16@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_v2bf16@abs32@lo
+; GFX11-NEXT:    s_mov_b32 exec_lo, s1
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v2bf16@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v2bf16@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
 ; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
@@ -17020,19 +17697,19 @@ define amdgpu_gfx void @test_call_external_void_func_v2bf16_inreg(i32 inreg %arg
 ; GFX10-SCRATCH-LABEL: test_call_external_void_func_v2bf16_inreg:
 ; GFX10-SCRATCH:       ; %bb.0:
 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, s33
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, s33
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
-; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s2, -1
+; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s2
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s1, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_v2bf16@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_v2bf16@abs32@lo
+; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v2bf16@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v2bf16@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
@@ -17062,8 +17739,6 @@ define amdgpu_gfx void @test_call_external_void_func_v3bf16_inreg(<3 x i16> inre
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v3bf16@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v3bf16@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s1, s5
-; GFX9-NEXT:    s_mov_b32 s0, s4
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
@@ -17090,10 +17765,8 @@ define amdgpu_gfx void @test_call_external_void_func_v3bf16_inreg(<3 x i16> inre
 ; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v3bf16@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v3bf16@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s1, s5
-; GFX10-NEXT:    s_mov_b32 s0, s4
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
 ; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
@@ -17111,18 +17784,18 @@ define amdgpu_gfx void @test_call_external_void_func_v3bf16_inreg(<3 x i16> inre
 ; GFX11-LABEL: test_call_external_void_func_v3bf16_inreg:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s2, s33
+; GFX11-NEXT:    s_mov_b32 s0, s33
 ; GFX11-NEXT:    s_mov_b32 s33, s32
-; GFX11-NEXT:    s_or_saveexec_b32 s3, -1
+; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
-; GFX11-NEXT:    s_mov_b32 exec_lo, s3
-; GFX11-NEXT:    v_writelane_b32 v40, s2, 2
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_v3bf16@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_v3bf16@abs32@lo
+; GFX11-NEXT:    s_mov_b32 exec_lo, s1
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v3bf16@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v3bf16@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
 ; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
@@ -17138,19 +17811,19 @@ define amdgpu_gfx void @test_call_external_void_func_v3bf16_inreg(<3 x i16> inre
 ; GFX10-SCRATCH-LABEL: test_call_external_void_func_v3bf16_inreg:
 ; GFX10-SCRATCH:       ; %bb.0:
 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, s33
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, s33
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
-; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s3, -1
+; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s3
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s2, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_v3bf16@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_v3bf16@abs32@lo
+; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v3bf16@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v3bf16@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
@@ -17180,8 +17853,6 @@ define amdgpu_gfx void @test_call_external_void_func_v4bf16_inreg(<4 x i16> inre
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v4bf16@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v4bf16@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s1, s5
-; GFX9-NEXT:    s_mov_b32 s0, s4
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
@@ -17208,10 +17879,8 @@ define amdgpu_gfx void @test_call_external_void_func_v4bf16_inreg(<4 x i16> inre
 ; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v4bf16@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v4bf16@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s1, s5
-; GFX10-NEXT:    s_mov_b32 s0, s4
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
 ; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
@@ -17229,18 +17898,18 @@ define amdgpu_gfx void @test_call_external_void_func_v4bf16_inreg(<4 x i16> inre
 ; GFX11-LABEL: test_call_external_void_func_v4bf16_inreg:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s2, s33
+; GFX11-NEXT:    s_mov_b32 s0, s33
 ; GFX11-NEXT:    s_mov_b32 s33, s32
-; GFX11-NEXT:    s_or_saveexec_b32 s3, -1
+; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
-; GFX11-NEXT:    s_mov_b32 exec_lo, s3
-; GFX11-NEXT:    v_writelane_b32 v40, s2, 2
-; GFX11-NEXT:    s_mov_b32 s3, external_void_func_v4bf16@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s2, external_void_func_v4bf16@abs32@lo
+; GFX11-NEXT:    s_mov_b32 exec_lo, s1
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v4bf16@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v4bf16@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
 ; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
@@ -17256,19 +17925,19 @@ define amdgpu_gfx void @test_call_external_void_func_v4bf16_inreg(<4 x i16> inre
 ; GFX10-SCRATCH-LABEL: test_call_external_void_func_v4bf16_inreg:
 ; GFX10-SCRATCH:       ; %bb.0:
 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, s33
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, s33
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
-; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s3, -1
+; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s3
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s2, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s3, external_void_func_v4bf16@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s2, external_void_func_v4bf16@abs32@lo
+; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v4bf16@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v4bf16@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
@@ -17298,10 +17967,6 @@ define amdgpu_gfx void @test_call_external_void_func_v8bf16_inreg(<8 x i16> inre
 ; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v8bf16@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v8bf16@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s3, s7
-; GFX9-NEXT:    s_mov_b32 s2, s6
-; GFX9-NEXT:    s_mov_b32 s1, s5
-; GFX9-NEXT:    s_mov_b32 s0, s4
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
 ; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
@@ -17328,12 +17993,8 @@ define amdgpu_gfx void @test_call_external_void_func_v8bf16_inreg(<8 x i16> inre
 ; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v8bf16@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v8bf16@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s3, s7
-; GFX10-NEXT:    s_mov_b32 s2, s6
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    s_mov_b32 s1, s5
-; GFX10-NEXT:    s_mov_b32 s0, s4
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
 ; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
@@ -17351,18 +18012,18 @@ define amdgpu_gfx void @test_call_external_void_func_v8bf16_inreg(<8 x i16> inre
 ; GFX11-LABEL: test_call_external_void_func_v8bf16_inreg:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s34, s33
+; GFX11-NEXT:    s_mov_b32 s0, s33
 ; GFX11-NEXT:    s_mov_b32 s33, s32
-; GFX11-NEXT:    s_or_saveexec_b32 s35, -1
+; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
-; GFX11-NEXT:    s_mov_b32 exec_lo, s35
-; GFX11-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v8bf16@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v8bf16@abs32@lo
+; GFX11-NEXT:    s_mov_b32 exec_lo, s1
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v8bf16@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v8bf16@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
 ; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
@@ -17378,19 +18039,19 @@ define amdgpu_gfx void @test_call_external_void_func_v8bf16_inreg(<8 x i16> inre
 ; GFX10-SCRATCH-LABEL: test_call_external_void_func_v8bf16_inreg:
 ; GFX10-SCRATCH:       ; %bb.0:
 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, s33
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, s33
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
-; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s35, -1
+; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v8bf16@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v8bf16@abs32@lo
+; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v8bf16@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v8bf16@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
@@ -17416,32 +18077,16 @@ define amdgpu_gfx void @test_call_external_void_func_v16bf16_inreg(<16 x i16> in
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
-; GFX9-NEXT:    v_writelane_b32 v40, s34, 6
-; GFX9-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX9-NEXT:    v_writelane_b32 v40, s5, 1
-; GFX9-NEXT:    v_writelane_b32 v40, s6, 2
-; GFX9-NEXT:    v_writelane_b32 v40, s7, 3
-; GFX9-NEXT:    v_writelane_b32 v40, s30, 4
+; GFX9-NEXT:    v_writelane_b32 v40, s34, 2
+; GFX9-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX9-NEXT:    s_mov_b32 s35, external_void_func_v16bf16@abs32@hi
 ; GFX9-NEXT:    s_mov_b32 s34, external_void_func_v16bf16@abs32@lo
-; GFX9-NEXT:    s_mov_b32 s3, s7
-; GFX9-NEXT:    s_mov_b32 s2, s6
-; GFX9-NEXT:    s_mov_b32 s1, s5
-; GFX9-NEXT:    s_mov_b32 s0, s4
-; GFX9-NEXT:    s_mov_b32 s4, s8
-; GFX9-NEXT:    s_mov_b32 s5, s9
-; GFX9-NEXT:    s_mov_b32 s6, s10
-; GFX9-NEXT:    s_mov_b32 s7, s11
 ; GFX9-NEXT:    s_addk_i32 s32, 0x400
-; GFX9-NEXT:    v_writelane_b32 v40, s31, 5
+; GFX9-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX9-NEXT:    v_readlane_b32 s31, v40, 5
-; GFX9-NEXT:    v_readlane_b32 s30, v40, 4
-; GFX9-NEXT:    v_readlane_b32 s7, v40, 3
-; GFX9-NEXT:    v_readlane_b32 s6, v40, 2
-; GFX9-NEXT:    v_readlane_b32 s5, v40, 1
-; GFX9-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX9-NEXT:    v_readlane_b32 s34, v40, 6
+; GFX9-NEXT:    v_readlane_b32 s31, v40, 1
+; GFX9-NEXT:    v_readlane_b32 s30, v40, 0
+; GFX9-NEXT:    v_readlane_b32 s34, v40, 2
 ; GFX9-NEXT:    s_or_saveexec_b64 s[36:37], -1
 ; GFX9-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX9-NEXT:    s_mov_b64 exec, s[36:37]
@@ -17459,32 +18104,16 @@ define amdgpu_gfx void @test_call_external_void_func_v16bf16_inreg(<16 x i16> in
 ; GFX10-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 6
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
 ; GFX10-NEXT:    s_mov_b32 s35, external_void_func_v16bf16@abs32@hi
 ; GFX10-NEXT:    s_mov_b32 s34, external_void_func_v16bf16@abs32@lo
-; GFX10-NEXT:    s_mov_b32 s3, s7
-; GFX10-NEXT:    s_mov_b32 s2, s6
-; GFX10-NEXT:    v_writelane_b32 v40, s4, 0
-; GFX10-NEXT:    s_mov_b32 s1, s5
-; GFX10-NEXT:    s_mov_b32 s0, s4
-; GFX10-NEXT:    s_mov_b32 s4, s8
 ; GFX10-NEXT:    s_addk_i32 s32, 0x200
-; GFX10-NEXT:    v_writelane_b32 v40, s5, 1
-; GFX10-NEXT:    s_mov_b32 s5, s9
-; GFX10-NEXT:    v_writelane_b32 v40, s6, 2
-; GFX10-NEXT:    s_mov_b32 s6, s10
-; GFX10-NEXT:    v_writelane_b32 v40, s7, 3
-; GFX10-NEXT:    s_mov_b32 s7, s11
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 4
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 5
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[34:35]
-; GFX10-NEXT:    v_readlane_b32 s31, v40, 5
-; GFX10-NEXT:    v_readlane_b32 s30, v40, 4
-; GFX10-NEXT:    v_readlane_b32 s7, v40, 3
-; GFX10-NEXT:    v_readlane_b32 s6, v40, 2
-; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
-; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
-; GFX10-NEXT:    v_readlane_b32 s34, v40, 6
+; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
+; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
+; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
 ; GFX10-NEXT:    s_or_saveexec_b32 s35, -1
 ; GFX10-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
@@ -17497,18 +18126,18 @@ define amdgpu_gfx void @test_call_external_void_func_v16bf16_inreg(<16 x i16> in
 ; GFX11-LABEL: test_call_external_void_func_v16bf16_inreg:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b32 s34, s33
+; GFX11-NEXT:    s_mov_b32 s0, s33
 ; GFX11-NEXT:    s_mov_b32 s33, s32
-; GFX11-NEXT:    s_or_saveexec_b32 s35, -1
+; GFX11-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
-; GFX11-NEXT:    s_mov_b32 exec_lo, s35
-; GFX11-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX11-NEXT:    s_mov_b32 s35, external_void_func_v16bf16@abs32@hi
-; GFX11-NEXT:    s_mov_b32 s34, external_void_func_v16bf16@abs32@lo
+; GFX11-NEXT:    s_mov_b32 exec_lo, s1
+; GFX11-NEXT:    v_writelane_b32 v40, s0, 2
+; GFX11-NEXT:    s_mov_b32 s1, external_void_func_v16bf16@abs32@hi
+; GFX11-NEXT:    s_mov_b32 s0, external_void_func_v16bf16@abs32@lo
 ; GFX11-NEXT:    s_add_i32 s32, s32, 16
 ; GFX11-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX11-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX11-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX11-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX11-NEXT:    v_readlane_b32 s30, v40, 0
@@ -17524,19 +18153,19 @@ define amdgpu_gfx void @test_call_external_void_func_v16bf16_inreg(<16 x i16> in
 ; GFX10-SCRATCH-LABEL: test_call_external_void_func_v16bf16_inreg:
 ; GFX10-SCRATCH:       ; %bb.0:
 ; GFX10-SCRATCH-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, s33
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, s33
 ; GFX10-SCRATCH-NEXT:    s_mov_b32 s33, s32
-; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s35, -1
+; GFX10-SCRATCH-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX10-SCRATCH-NEXT:    scratch_store_dword off, v40, s33 ; 4-byte Folded Spill
 ; GFX10-SCRATCH-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s35
-; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s34, 2
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s35, external_void_func_v16bf16@abs32@hi
-; GFX10-SCRATCH-NEXT:    s_mov_b32 s34, external_void_func_v16bf16@abs32@lo
+; GFX10-SCRATCH-NEXT:    s_mov_b32 exec_lo, s1
+; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s0, 2
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s1, external_void_func_v16bf16@abs32@hi
+; GFX10-SCRATCH-NEXT:    s_mov_b32 s0, external_void_func_v16bf16@abs32@lo
 ; GFX10-SCRATCH-NEXT:    s_add_i32 s32, s32, 16
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-SCRATCH-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[34:35]
+; GFX10-SCRATCH-NEXT:    s_swappc_b64 s[30:31], s[0:1]
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s30, v40, 0
 ; GFX10-SCRATCH-NEXT:    v_readlane_b32 s0, v40, 2
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-call.ll b/llvm/test/CodeGen/AMDGPU/indirect-call.ll
index 25c684004446..7799b9509ceb 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-call.ll
@@ -847,11 +847,11 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) {
 ; GCN-LABEL: test_indirect_call_vgpr_ptr_inreg_arg:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_mov_b32 s10, s33
+; GCN-NEXT:    s_mov_b32 s5, s33
 ; GCN-NEXT:    s_mov_b32 s33, s32
-; GCN-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GCN-NEXT:    s_or_saveexec_b64 s[6:7], -1
 ; GCN-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
-; GCN-NEXT:    s_mov_b64 exec, s[4:5]
+; GCN-NEXT:    s_mov_b64 exec, s[6:7]
 ; GCN-NEXT:    s_addk_i32 s32, 0x400
 ; GCN-NEXT:    v_writelane_b32 v40, s30, 0
 ; GCN-NEXT:    v_writelane_b32 v40, s31, 1
@@ -885,19 +885,19 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) {
 ; GCN-NEXT:    v_writelane_b32 v40, s61, 29
 ; GCN-NEXT:    v_writelane_b32 v40, s62, 30
 ; GCN-NEXT:    v_writelane_b32 v40, s63, 31
-; GCN-NEXT:    s_mov_b64 s[4:5], exec
+; GCN-NEXT:    s_mov_b64 s[6:7], exec
+; GCN-NEXT:    s_movk_i32 s4, 0x7b
 ; GCN-NEXT:  .LBB6_1: ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT:    v_readfirstlane_b32 s6, v0
-; GCN-NEXT:    v_readfirstlane_b32 s7, v1
-; GCN-NEXT:    v_cmp_eq_u64_e32 vcc, s[6:7], v[0:1]
-; GCN-NEXT:    s_and_saveexec_b64 s[8:9], vcc
-; GCN-NEXT:    s_movk_i32 s0, 0x7b
-; GCN-NEXT:    s_swappc_b64 s[30:31], s[6:7]
+; GCN-NEXT:    v_readfirstlane_b32 s8, v0
+; GCN-NEXT:    v_readfirstlane_b32 s9, v1
+; GCN-NEXT:    v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GCN-NEXT:    s_and_saveexec_b64 s[10:11], vcc
+; GCN-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; GCN-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN-NEXT:    s_xor_b64 exec, exec, s[8:9]
+; GCN-NEXT:    s_xor_b64 exec, exec, s[10:11]
 ; GCN-NEXT:    s_cbranch_execnz .LBB6_1
 ; GCN-NEXT:  ; %bb.2:
-; GCN-NEXT:    s_mov_b64 exec, s[4:5]
+; GCN-NEXT:    s_mov_b64 exec, s[6:7]
 ; GCN-NEXT:    v_readlane_b32 s63, v40, 31
 ; GCN-NEXT:    v_readlane_b32 s62, v40, 30
 ; GCN-NEXT:    v_readlane_b32 s61, v40, 29
@@ -930,22 +930,22 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) {
 ; GCN-NEXT:    v_readlane_b32 s34, v40, 2
 ; GCN-NEXT:    v_readlane_b32 s31, v40, 1
 ; GCN-NEXT:    v_readlane_b32 s30, v40, 0
-; GCN-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GCN-NEXT:    s_or_saveexec_b64 s[6:7], -1
 ; GCN-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
-; GCN-NEXT:    s_mov_b64 exec, s[4:5]
+; GCN-NEXT:    s_mov_b64 exec, s[6:7]
 ; GCN-NEXT:    s_addk_i32 s32, 0xfc00
-; GCN-NEXT:    s_mov_b32 s33, s10
+; GCN-NEXT:    s_mov_b32 s33, s5
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GISEL-LABEL: test_indirect_call_vgpr_ptr_inreg_arg:
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    s_mov_b32 s10, s33
+; GISEL-NEXT:    s_mov_b32 s5, s33
 ; GISEL-NEXT:    s_mov_b32 s33, s32
-; GISEL-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GISEL-NEXT:    s_or_saveexec_b64 s[6:7], -1
 ; GISEL-NEXT:    buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
-; GISEL-NEXT:    s_mov_b64 exec, s[4:5]
+; GISEL-NEXT:    s_mov_b64 exec, s[6:7]
 ; GISEL-NEXT:    s_addk_i32 s32, 0x400
 ; GISEL-NEXT:    v_writelane_b32 v40, s30, 0
 ; GISEL-NEXT:    v_writelane_b32 v40, s31, 1
@@ -979,19 +979,19 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) {
 ; GISEL-NEXT:    v_writelane_b32 v40, s61, 29
 ; GISEL-NEXT:    v_writelane_b32 v40, s62, 30
 ; GISEL-NEXT:    v_writelane_b32 v40, s63, 31
-; GISEL-NEXT:    s_mov_b64 s[4:5], exec
+; GISEL-NEXT:    s_mov_b64 s[6:7], exec
+; GISEL-NEXT:    s_movk_i32 s4, 0x7b
 ; GISEL-NEXT:  .LBB6_1: ; =>This Inner Loop Header: Depth=1
-; GISEL-NEXT:    v_readfirstlane_b32 s6, v0
-; GISEL-NEXT:    v_readfirstlane_b32 s7, v1
-; GISEL-NEXT:    v_cmp_eq_u64_e32 vcc, s[6:7], v[0:1]
-; GISEL-NEXT:    s_and_saveexec_b64 s[8:9], vcc
-; GISEL-NEXT:    s_movk_i32 s0, 0x7b
-; GISEL-NEXT:    s_swappc_b64 s[30:31], s[6:7]
+; GISEL-NEXT:    v_readfirstlane_b32 s8, v0
+; GISEL-NEXT:    v_readfirstlane_b32 s9, v1
+; GISEL-NEXT:    v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
+; GISEL-NEXT:    s_and_saveexec_b64 s[10:11], vcc
+; GISEL-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; GISEL-NEXT:    ; implicit-def: $vgpr0
-; GISEL-NEXT:    s_xor_b64 exec, exec, s[8:9]
+; GISEL-NEXT:    s_xor_b64 exec, exec, s[10:11]
 ; GISEL-NEXT:    s_cbranch_execnz .LBB6_1
 ; GISEL-NEXT:  ; %bb.2:
-; GISEL-NEXT:    s_mov_b64 exec, s[4:5]
+; GISEL-NEXT:    s_mov_b64 exec, s[6:7]
 ; GISEL-NEXT:    v_readlane_b32 s63, v40, 31
 ; GISEL-NEXT:    v_readlane_b32 s62, v40, 30
 ; GISEL-NEXT:    v_readlane_b32 s61, v40, 29
@@ -1024,11 +1024,11 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) {
 ; GISEL-NEXT:    v_readlane_b32 s34, v40, 2
 ; GISEL-NEXT:    v_readlane_b32 s31, v40, 1
 ; GISEL-NEXT:    v_readlane_b32 s30, v40, 0
-; GISEL-NEXT:    s_or_saveexec_b64 s[4:5], -1
+; GISEL-NEXT:    s_or_saveexec_b64 s[6:7], -1
 ; GISEL-NEXT:    buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
-; GISEL-NEXT:    s_mov_b64 exec, s[4:5]
+; GISEL-NEXT:    s_mov_b64 exec, s[6:7]
 ; GISEL-NEXT:    s_addk_i32 s32, 0xfc00
-; GISEL-NEXT:    s_mov_b32 s33, s10
+; GISEL-NEXT:    s_mov_b32 s33, s5
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
   call amdgpu_gfx void %fptr(i32 inreg 123)
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-addrspaces.ll b/llvm/test/CodeGen/AMDGPU/schedule-addrspaces.ll
index 27e9d0fce942..0139c52db1d5 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-addrspaces.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-addrspaces.ll
@@ -10,7 +10,7 @@ define amdgpu_gfx void @example(<4 x i32> inreg %rsrc, ptr addrspace(5) %src, i3
 ; CHECK-NEXT:    scratch_load_b32 v2, v0, off
 ; CHECK-NEXT:    scratch_load_b32 v3, v3, off
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_b64 v[2:3], v1, s[0:3], 0 offen
+; CHECK-NEXT:    buffer_store_b64 v[2:3], v1, s[4:7], 0 offen
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
 
   %x0 = load i32, ptr addrspace(5) %src
diff --git a/llvm/test/CodeGen/AMDGPU/scratch-pointer-sink.ll b/llvm/test/CodeGen/AMDGPU/scratch-pointer-sink.ll
index ec1de0222d5d..cdaac14833e0 100644
--- a/llvm/test/CodeGen/AMDGPU/scratch-pointer-sink.ll
+++ b/llvm/test/CodeGen/AMDGPU/scratch-pointer-sink.ll
@@ -6,7 +6,7 @@ define amdgpu_gfx i32 @sink_scratch_pointer(ptr addrspace(5) %stack, i32 inreg %
 ; GCN-LABEL: sink_scratch_pointer:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_cmp_lg_u32 s0, 0
+; GCN-NEXT:    s_cmp_lg_u32 s4, 0
 ; GCN-NEXT:    s_cbranch_scc0 .LBB0_2
 ; GCN-NEXT:  ; %bb.1: ; %bb2
 ; GCN-NEXT:    scratch_load_b32 v0, v0, off offset:-4
@@ -21,7 +21,7 @@ define amdgpu_gfx i32 @sink_scratch_pointer(ptr addrspace(5) %stack, i32 inreg %
 ; GISEL-LABEL: sink_scratch_pointer:
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    s_cmp_lg_u32 s0, 0
+; GISEL-NEXT:    s_cmp_lg_u32 s4, 0
 ; GISEL-NEXT:    s_cbranch_scc0 .LBB0_2
 ; GISEL-NEXT:  ; %bb.1: ; %bb2
 ; GISEL-NEXT:    scratch_load_b32 v0, v0, off offset:-4
-- 
GitLab


From 77cbc9bf601861ed92e36f7833422e31c22dd159 Mon Sep 17 00:00:00 2001
From: Victor Perez <victor.perez@codeplay.com>
Date: Tue, 26 Mar 2024 11:02:50 +0100
Subject: [PATCH 255/404] [MLIR][LLVM] Add
 `llvm.experimental.constrained.fptrunc` operation (#86260)

Add operation mapping to the LLVM
`llvm.experimental.constrained.fptrunc.*` intrinsic.

The new operation implements the new
`LLVM::FPExceptionBehaviorOpInterface` and
`LLVM::RoundingModeOpInterface` interfaces.

---------

Signed-off-by: Victor Perez <victor.perez@codeplay.com>
---
 mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td | 57 +++++++++++++
 .../mlir/Dialect/LLVMIR/LLVMInterfaces.td     | 67 +++++++++++++++
 .../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td   | 85 +++++++++++++++++++
 .../include/mlir/Dialect/LLVMIR/LLVMOpBase.td |  4 +
 .../include/mlir/Target/LLVMIR/ModuleImport.h |  8 ++
 .../mlir/Target/LLVMIR/ModuleTranslation.h    |  7 ++
 mlir/lib/Target/LLVMIR/ModuleImport.cpp       | 21 +++++
 mlir/lib/Target/LLVMIR/ModuleTranslation.cpp  | 10 +++
 mlir/test/Dialect/LLVMIR/roundtrip.mlir       | 15 ++++
 mlir/test/Target/LLVMIR/Import/intrinsic.ll   | 19 +++++
 .../test/Target/LLVMIR/llvmir-intrinsics.mlir | 31 +++++++
 .../tools/mlir-tblgen/LLVMIRConversionGen.cpp |  4 +
 12 files changed, 328 insertions(+)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
index a7b269eb41ee..04d797031245 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
@@ -705,4 +705,61 @@ def FramePointerKindEnum : LLVM_EnumAttr<
   let cppNamespace = "::mlir::LLVM::framePointerKind";
 }
 
+//===----------------------------------------------------------------------===//
+// RoundingMode
+//===----------------------------------------------------------------------===//
+
+// These values must match llvm::RoundingMode ones.
+// See llvm/include/llvm/ADT/FloatingPointMode.h.
+def RoundTowardZero
+    : LLVM_EnumAttrCase<"TowardZero", "towardzero", "TowardZero", 0>;
+def RoundNearestTiesToEven
+    : LLVM_EnumAttrCase<"NearestTiesToEven", "tonearest", "NearestTiesToEven", 1>;
+def RoundTowardPositive
+    : LLVM_EnumAttrCase<"TowardPositive", "upward", "TowardPositive", 2>;
+def RoundTowardNegative
+    : LLVM_EnumAttrCase<"TowardNegative", "downward", "TowardNegative", 3>;
+def RoundNearestTiesToAway
+    : LLVM_EnumAttrCase<"NearestTiesToAway", "tonearestaway", "NearestTiesToAway", 4>;
+def RoundDynamic
+    : LLVM_EnumAttrCase<"Dynamic", "dynamic", "Dynamic", 7>;
+// Needed as llvm::RoundingMode defines this.
+def RoundInvalid
+    : LLVM_EnumAttrCase<"Invalid", "invalid", "Invalid", -1>;
+
+// RoundingModeAttr should not be used in operations definitions.
+// Use ValidRoundingModeAttr instead.
+def RoundingModeAttr : LLVM_EnumAttr<
+    "RoundingMode",
+    "::llvm::RoundingMode",
+    "LLVM Rounding Mode",
+    [RoundTowardZero, RoundNearestTiesToEven, RoundTowardPositive,
+     RoundTowardNegative, RoundNearestTiesToAway, RoundDynamic, RoundInvalid]> {
+  let cppNamespace = "::mlir::LLVM";
+}
+
+def ValidRoundingModeAttr : ConfinedAttr<RoundingModeAttr, [IntMinValue<0>]>;
+
+//===----------------------------------------------------------------------===//
+// FPExceptionBehavior
+//===----------------------------------------------------------------------===//
+
+// These values must match llvm::fp::ExceptionBehavior ones.
+// See llvm/include/llvm/IR/FPEnv.h.
+def FPExceptionBehaviorIgnore
+    : LLVM_EnumAttrCase<"Ignore", "ignore", "ebIgnore", 0>;
+def FPExceptionBehaviorMayTrap
+    : LLVM_EnumAttrCase<"MayTrap", "maytrap", "ebMayTrap", 1>;
+def FPExceptionBehaviorStrict
+    : LLVM_EnumAttrCase<"Strict", "strict", "ebStrict", 2>;
+
+def FPExceptionBehaviorAttr : LLVM_EnumAttr<
+    "FPExceptionBehavior",
+    "::llvm::fp::ExceptionBehavior",
+    "LLVM Exception Behavior",
+    [FPExceptionBehaviorIgnore, FPExceptionBehaviorMayTrap,
+     FPExceptionBehaviorStrict]> {
+  let cppNamespace = "::mlir::LLVM";
+}
+
 #endif // LLVMIR_ENUMS
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
index e7a1da8ee560..cee752aeb269 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
@@ -290,6 +290,73 @@ def GetResultPtrElementType : OpInterface<"GetResultPtrElementType"> {
   ];
 }
 
+def FPExceptionBehaviorOpInterface : OpInterface<"FPExceptionBehaviorOpInterface"> {
+  let description = [{
+    An interface for operations receiving an exception behavior attribute
+    controlling FP exception behavior.
+  }];
+
+  let cppNamespace = "::mlir::LLVM";
+
+  let methods = [
+    InterfaceMethod<
+      /*desc=*/        "Returns a FPExceptionBehavior attribute for the operation",
+      /*returnType=*/  "FPExceptionBehaviorAttr",
+      /*methodName=*/  "getFPExceptionBehaviorAttr",
+      /*args=*/        (ins),
+      /*methodBody=*/  [{}],
+      /*defaultImpl=*/ [{
+        auto op = cast<ConcreteOp>(this->getOperation());
+        return op.getFpExceptionBehaviorAttr();
+      }]
+    >,
+    StaticInterfaceMethod<
+      /*desc=*/        [{Returns the name of the FPExceptionBehaviorAttr
+                        attribute for the operation}],
+      /*returnType=*/  "StringRef",
+      /*methodName=*/  "getFPExceptionBehaviorAttrName",
+      /*args=*/        (ins),
+      /*methodBody=*/  [{}],
+      /*defaultImpl=*/ [{
+        return "fpExceptionBehavior";
+      }]
+    >
+  ];
+}
+
+def RoundingModeOpInterface : OpInterface<"RoundingModeOpInterface"> {
+  let description = [{
+    An interface for operations receiving a rounding mode attribute
+    controlling FP rounding mode.
+  }];
+
+  let cppNamespace = "::mlir::LLVM";
+
+  let methods = [
+    InterfaceMethod<
+      /*desc=*/        "Returns a RoundingMode attribute for the operation",
+      /*returnType=*/  "RoundingModeAttr",
+      /*methodName=*/  "getRoundingModeAttr",
+      /*args=*/        (ins),
+      /*methodBody=*/  [{}],
+      /*defaultImpl=*/ [{
+        auto op = cast<ConcreteOp>(this->getOperation());
+        return op.getRoundingmodeAttr();
+      }]
+    >,
+    StaticInterfaceMethod<
+      /*desc=*/        [{Returns the name of the RoundingModeAttr attribute
+                         for the operation}],
+      /*returnType=*/  "StringRef",
+      /*methodName=*/  "getRoundingModeAttrName",
+      /*args=*/        (ins),
+      /*methodBody=*/  [{}],
+      /*defaultImpl=*/ [{
+        return "roundingmode";
+      }]
+    >,
+  ];
+}
 
 //===----------------------------------------------------------------------===//
 // LLVM dialect type interfaces.
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index b88f1186a44b..f4bac9376f2e 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -311,6 +311,91 @@ def LLVM_InvariantEndOp : LLVM_ZeroResultIntrOp<"invariant.end", [2],
       "qualified(type($ptr))";
 }
 
+// Constrained Floating-Point Intrinsics.
+
+class LLVM_ConstrainedIntr<string mnem, int numArgs,
+                           bit overloadedResult, list<int> overloadedOperands,
+                           bit hasRoundingMode>
+    : LLVM_OneResultIntrOp<"experimental.constrained." # mnem,
+                           /*overloadedResults=*/
+                           !cond(!gt(overloadedResult, 0) : [0],
+                                 true : []),
+                           overloadedOperands,
+                           /*traits=*/[Pure, DeclareOpInterfaceMethods<FPExceptionBehaviorOpInterface>]
+                           # !cond(
+                               !gt(hasRoundingMode, 0) : [DeclareOpInterfaceMethods<RoundingModeOpInterface>],
+                               true : []),
+                           /*requiresFastmath=*/0,
+                           /*immArgPositions=*/[],
+                           /*immArgAttrNames=*/[]> {
+  dag regularArgs = !dag(ins, !listsplat(LLVM_Type, numArgs), !foreach(i, !range(numArgs), "arg_" #i));
+  dag attrArgs = !con(!cond(!gt(hasRoundingMode, 0) : (ins ValidRoundingModeAttr:$roundingmode),
+                            true : (ins)),
+                      (ins FPExceptionBehaviorAttr:$fpExceptionBehavior));
+  let arguments = !con(regularArgs, attrArgs);
+  let llvmBuilder = [{
+    SmallVector<llvm::Value *> args =
+      moduleTranslation.lookupValues(opInst.getOperands());
+    SmallVector<llvm::Type *> overloadedTypes; }] #
+    !cond(!gt(overloadedResult, 0) : [{
+    // Take into account overloaded result type.
+    overloadedTypes.push_back($_resultType); }],
+    // No overloaded result type.
+          true : "") # [{
+    llvm::transform(ArrayRef<unsigned>}] # overloadedOperandsCpp # [{,
+                    std::back_inserter(overloadedTypes),
+                    [&args](unsigned index) { return args[index]->getType(); });
+    llvm::Module *module = builder.GetInsertBlock()->getModule();
+    llvm::Function *callee =
+      llvm::Intrinsic::getDeclaration(module,
+        llvm::Intrinsic::experimental_constrained_}] #
+    mnem # [{, overloadedTypes); }] #
+    !cond(!gt(hasRoundingMode, 0) : [{
+    // Get rounding mode using interface.
+    llvm::RoundingMode rounding =
+        moduleTranslation.translateRoundingMode($roundingmode); }],
+          true : [{
+    // No rounding mode.
+    std::optional<llvm::RoundingMode> rounding; }]) # [{
+    llvm::fp::ExceptionBehavior except =
+      moduleTranslation.translateFPExceptionBehavior($fpExceptionBehavior);
+    $res = builder.CreateConstrainedFPCall(callee, args, "", rounding, except);
+  }];
+  let mlirBuilder = [{
+    SmallVector<Value> mlirOperands;
+    SmallVector<NamedAttribute> mlirAttrs;
+    if (failed(moduleImport.convertIntrinsicArguments(
+        llvmOperands.take_front( }] # numArgs # [{),
+        {}, {}, mlirOperands, mlirAttrs))) {
+      return failure();
+    }
+
+    FPExceptionBehaviorAttr fpExceptionBehaviorAttr =
+        $_fpExceptionBehavior_attr($fpExceptionBehavior);
+    mlirAttrs.push_back(
+        $_builder.getNamedAttr(
+            $_qualCppClassName::getFPExceptionBehaviorAttrName(),
+            fpExceptionBehaviorAttr)); }] #
+    !cond(!gt(hasRoundingMode, 0) : [{
+    RoundingModeAttr roundingModeAttr = $_roundingMode_attr($roundingmode);
+    mlirAttrs.push_back(
+        $_builder.getNamedAttr($_qualCppClassName::getRoundingModeAttrName(),
+                               roundingModeAttr));
+    }], true : "") # [{
+    $res = $_builder.create<$_qualCppClassName>($_location,
+      $_resultType, mlirOperands, mlirAttrs);
+  }];
+}
+
+def LLVM_ConstrainedFPTruncIntr
+    : LLVM_ConstrainedIntr<"fptrunc", /*numArgs=*/1,
+        /*overloadedResult=*/1, /*overloadedOperands=*/[0],
+        /*hasRoundingMode=*/1> {
+  let assemblyFormat = [{
+    $arg_0 $roundingmode $fpExceptionBehavior attr-dict `:` type($arg_0) `to` type(results)
+  }];
+}
+
 // Intrinsics with multiple returns.
 
 class LLVM_ArithWithOverflowOp<string mnem>
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
index b6aa73dad229..7b9a9cf017c5 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
@@ -170,6 +170,10 @@ class LLVM_OpBase<Dialect dialect, string mnemonic, list<Trait> traits = []> :
   //   - $_float_attr - substituted by a call to a float attribute matcher;
   //   - $_var_attr - substituted by a call to a variable attribute matcher;
   //   - $_label_attr - substituted by a call to a label attribute matcher;
+  //   - $_roundingMode_attr - substituted by a call to a rounding mode
+  //     attribute matcher;
+  //   - $_fpExceptionBehavior_attr - substituted by a call to a FP exception
+  //     behavior attribute matcher;
   //   - $_resultType - substituted with the MLIR result type;
   //   - $_location - substituted with the MLIR location;
   //   - $_builder - substituted with the MLIR builder;
diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h
index b49d2f539453..b551eb937cfe 100644
--- a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h
+++ b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h
@@ -152,6 +152,14 @@ public:
   /// Converts `value` to a label attribute. Asserts if the matching fails.
   DILabelAttr matchLabelAttr(llvm::Value *value);
 
+  /// Converts `value` to a FP exception behavior attribute. Asserts if the
+  /// matching fails.
+  FPExceptionBehaviorAttr matchFPExceptionBehaviorAttr(llvm::Value *value);
+
+  /// Converts `value` to a rounding mode attribute. Asserts if the matching
+  /// fails.
+  RoundingModeAttr matchRoundingModeAttr(llvm::Value *value);
+
   /// Converts `value` to an array of alias scopes or returns failure if the
   /// conversion fails.
   FailureOr<SmallVector<AliasScopeAttr>>
diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
index fb4392eb223c..310a43e0de96 100644
--- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
+++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
@@ -201,6 +201,13 @@ public:
   /// Translates the given LLVM debug info metadata.
   llvm::Metadata *translateDebugInfo(LLVM::DINodeAttr attr);
 
+  /// Translates the given LLVM rounding mode metadata.
+  llvm::RoundingMode translateRoundingMode(LLVM::RoundingMode rounding);
+
+  /// Translates the given LLVM FP exception behavior metadata.
+  llvm::fp::ExceptionBehavior
+  translateFPExceptionBehavior(LLVM::FPExceptionBehavior exceptionBehavior);
+
   /// Translates the contents of the given block to LLVM IR using this
   /// translator. The LLVM IR basic block corresponding to the given block is
   /// expected to exist in the mapping of this translator. Uses `builder` to
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index d63ea12ecd49..6e70d52fa760 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -1290,6 +1290,27 @@ DILabelAttr ModuleImport::matchLabelAttr(llvm::Value *value) {
   return debugImporter->translate(node);
 }
 
+FPExceptionBehaviorAttr
+ModuleImport::matchFPExceptionBehaviorAttr(llvm::Value *value) {
+  auto *metadata = cast<llvm::MetadataAsValue>(value);
+  auto *mdstr = cast<llvm::MDString>(metadata->getMetadata());
+  std::optional<llvm::fp::ExceptionBehavior> optLLVM =
+      llvm::convertStrToExceptionBehavior(mdstr->getString());
+  assert(optLLVM && "Expecting FP exception behavior");
+  return builder.getAttr<FPExceptionBehaviorAttr>(
+      convertFPExceptionBehaviorFromLLVM(*optLLVM));
+}
+
+RoundingModeAttr ModuleImport::matchRoundingModeAttr(llvm::Value *value) {
+  auto *metadata = cast<llvm::MetadataAsValue>(value);
+  auto *mdstr = cast<llvm::MDString>(metadata->getMetadata());
+  std::optional<llvm::RoundingMode> optLLVM =
+      llvm::convertStrToRoundingMode(mdstr->getString());
+  assert(optLLVM && "Expecting rounding mode");
+  return builder.getAttr<RoundingModeAttr>(
+      convertRoundingModeFromLLVM(*optLLVM));
+}
+
 FailureOr<SmallVector<AliasScopeAttr>>
 ModuleImport::matchAliasScopeAttrs(llvm::Value *value) {
   auto *nodeAsVal = cast<llvm::MetadataAsValue>(value);
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index f90495d407fd..669b95a9c6a5 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -1721,6 +1721,16 @@ llvm::Metadata *ModuleTranslation::translateDebugInfo(LLVM::DINodeAttr attr) {
   return debugTranslation->translate(attr);
 }
 
+llvm::RoundingMode
+ModuleTranslation::translateRoundingMode(LLVM::RoundingMode rounding) {
+  return convertRoundingModeToLLVM(rounding);
+}
+
+llvm::fp::ExceptionBehavior ModuleTranslation::translateFPExceptionBehavior(
+    LLVM::FPExceptionBehavior exceptionBehavior) {
+  return convertFPExceptionBehaviorToLLVM(exceptionBehavior);
+}
+
 llvm::NamedMDNode *
 ModuleTranslation::getOrInsertNamedModuleMetadata(StringRef name) {
   return llvmModule->getOrInsertNamedMetadata(name);
diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
index b157cf001418..31acf2b95e46 100644
--- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir
+++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
@@ -647,3 +647,18 @@ llvm.func @experimental_noalias_scope_decl() {
   llvm.intr.experimental.noalias.scope.decl #alias_scope
   llvm.return
 }
+
+// CHECK-LABEL: @experimental_constrained_fptrunc
+llvm.func @experimental_constrained_fptrunc(%in: f64) {
+  // CHECK: llvm.intr.experimental.constrained.fptrunc %{{.*}} towardzero ignore : f64 to f32
+  %0 = llvm.intr.experimental.constrained.fptrunc %in towardzero ignore : f64 to f32
+  // CHECK: llvm.intr.experimental.constrained.fptrunc %{{.*}} tonearest maytrap : f64 to f32
+  %1 = llvm.intr.experimental.constrained.fptrunc %in tonearest maytrap : f64 to f32
+  // CHECK: llvm.intr.experimental.constrained.fptrunc %{{.*}} upward strict : f64 to f32
+  %2 = llvm.intr.experimental.constrained.fptrunc %in upward strict : f64 to f32
+  // CHECK: llvm.intr.experimental.constrained.fptrunc %{{.*}} downward ignore : f64 to f32
+  %3 = llvm.intr.experimental.constrained.fptrunc %in downward ignore : f64 to f32
+  // CHECK: llvm.intr.experimental.constrained.fptrunc %{{.*}} tonearestaway ignore : f64 to f32
+  %4 = llvm.intr.experimental.constrained.fptrunc %in tonearestaway ignore : f64 to f32
+  llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index 1ec9005458c5..85561839f31a 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -894,6 +894,23 @@ define float @ssa_copy(float %0) {
   ret float %2
 }
 
+; CHECK-LABEL: experimental_constrained_fptrunc
+define void @experimental_constrained_fptrunc(double %s, <4 x double> %v) {
+  ; CHECK: llvm.intr.experimental.constrained.fptrunc %{{.*}} towardzero ignore : f64 to f32
+  %1 = call float @llvm.experimental.constrained.fptrunc.f32.f64(double %s, metadata !"round.towardzero", metadata !"fpexcept.ignore")
+  ; CHECK: llvm.intr.experimental.constrained.fptrunc %{{.*}} tonearest maytrap : f64 to f32
+  %2 = call float @llvm.experimental.constrained.fptrunc.f32.f64(double %s, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  ; CHECK: llvm.intr.experimental.constrained.fptrunc %{{.*}} upward strict : f64 to f32
+  %3 = call float @llvm.experimental.constrained.fptrunc.f32.f64(double %s, metadata !"round.upward", metadata !"fpexcept.strict")
+  ; CHECK: llvm.intr.experimental.constrained.fptrunc %{{.*}} downward ignore : f64 to f32
+  %4 = call float @llvm.experimental.constrained.fptrunc.f32.f64(double %s, metadata !"round.downward", metadata !"fpexcept.ignore")
+  ; CHECK: llvm.intr.experimental.constrained.fptrunc %{{.*}} tonearestaway ignore : f64 to f32
+  %5 = call float @llvm.experimental.constrained.fptrunc.f32.f64(double %s, metadata !"round.tonearestaway", metadata !"fpexcept.ignore")
+  ; CHECK: llvm.intr.experimental.constrained.fptrunc %{{.*}} tonearestaway ignore : vector<4xf64> to vector<4xf16>
+  %6 = call <4 x half> @llvm.experimental.constrained.fptrunc.v4f16.v4f64(<4 x double> %v, metadata !"round.tonearestaway", metadata !"fpexcept.ignore")
+  ret void
+}
+
 declare float @llvm.fmuladd.f32(float, float, float)
 declare <8 x float> @llvm.fmuladd.v8f32(<8 x float>, <8 x float>, <8 x float>)
 declare float @llvm.fma.f32(float, float, float)
@@ -1120,3 +1137,5 @@ declare void @llvm.assume(i1)
 declare float @llvm.ssa.copy.f32(float returned)
 declare <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float>, <4 x float>, i64)
 declare <4 x float> @llvm.vector.extract.v4f32.nxv4f32(<vscale x 4 x float>, i64)
+declare <4 x half> @llvm.experimental.constrained.fptrunc.v4f16.v4f64(<4 x double>, metadata, metadata)
+declare float @llvm.experimental.constrained.fptrunc.f32.f64(double, metadata, metadata)
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index fc2e0fd201a7..0013522582a7 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -964,6 +964,35 @@ llvm.func @ssa_copy(%arg: f32) -> f32 {
   llvm.return %0 : f32
 }
 
+// CHECK-LABEL: @experimental_constrained_fptrunc
+llvm.func @experimental_constrained_fptrunc(%s: f64, %v: vector<4xf32>) {
+  // CHECK: call float @llvm.experimental.constrained.fptrunc.f32.f64(
+  // CHECK: metadata !"round.towardzero"
+  // CHECK: metadata !"fpexcept.ignore"
+  %0 = llvm.intr.experimental.constrained.fptrunc %s towardzero ignore : f64 to f32
+  // CHECK: call float @llvm.experimental.constrained.fptrunc.f32.f64(
+  // CHECK: metadata !"round.tonearest"
+  // CHECK: metadata !"fpexcept.maytrap"
+  %1 = llvm.intr.experimental.constrained.fptrunc %s tonearest maytrap : f64 to f32
+  // CHECK: call float @llvm.experimental.constrained.fptrunc.f32.f64(
+  // CHECK: metadata !"round.upward"
+  // CHECK: metadata !"fpexcept.strict"
+  %2 = llvm.intr.experimental.constrained.fptrunc %s upward strict : f64 to f32
+  // CHECK: call float @llvm.experimental.constrained.fptrunc.f32.f64(
+  // CHECK: metadata !"round.downward"
+  // CHECK: metadata !"fpexcept.ignore"
+  %3 = llvm.intr.experimental.constrained.fptrunc %s downward ignore : f64 to f32
+  // CHECK: call float @llvm.experimental.constrained.fptrunc.f32.f64(
+  // CHECK: metadata !"round.tonearestaway"
+  // CHECK: metadata !"fpexcept.ignore"
+  %4 = llvm.intr.experimental.constrained.fptrunc %s tonearestaway ignore : f64 to f32
+  // CHECK: call <4 x half> @llvm.experimental.constrained.fptrunc.v4f16.v4f32(
+  // CHECK: metadata !"round.upward"
+  // CHECK: metadata !"fpexcept.strict"
+  %5 = llvm.intr.experimental.constrained.fptrunc %v upward strict : vector<4xf32> to vector<4xf16>
+  llvm.return
+}
+
 // Check that intrinsics are declared with appropriate types.
 // CHECK-DAG: declare float @llvm.fma.f32(float, float, float)
 // CHECK-DAG: declare <8 x float> @llvm.fma.v8f32(<8 x float>, <8 x float>, <8 x float>) #0
@@ -1126,3 +1155,5 @@ llvm.func @ssa_copy(%arg: f32) -> f32 {
 // CHECK-DAG: declare ptr addrspace(1) @llvm.stacksave.p1()
 // CHECK-DAG: declare void @llvm.stackrestore.p0(ptr)
 // CHECK-DAG: declare void @llvm.stackrestore.p1(ptr addrspace(1))
+// CHECK-DAG: declare float @llvm.experimental.constrained.fptrunc.f32.f64(double, metadata, metadata)
+// CHECK-DAG: declare <4 x half> @llvm.experimental.constrained.fptrunc.v4f16.v4f32(<4 x float>, metadata, metadata)
diff --git a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp
index 23bc9b00dc90..2c7acec3b1b8 100644
--- a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp
+++ b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp
@@ -272,6 +272,10 @@ static LogicalResult emitOneMLIRBuilder(const Record &record, raw_ostream &os,
       bs << "moduleImport.matchLocalVariableAttr";
     } else if (name == "_label_attr") {
       bs << "moduleImport.matchLabelAttr";
+    } else if (name == "_fpExceptionBehavior_attr") {
+      bs << "moduleImport.matchFPExceptionBehaviorAttr";
+    } else if (name == "_roundingMode_attr") {
+      bs << "moduleImport.matchRoundingModeAttr";
     } else if (name == "_resultType") {
       bs << "moduleImport.convertType(inst->getType())";
     } else if (name == "_location") {
-- 
GitLab


From c7198e0af34fc94722876d61af6e65758c316af5 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 26 Mar 2024 10:03:42 +0000
Subject: [PATCH 256/404] [DAG] Fold insert_subvector(N0, extract_subvector(N0,
 N2), N2) --> N0 (#86487)

Handle the case where we've ended up inserting back into the source vector we extracted the subvector from.
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |   7 +
 .../RISCV/rvv/vector-interleave-store.ll      |  30 +-
 .../vector-interleaved-store-i8-stride-6.ll   | 736 +++++++++---------
 3 files changed, 380 insertions(+), 393 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 05b4ce3aaa2c..6f1b1080cc88 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -26137,6 +26137,13 @@ SDValue DAGCombiner::visitINSERT_SUBVECTOR(SDNode *N) {
     }
   }
 
+  // Handle case where we've ended up inserting back into the source vector
+  // we extracted the subvector from.
+  // insert_subvector(N0, extract_subvector(N0, N2), N2) --> N0
+  if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR && N1.getOperand(0) == N0 &&
+      N1.getOperand(1) == N2)
+    return N0;
+
   // Simplify scalar inserts into an undef vector:
   // insert_subvector undef, (splat X), N2 -> splat X
   if (N0.isUndef() && N1.getOpcode() == ISD::SPLAT_VECTOR)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll
index 4aae8b8bd1dc..9a5e86d61c26 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll
@@ -101,40 +101,36 @@ define void @vector_interleave_store_nxv16i64_nxv8i64(<vscale x 8 x i64> %a, <vs
 ; CHECK-NEXT:    slli a1, a1, 4
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 3
-; CHECK-NEXT:    add a1, sp, a1
-; CHECK-NEXT:    addi a1, a1, 16
+; CHECK-NEXT:    addi a1, sp, 16
 ; CHECK-NEXT:    vs8r.v v8, (a1) # Unknown-size Folded Spill
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 1
 ; CHECK-NEXT:    vsetvli a3, zero, e16, m2, ta, mu
 ; CHECK-NEXT:    vid.v v24
 ; CHECK-NEXT:    vand.vi v26, v24, 1
-; CHECK-NEXT:    vmsne.vi v0, v26, 0
-; CHECK-NEXT:    vsrl.vi v6, v24, 1
-; CHECK-NEXT:    csrr a3, vlenb
-; CHECK-NEXT:    slli a3, a3, 3
-; CHECK-NEXT:    add a3, sp, a3
-; CHECK-NEXT:    addi a3, a3, 16
-; CHECK-NEXT:    vl8r.v v8, (a3) # Unknown-size Folded Reload
-; CHECK-NEXT:    vadd.vx v6, v6, a2, v0.t
+; CHECK-NEXT:    vmsne.vi v28, v26, 0
+; CHECK-NEXT:    vsrl.vi v24, v24, 1
+; CHECK-NEXT:    vmv1r.v v0, v28
+; CHECK-NEXT:    vadd.vx v24, v24, a2, v0.t
 ; CHECK-NEXT:    vmv4r.v v12, v16
 ; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v24, v8, v6
-; CHECK-NEXT:    addi a2, sp, 16
-; CHECK-NEXT:    vs8r.v v24, (a2) # Unknown-size Folded Spill
+; CHECK-NEXT:    vrgatherei16.vv v0, v8, v24
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    slli a2, a2, 3
 ; CHECK-NEXT:    add a2, sp, a2
 ; CHECK-NEXT:    addi a2, a2, 16
+; CHECK-NEXT:    vs8r.v v0, (a2) # Unknown-size Folded Spill
+; CHECK-NEXT:    addi a2, sp, 16
 ; CHECK-NEXT:    vl8r.v v8, (a2) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vmv4r.v v16, v12
-; CHECK-NEXT:    vrgatherei16.vv v8, v16, v6
+; CHECK-NEXT:    vrgatherei16.vv v8, v16, v24
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    add a1, a0, a1
 ; CHECK-NEXT:    vs8r.v v8, (a1)
-; CHECK-NEXT:    addi a1, sp, 16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 3
+; CHECK-NEXT:    add a1, sp, a1
+; CHECK-NEXT:    addi a1, a1, 16
 ; CHECK-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
 ; CHECK-NEXT:    vs8r.v v8, (a0)
 ; CHECK-NEXT:    csrr a0, vlenb
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll
index 88144e7880e3..de34e48c01d7 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll
@@ -3004,428 +3004,412 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-LABEL: store_i8_stride6_vf32:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm3
-; AVX512BW-NEXT:    vmovdqa (%rsi), %ymm5
-; AVX512BW-NEXT:    vmovdqa (%rdx), %ymm2
-; AVX512BW-NEXT:    vmovdqa (%rcx), %ymm4
-; AVX512BW-NEXT:    vmovdqa (%r8), %ymm0
-; AVX512BW-NEXT:    vmovdqa (%r9), %ymm1
-; AVX512BW-NEXT:    vmovdqa (%rsi), %xmm9
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} xmm6 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512BW-NEXT:    vpshufb %xmm6, %xmm9, %xmm7
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm10
-; AVX512BW-NEXT:    vpshufb %xmm6, %xmm10, %xmm6
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm6[8],xmm7[8],xmm6[9],xmm7[9],xmm6[10],xmm7[10],xmm6[11],xmm7[11],xmm6[12],xmm7[12],xmm6[13],xmm7[13],xmm6[14],xmm7[14],xmm6[15],xmm7[15]
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[0,0,0,1]
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
-; AVX512BW-NEXT:    vmovdqa (%rcx), %xmm11
-; AVX512BW-NEXT:    vpshufb %xmm6, %xmm11, %xmm8
-; AVX512BW-NEXT:    vmovdqa (%rdx), %xmm12
-; AVX512BW-NEXT:    vpshufb %xmm6, %xmm12, %xmm13
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3],xmm13[4],xmm8[4],xmm13[5],xmm8[5],xmm13[6],xmm8[6],xmm13[7],xmm8[7]
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,0,1]
-; AVX512BW-NEXT:    movw $18724, %cx # imm = 0x4924
+; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm9
+; AVX512BW-NEXT:    vmovdqa (%rsi), %ymm10
+; AVX512BW-NEXT:    vmovdqa (%rdx), %ymm11
+; AVX512BW-NEXT:    vmovdqa (%rcx), %ymm12
+; AVX512BW-NEXT:    vmovdqa (%r8), %ymm7
+; AVX512BW-NEXT:    vmovdqa (%r9), %ymm8
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm9[0],ymm10[0],ymm9[1],ymm10[1],ymm9[2],ymm10[2],ymm9[3],ymm10[3],ymm9[4],ymm10[4],ymm9[5],ymm10[5],ymm9[6],ymm10[6],ymm9[7],ymm10[7],ymm9[16],ymm10[16],ymm9[17],ymm10[17],ymm9[18],ymm10[18],ymm9[19],ymm10[19],ymm9[20],ymm10[20],ymm9[21],ymm10[21],ymm9[22],ymm10[22],ymm9[23],ymm10[23]
+; AVX512BW-NEXT:    vmovdqa (%rsi), %xmm1
+; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm2
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7,24,27,26,25,24,27,26,25,24,27,26,25,28,29,30,29]
+; AVX512BW-NEXT:    vpermw %zmm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vmovdqa (%rcx), %xmm3
+; AVX512BW-NEXT:    vmovdqa (%rdx), %xmm4
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm6 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7]
+; AVX512BW-NEXT:    vpermw %ymm5, %ymm6, %ymm5
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm11[0],ymm12[0],ymm11[1],ymm12[1],ymm11[2],ymm12[2],ymm11[3],ymm12[3],ymm11[4],ymm12[4],ymm11[5],ymm12[5],ymm11[6],ymm12[6],ymm11[7],ymm12[7],ymm11[16],ymm12[16],ymm11[17],ymm12[17],ymm11[18],ymm12[18],ymm11[19],ymm12[19],ymm11[20],ymm12[20],ymm11[21],ymm12[21],ymm11[22],ymm12[22],ymm11[23],ymm12[23]
+; AVX512BW-NEXT:    vprold $16, %ymm6, %ymm6
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,2,2,3]
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm5
+; AVX512BW-NEXT:    movl $613566756, %ecx # imm = 0x24924924
 ; AVX512BW-NEXT:    kmovd %ecx, %k1
-; AVX512BW-NEXT:    vmovdqu16 %ymm7, %ymm8 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm7
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3],xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm13 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5]
-; AVX512BW-NEXT:    vpermw %ymm8, %ymm13, %ymm8
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3],xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]
-; AVX512BW-NEXT:    vprold $16, %xmm13, %xmm13
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,0,1]
-; AVX512BW-NEXT:    movw $9362, %cx # imm = 0x2492
+; AVX512BW-NEXT:    vmovdqu16 %zmm5, %zmm0 {%k1}
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm13 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[16],ymm8[16],ymm7[17],ymm8[17],ymm7[18],ymm8[18],ymm7[19],ymm8[19],ymm7[20],ymm8[20],ymm7[21],ymm8[21],ymm7[22],ymm8[22],ymm7[23],ymm8[23]
+; AVX512BW-NEXT:    vmovdqa (%r9), %xmm5
+; AVX512BW-NEXT:    vmovdqa (%r8), %xmm6
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm14 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15]
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm13, %zmm14, %zmm13
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,26,25,24,27,26,25,24,27,26,25,24,27,28,28,28,28]
+; AVX512BW-NEXT:    movl $1227133513, %ecx # imm = 0x49249249
 ; AVX512BW-NEXT:    kmovd %ecx, %k2
-; AVX512BW-NEXT:    vmovdqu16 %ymm13, %ymm8 {%k2}
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm8[0,1,2,3],zmm7[4,5,6,7]
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
-; AVX512BW-NEXT:    vmovdqa (%r9), %xmm13
-; AVX512BW-NEXT:    vpshufb %xmm8, %xmm13, %xmm14
-; AVX512BW-NEXT:    vmovdqa (%r8), %xmm15
-; AVX512BW-NEXT:    vpshufb %xmm8, %xmm15, %xmm16
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm16[0],xmm14[0],xmm16[1],xmm14[1],xmm16[2],xmm14[2],xmm16[3],xmm14[3],xmm16[4],xmm14[4],xmm16[5],xmm14[5],xmm16[6],xmm14[6],xmm16[7],xmm14[7]
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,0,0,1]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm16 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3],xmm15[4],xmm13[4],xmm15[5],xmm13[5],xmm15[6],xmm13[6],xmm15[7],xmm13[7]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm17 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4]
-; AVX512BW-NEXT:    vpermw %ymm16, %ymm17, %ymm16
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm14, %zmm16, %zmm14
-; AVX512BW-NEXT:    movl $613566756, %ecx # imm = 0x24924924
-; AVX512BW-NEXT:    kmovd %ecx, %k3
-; AVX512BW-NEXT:    vmovdqu16 %zmm14, %zmm7 {%k3}
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm14 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[4],ymm5[4],ymm3[5],ymm5[5],ymm3[6],ymm5[6],ymm3[7],ymm5[7],ymm3[16],ymm5[16],ymm3[17],ymm5[17],ymm3[18],ymm5[18],ymm3[19],ymm5[19],ymm3[20],ymm5[20],ymm3[21],ymm5[21],ymm3[22],ymm5[22],ymm3[23],ymm5[23]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm16 = [8,11,10,9,8,11,10,9,8,11,10,9,12,13,14,13]
-; AVX512BW-NEXT:    vpermw %ymm14, %ymm16, %ymm14
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm16 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[16],ymm4[16],ymm2[17],ymm4[17],ymm2[18],ymm4[18],ymm2[19],ymm4[19],ymm2[20],ymm4[20],ymm2[21],ymm4[21],ymm2[22],ymm4[22],ymm2[23],ymm4[23]
-; AVX512BW-NEXT:    vprold $16, %ymm16, %ymm16
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm16 = ymm16[2,2,2,3]
-; AVX512BW-NEXT:    vmovdqu16 %ymm16, %ymm14 {%k2}
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm14, %zmm0, %zmm14
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm11 = xmm12[8],xmm11[8],xmm12[9],xmm11[9],xmm12[10],xmm11[10],xmm12[11],xmm11[11],xmm12[12],xmm11[12],xmm12[13],xmm11[13],xmm12[14],xmm11[14],xmm12[15],xmm11[15]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm10 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7]
+; AVX512BW-NEXT:    vpermw %zmm13, %zmm14, %zmm0 {%k2}
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm13 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512BW-NEXT:    vpshufb %ymm13, %ymm10, %ymm14
+; AVX512BW-NEXT:    vpshufb %ymm13, %ymm9, %ymm13
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm13 = ymm13[0],ymm14[0],ymm13[1],ymm14[1],ymm13[2],ymm14[2],ymm13[3],ymm14[3],ymm13[4],ymm14[4],ymm13[5],ymm14[5],ymm13[6],ymm14[6],ymm13[7],ymm14[7],ymm13[16],ymm14[16],ymm13[17],ymm14[17],ymm13[18],ymm14[18],ymm13[19],ymm14[19],ymm13[20],ymm14[20],ymm13[21],ymm14[21],ymm13[22],ymm14[22],ymm13[23],ymm14[23]
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[2,2,2,3]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm14 = ymm11[8],ymm12[8],ymm11[9],ymm12[9],ymm11[10],ymm12[10],ymm11[11],ymm12[11],ymm11[12],ymm12[12],ymm11[13],ymm12[13],ymm11[14],ymm12[14],ymm11[15],ymm12[15],ymm11[24],ymm12[24],ymm11[25],ymm12[25],ymm11[26],ymm12[26],ymm11[27],ymm12[27],ymm11[28],ymm12[28],ymm11[29],ymm12[29],ymm11[30],ymm12[30],ymm11[31],ymm12[31]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm15 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
+; AVX512BW-NEXT:    vpermw %ymm14, %ymm15, %ymm14
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm14, %zmm13, %zmm13
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm14 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
+; AVX512BW-NEXT:    vpshufb %ymm14, %ymm12, %ymm12
+; AVX512BW-NEXT:    vpshufb %ymm14, %ymm11, %ymm11
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm11 = ymm11[0],ymm12[0],ymm11[1],ymm12[1],ymm11[2],ymm12[2],ymm11[3],ymm12[3],ymm11[4],ymm12[4],ymm11[5],ymm12[5],ymm11[6],ymm12[6],ymm11[7],ymm12[7],ymm11[16],ymm12[16],ymm11[17],ymm12[17],ymm11[18],ymm12[18],ymm11[19],ymm12[19],ymm11[20],ymm12[20],ymm11[21],ymm12[21],ymm11[22],ymm12[22],ymm11[23],ymm12[23]
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,2,2,3]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm9[8],ymm10[8],ymm9[9],ymm10[9],ymm9[10],ymm10[10],ymm9[11],ymm10[11],ymm9[12],ymm10[12],ymm9[13],ymm10[13],ymm9[14],ymm10[14],ymm9[15],ymm10[15],ymm9[24],ymm10[24],ymm9[25],ymm10[25],ymm9[26],ymm10[26],ymm9[27],ymm10[27],ymm9[28],ymm10[28],ymm9[29],ymm10[29],ymm9[30],ymm10[30],ymm9[31],ymm10[31]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm10 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
 ; AVX512BW-NEXT:    vpermw %ymm9, %ymm10, %ymm9
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm10 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7]
-; AVX512BW-NEXT:    vpermw %ymm11, %ymm10, %ymm9 {%k1}
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm9 = zmm9[0,1,2,3],zmm14[4,5,6,7]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm10 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm11 = xmm15[8],xmm13[8],xmm15[9],xmm13[9],xmm15[10],xmm13[10],xmm15[11],xmm13[11],xmm15[12],xmm13[12],xmm15[13],xmm13[13],xmm15[14],xmm13[14],xmm15[15],xmm13[15]
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm10, %zmm11, %zmm10
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,26,25,24,27,26,25,24,27,26,25,24,27,28,28,28,28]
-; AVX512BW-NEXT:    movl $1227133513, %ecx # imm = 0x49249249
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm9, %zmm11, %zmm9
+; AVX512BW-NEXT:    movl $1227114788, %ecx # imm = 0x49244924
 ; AVX512BW-NEXT:    kmovd %ecx, %k2
-; AVX512BW-NEXT:    vpermw %zmm10, %zmm11, %zmm9 {%k2}
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm2[8],ymm4[8],ymm2[9],ymm4[9],ymm2[10],ymm4[10],ymm2[11],ymm4[11],ymm2[12],ymm4[12],ymm2[13],ymm4[13],ymm2[14],ymm4[14],ymm2[15],ymm4[15],ymm2[24],ymm4[24],ymm2[25],ymm4[25],ymm2[26],ymm4[26],ymm2[27],ymm4[27],ymm2[28],ymm4[28],ymm2[29],ymm4[29],ymm2[30],ymm4[30],ymm2[31],ymm4[31]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm11 = ymm3[8],ymm5[8],ymm3[9],ymm5[9],ymm3[10],ymm5[10],ymm3[11],ymm5[11],ymm3[12],ymm5[12],ymm3[13],ymm5[13],ymm3[14],ymm5[14],ymm3[15],ymm5[15],ymm3[24],ymm5[24],ymm3[25],ymm5[25],ymm3[26],ymm5[26],ymm3[27],ymm5[27],ymm3[28],ymm5[28],ymm3[29],ymm5[29],ymm3[30],ymm5[30],ymm3[31],ymm5[31]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm12 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
-; AVX512BW-NEXT:    vpermw %ymm11, %ymm12, %ymm11
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm12 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
-; AVX512BW-NEXT:    vpermw %ymm10, %ymm12, %ymm11 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm10
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm11 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512BW-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
-; AVX512BW-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[4],ymm5[4],ymm3[5],ymm5[5],ymm3[6],ymm5[6],ymm3[7],ymm5[7],ymm3[16],ymm5[16],ymm3[17],ymm5[17],ymm3[18],ymm5[18],ymm3[19],ymm5[19],ymm3[20],ymm5[20],ymm3[21],ymm5[21],ymm3[22],ymm5[22],ymm3[23],ymm5[23]
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
-; AVX512BW-NEXT:    vpshufb %ymm6, %ymm4, %ymm4
-; AVX512BW-NEXT:    vpshufb %ymm6, %ymm2, %ymm2
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[16],ymm4[16],ymm2[17],ymm4[17],ymm2[18],ymm4[18],ymm2[19],ymm4[19],ymm2[20],ymm4[20],ymm2[21],ymm4[21],ymm2[22],ymm4[22],ymm2[23],ymm4[23]
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]
-; AVX512BW-NEXT:    vmovdqu16 %ymm3, %ymm2 {%k1}
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm10[4,5,6,7]
-; AVX512BW-NEXT:    vpshufb %ymm8, %ymm1, %ymm3
-; AVX512BW-NEXT:    vpshufb %ymm8, %ymm0, %ymm4
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[16],ymm3[16],ymm4[17],ymm3[17],ymm4[18],ymm3[18],ymm4[19],ymm3[19],ymm4[20],ymm3[20],ymm4[21],ymm3[21],ymm4[22],ymm3[22],ymm4[23],ymm3[23]
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
-; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [10,13,12,11,10,13,12,11,10,13,12,11,14,13,14,15]
-; AVX512BW-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vmovdqu16 %zmm13, %zmm9 {%k2}
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm10 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
+; AVX512BW-NEXT:    vpshufb %ymm10, %ymm8, %ymm11
+; AVX512BW-NEXT:    vpshufb %ymm10, %ymm7, %ymm12
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} ymm11 = ymm12[0],ymm11[0],ymm12[1],ymm11[1],ymm12[2],ymm11[2],ymm12[3],ymm11[3],ymm12[4],ymm11[4],ymm12[5],ymm11[5],ymm12[6],ymm11[6],ymm12[7],ymm11[7],ymm12[16],ymm11[16],ymm12[17],ymm11[17],ymm12[18],ymm11[18],ymm12[19],ymm11[19],ymm12[20],ymm11[20],ymm12[21],ymm11[21],ymm12[22],ymm11[22],ymm12[23],ymm11[23]
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,2,2,3]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15],ymm7[24],ymm8[24],ymm7[25],ymm8[25],ymm7[26],ymm8[26],ymm7[27],ymm8[27],ymm7[28],ymm8[28],ymm7[29],ymm8[29],ymm7[30],ymm8[30],ymm7[31],ymm8[31]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm8 = [10,13,12,11,10,13,12,11,10,13,12,11,14,13,14,15]
+; AVX512BW-NEXT:    vpermw %ymm7, %ymm8, %ymm7
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm7, %zmm11, %zmm7
 ; AVX512BW-NEXT:    movl $-1840700270, %ecx # imm = 0x92492492
-; AVX512BW-NEXT:    kmovd %ecx, %k1
-; AVX512BW-NEXT:    vmovdqu16 %zmm0, %zmm2 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, 128(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, 64(%rax)
+; AVX512BW-NEXT:    kmovd %ecx, %k2
+; AVX512BW-NEXT:    vmovdqu16 %zmm7, %zmm9 {%k2}
+; AVX512BW-NEXT:    vpshufb %xmm14, %xmm3, %xmm7
+; AVX512BW-NEXT:    vpshufb %xmm14, %xmm4, %xmm8
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3],xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,0,0,1]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5]
+; AVX512BW-NEXT:    vpermw %ymm8, %ymm11, %ymm8
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm7, %zmm8, %zmm7
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} xmm8 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512BW-NEXT:    vpshufb %xmm8, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpshufb %xmm8, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; AVX512BW-NEXT:    vprold $16, %xmm2, %xmm2
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512BW-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[0,0,0,1,4,4,4,5]
+; AVX512BW-NEXT:    movl $1227105426, %ecx # imm = 0x49242492
+; AVX512BW-NEXT:    kmovd %ecx, %k2
+; AVX512BW-NEXT:    vmovdqu16 %zmm1, %zmm7 {%k2}
+; AVX512BW-NEXT:    vpshufb %xmm10, %xmm5, %xmm1
+; AVX512BW-NEXT:    vpshufb %xmm10, %xmm6, %xmm2
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,0,1]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm3 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4]
+; AVX512BW-NEXT:    vpermw %ymm2, %ymm3, %ymm2
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqu16 %zmm1, %zmm7 {%k1}
 ; AVX512BW-NEXT:    vmovdqa64 %zmm7, (%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, 128(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, 64(%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512BW-FCP-LABEL: store_i8_stride6_vf32:
 ; AVX512BW-FCP:       # %bb.0:
 ; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %ymm3
-; AVX512BW-FCP-NEXT:    vmovdqa (%rsi), %ymm5
-; AVX512BW-FCP-NEXT:    vmovdqa (%rdx), %ymm2
-; AVX512BW-FCP-NEXT:    vmovdqa (%rcx), %ymm4
+; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %ymm2
+; AVX512BW-FCP-NEXT:    vmovdqa (%rsi), %ymm3
+; AVX512BW-FCP-NEXT:    vmovdqa (%rdx), %ymm5
+; AVX512BW-FCP-NEXT:    vmovdqa (%rcx), %ymm6
 ; AVX512BW-FCP-NEXT:    vmovdqa (%r8), %ymm0
 ; AVX512BW-FCP-NEXT:    vmovdqa (%r9), %ymm1
-; AVX512BW-FCP-NEXT:    vmovdqa (%rcx), %xmm9
-; AVX512BW-FCP-NEXT:    vmovdqa (%rdx), %xmm10
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3],xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]
-; AVX512BW-FCP-NEXT:    vmovdqa (%rsi), %xmm11
+; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm7
+; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm4
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm7[0],ymm4[1],ymm7[1],ymm4[2],ymm7[2],ymm4[3],ymm7[3],ymm4[4],ymm7[4],ymm4[5],ymm7[5],ymm4[6],ymm7[6],ymm4[7],ymm7[7],ymm4[16],ymm7[16],ymm4[17],ymm7[17],ymm4[18],ymm7[18],ymm4[19],ymm7[19],ymm4[20],ymm7[20],ymm4[21],ymm7[21],ymm4[22],ymm7[22],ymm4[23],ymm7[23]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm5[8],ymm6[8],ymm5[9],ymm6[9],ymm5[10],ymm6[10],ymm5[11],ymm6[11],ymm5[12],ymm6[12],ymm5[13],ymm6[13],ymm5[14],ymm6[14],ymm5[15],ymm6[15],ymm5[24],ymm6[24],ymm5[25],ymm6[25],ymm5[26],ymm6[26],ymm5[27],ymm6[27],ymm5[28],ymm6[28],ymm5[29],ymm6[29],ymm5[30],ymm6[30],ymm5[31],ymm6[31]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm8 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
+; AVX512BW-FCP-NEXT:    vpermw %ymm7, %ymm8, %ymm7
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm4, %zmm7
+; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm8, %ymm6, %ymm4
+; AVX512BW-FCP-NEXT:    vpshufb %ymm8, %ymm5, %ymm9
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm9[0],ymm4[0],ymm9[1],ymm4[1],ymm9[2],ymm4[2],ymm9[3],ymm4[3],ymm9[4],ymm4[4],ymm9[5],ymm4[5],ymm9[6],ymm4[6],ymm9[7],ymm4[7],ymm9[16],ymm4[16],ymm9[17],ymm4[17],ymm9[18],ymm4[18],ymm9[19],ymm4[19],ymm9[20],ymm4[20],ymm9[21],ymm4[21],ymm9[22],ymm4[22],ymm9[23],ymm4[23]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm10 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
+; AVX512BW-FCP-NEXT:    vpermw %ymm9, %ymm10, %ymm9
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm4, %zmm4
+; AVX512BW-FCP-NEXT:    movl $1227114788, %r10d # imm = 0x49244924
+; AVX512BW-FCP-NEXT:    kmovd %r10d, %k1
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm7, %zmm4 {%k1}
+; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm7, %ymm1, %ymm9
+; AVX512BW-FCP-NEXT:    vpshufb %ymm7, %ymm0, %ymm10
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm9 = ymm10[0],ymm9[0],ymm10[1],ymm9[1],ymm10[2],ymm9[2],ymm10[3],ymm9[3],ymm10[4],ymm9[4],ymm10[5],ymm9[5],ymm10[6],ymm9[6],ymm10[7],ymm9[7],ymm10[16],ymm9[16],ymm10[17],ymm9[17],ymm10[18],ymm9[18],ymm10[19],ymm9[19],ymm10[20],ymm9[20],ymm10[21],ymm9[21],ymm10[22],ymm9[22],ymm10[23],ymm9[23]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [10,13,12,11,10,13,12,11,10,13,12,11,14,13,14,15]
+; AVX512BW-FCP-NEXT:    vpermw %ymm10, %ymm11, %ymm10
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
+; AVX512BW-FCP-NEXT:    movl $-1840700270, %r10d # imm = 0x92492492
+; AVX512BW-FCP-NEXT:    kmovd %r10d, %k1
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm9, %zmm4 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa (%rsi), %xmm9
+; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm10 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm10, %xmm9, %xmm11
 ; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %xmm12
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3],xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm8 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5]
-; AVX512BW-FCP-NEXT:    vpermw %ymm7, %ymm8, %ymm8
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm7 = [1,0,3,2,1,0,3,2,1,0,3,2,5,4,7,6]
-; AVX512BW-FCP-NEXT:    movw $9362, %cx # imm = 0x2492
-; AVX512BW-FCP-NEXT:    kmovd %ecx, %k2
-; AVX512BW-FCP-NEXT:    vpermw %ymm6, %ymm7, %ymm8 {%k2}
-; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm6 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm6, %xmm11, %xmm7
-; AVX512BW-FCP-NEXT:    vpshufb %xmm6, %xmm12, %xmm6
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm6[8],xmm7[8],xmm6[9],xmm7[9],xmm6[10],xmm7[10],xmm6[11],xmm7[11],xmm6[12],xmm7[12],xmm6[13],xmm7[13],xmm6[14],xmm7[14],xmm6[15],xmm7[15]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,0,1]
-; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm7, %xmm9, %xmm13
-; AVX512BW-FCP-NEXT:    vpshufb %xmm7, %xmm10, %xmm14
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3],xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,0,1]
-; AVX512BW-FCP-NEXT:    movw $18724, %cx # imm = 0x4924
+; AVX512BW-FCP-NEXT:    vpshufb %xmm10, %xmm12, %xmm10
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm10[8],xmm11[8],xmm10[9],xmm11[9],xmm10[10],xmm11[10],xmm10[11],xmm11[11],xmm10[12],xmm11[12],xmm10[13],xmm11[13],xmm10[14],xmm11[14],xmm10[15],xmm11[15]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,0,1]
+; AVX512BW-FCP-NEXT:    vmovdqa (%rcx), %xmm11
+; AVX512BW-FCP-NEXT:    vmovdqa (%rdx), %xmm13
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm13[0],xmm11[0],xmm13[1],xmm11[1],xmm13[2],xmm11[2],xmm13[3],xmm11[3],xmm13[4],xmm11[4],xmm13[5],xmm11[5],xmm13[6],xmm11[6],xmm13[7],xmm11[7]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm15 = [1,0,3,2,1,0,3,2,1,0,3,2,5,4,7,6]
+; AVX512BW-FCP-NEXT:    vpermw %ymm14, %ymm15, %ymm14
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm14, %zmm10
+; AVX512BW-FCP-NEXT:    vpshufb %xmm8, %xmm11, %xmm14
+; AVX512BW-FCP-NEXT:    vpshufb %xmm8, %xmm13, %xmm8
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm8[0],xmm14[0],xmm8[1],xmm14[1],xmm8[2],xmm14[2],xmm8[3],xmm14[3],xmm8[4],xmm14[4],xmm8[5],xmm14[5],xmm8[6],xmm14[6],xmm8[7],xmm14[7]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,0,1]
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm12[0],xmm9[0],xmm12[1],xmm9[1],xmm12[2],xmm9[2],xmm12[3],xmm9[3],xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm15 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5]
+; AVX512BW-FCP-NEXT:    vpermw %ymm14, %ymm15, %ymm14
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm14, %zmm8
+; AVX512BW-FCP-NEXT:    movl $1227105426, %ecx # imm = 0x49242492
 ; AVX512BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm6, %ymm13 {%k1}
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm0, %zmm6
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm8[0,1,2,3],zmm6[4,5,6,7]
-; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
-; AVX512BW-FCP-NEXT:    vmovdqa (%r9), %xmm13
-; AVX512BW-FCP-NEXT:    vpshufb %xmm8, %xmm13, %xmm14
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm10, %zmm8 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa (%r9), %xmm10
+; AVX512BW-FCP-NEXT:    vpshufb %xmm7, %xmm10, %xmm14
 ; AVX512BW-FCP-NEXT:    vmovdqa (%r8), %xmm15
-; AVX512BW-FCP-NEXT:    vpshufb %xmm8, %xmm15, %xmm16
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm16[0],xmm14[0],xmm16[1],xmm14[1],xmm16[2],xmm14[2],xmm16[3],xmm14[3],xmm16[4],xmm14[4],xmm16[5],xmm14[5],xmm16[6],xmm14[6],xmm16[7],xmm14[7]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,0,0,1]
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm16 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3],xmm15[4],xmm13[4],xmm15[5],xmm13[5],xmm15[6],xmm13[6],xmm15[7],xmm13[7]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm17 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4]
-; AVX512BW-FCP-NEXT:    vpermw %ymm16, %ymm17, %ymm16
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm16, %zmm14
+; AVX512BW-FCP-NEXT:    vpshufb %xmm7, %xmm15, %xmm7
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm7[0],xmm14[0],xmm7[1],xmm14[1],xmm7[2],xmm14[2],xmm7[3],xmm14[3],xmm7[4],xmm14[4],xmm7[5],xmm14[5],xmm7[6],xmm14[6],xmm7[7],xmm14[7]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,0,0,1]
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm15[0],xmm10[0],xmm15[1],xmm10[1],xmm15[2],xmm10[2],xmm15[3],xmm10[3],xmm15[4],xmm10[4],xmm15[5],xmm10[5],xmm15[6],xmm10[6],xmm15[7],xmm10[7]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm16 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4]
+; AVX512BW-FCP-NEXT:    vpermw %ymm14, %ymm16, %ymm14
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm14, %zmm7
 ; AVX512BW-FCP-NEXT:    movl $613566756, %ecx # imm = 0x24924924
-; AVX512BW-FCP-NEXT:    kmovd %ecx, %k3
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm14, %zmm6 {%k3}
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm14 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[16],ymm4[16],ymm2[17],ymm4[17],ymm2[18],ymm4[18],ymm2[19],ymm4[19],ymm2[20],ymm4[20],ymm2[21],ymm4[21],ymm2[22],ymm4[22],ymm2[23],ymm4[23]
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm16 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[4],ymm5[4],ymm3[5],ymm5[5],ymm3[6],ymm5[6],ymm3[7],ymm5[7],ymm3[16],ymm5[16],ymm3[17],ymm5[17],ymm3[18],ymm5[18],ymm3[19],ymm5[19],ymm3[20],ymm5[20],ymm3[21],ymm5[21],ymm3[22],ymm5[22],ymm3[23],ymm5[23]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm17 = [8,11,10,9,8,11,10,9,8,11,10,9,12,13,14,13]
-; AVX512BW-FCP-NEXT:    vpermw %ymm16, %ymm17, %ymm16
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm17 = [9,8,11,10,9,8,11,10,9,8,11,10,13,12,15,14]
-; AVX512BW-FCP-NEXT:    vpermw %ymm14, %ymm17, %ymm16 {%k2}
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm16, %zmm0, %zmm14
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm12[8],xmm11[8],xmm12[9],xmm11[9],xmm12[10],xmm11[10],xmm12[11],xmm11[11],xmm12[12],xmm11[12],xmm12[13],xmm11[13],xmm12[14],xmm11[14],xmm12[15],xmm11[15]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7]
-; AVX512BW-FCP-NEXT:    vpermw %ymm10, %ymm11, %ymm10
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7]
-; AVX512BW-FCP-NEXT:    vpermw %ymm9, %ymm11, %ymm10 {%k1}
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm9 = zmm10[0,1,2,3],zmm14[4,5,6,7]
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm10 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm11 = xmm15[8],xmm13[8],xmm15[9],xmm13[9],xmm15[10],xmm13[10],xmm15[11],xmm13[11],xmm15[12],xmm13[12],xmm15[13],xmm13[13],xmm15[14],xmm13[14],xmm15[15],xmm13[15]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm11, %zmm10
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,26,25,24,27,26,25,24,27,26,25,24,27,28,28,28,28]
+; AVX512BW-FCP-NEXT:    kmovd %ecx, %k1
+; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm7, %zmm8 {%k1}
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[4],ymm6[4],ymm5[5],ymm6[5],ymm5[6],ymm6[6],ymm5[7],ymm6[7],ymm5[16],ymm6[16],ymm5[17],ymm6[17],ymm5[18],ymm6[18],ymm5[19],ymm6[19],ymm5[20],ymm6[20],ymm5[21],ymm6[21],ymm5[22],ymm6[22],ymm5[23],ymm6[23]
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm13[8],xmm11[8],xmm13[9],xmm11[9],xmm13[10],xmm11[10],xmm13[11],xmm11[11],xmm13[12],xmm11[12],xmm13[13],xmm11[13],xmm13[14],xmm11[14],xmm13[15],xmm11[15]
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm6, %zmm5
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[16],ymm3[16],ymm2[17],ymm3[17],ymm2[18],ymm3[18],ymm2[19],ymm3[19],ymm2[20],ymm3[20],ymm2[21],ymm3[21],ymm2[22],ymm3[22],ymm2[23],ymm3[23]
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm12[8],xmm9[8],xmm12[9],xmm9[9],xmm12[10],xmm9[10],xmm12[11],xmm9[11],xmm12[12],xmm9[12],xmm12[13],xmm9[13],xmm12[14],xmm9[14],xmm12[15],xmm9[15]
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7,24,27,26,25,24,27,26,25,24,27,26,25,28,29,30,29]
+; AVX512BW-FCP-NEXT:    vpermw %zmm2, %zmm3, %zmm2
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7,25,24,27,26,25,24,27,26,25,24,27,26,29,28,31,30]
+; AVX512BW-FCP-NEXT:    vpermw %zmm5, %zmm3, %zmm2 {%k1}
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm15[8],xmm10[8],xmm15[9],xmm10[9],xmm15[10],xmm10[10],xmm15[11],xmm10[11],xmm15[12],xmm10[12],xmm15[13],xmm10[13],xmm15[14],xmm10[14],xmm15[15],xmm10[15]
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,26,25,24,27,26,25,24,27,26,25,24,27,28,28,28,28]
 ; AVX512BW-FCP-NEXT:    movl $1227133513, %ecx # imm = 0x49249249
-; AVX512BW-FCP-NEXT:    kmovd %ecx, %k2
-; AVX512BW-FCP-NEXT:    vpermw %zmm10, %zmm11, %zmm9 {%k2}
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm2[8],ymm4[8],ymm2[9],ymm4[9],ymm2[10],ymm4[10],ymm2[11],ymm4[11],ymm2[12],ymm4[12],ymm2[13],ymm4[13],ymm2[14],ymm4[14],ymm2[15],ymm4[15],ymm2[24],ymm4[24],ymm2[25],ymm4[25],ymm2[26],ymm4[26],ymm2[27],ymm4[27],ymm2[28],ymm4[28],ymm2[29],ymm4[29],ymm2[30],ymm4[30],ymm2[31],ymm4[31]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm11 = ymm3[8],ymm5[8],ymm3[9],ymm5[9],ymm3[10],ymm5[10],ymm3[11],ymm5[11],ymm3[12],ymm5[12],ymm3[13],ymm5[13],ymm3[14],ymm5[14],ymm3[15],ymm5[15],ymm3[24],ymm5[24],ymm3[25],ymm5[25],ymm3[26],ymm5[26],ymm3[27],ymm5[27],ymm3[28],ymm5[28],ymm3[29],ymm5[29],ymm3[30],ymm5[30],ymm3[31],ymm5[31]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm12 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
-; AVX512BW-FCP-NEXT:    vpermw %ymm11, %ymm12, %ymm11
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm12 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
-; AVX512BW-FCP-NEXT:    vpermw %ymm10, %ymm12, %ymm11 {%k1}
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm10
-; AVX512BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm11 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
-; AVX512BW-FCP-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[4],ymm5[4],ymm3[5],ymm5[5],ymm3[6],ymm5[6],ymm3[7],ymm5[7],ymm3[16],ymm5[16],ymm3[17],ymm5[17],ymm3[18],ymm5[18],ymm3[19],ymm5[19],ymm3[20],ymm5[20],ymm3[21],ymm5[21],ymm3[22],ymm5[22],ymm3[23],ymm5[23]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm7, %ymm4, %ymm4
-; AVX512BW-FCP-NEXT:    vpshufb %ymm7, %ymm2, %ymm2
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[16],ymm4[16],ymm2[17],ymm4[17],ymm2[18],ymm4[18],ymm2[19],ymm4[19],ymm2[20],ymm4[20],ymm2[21],ymm4[21],ymm2[22],ymm4[22],ymm2[23],ymm4[23]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]
-; AVX512BW-FCP-NEXT:    vmovdqu16 %ymm3, %ymm2 {%k1}
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm10[4,5,6,7]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm8, %ymm1, %ymm3
-; AVX512BW-FCP-NEXT:    vpshufb %ymm8, %ymm0, %ymm4
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[16],ymm3[16],ymm4[17],ymm3[17],ymm4[18],ymm3[18],ymm4[19],ymm3[19],ymm4[20],ymm3[20],ymm4[21],ymm3[21],ymm4[22],ymm3[22],ymm4[23],ymm3[23]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [10,13,12,11,10,13,12,11,10,13,12,11,14,13,14,15]
-; AVX512BW-FCP-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm3, %zmm0
-; AVX512BW-FCP-NEXT:    movl $-1840700270, %ecx # imm = 0x92492492
 ; AVX512BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu16 %zmm0, %zmm2 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm2, 128(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm9, 64(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm6, (%rax)
+; AVX512BW-FCP-NEXT:    vpermw %zmm0, %zmm1, %zmm2 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm2, 64(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm8, (%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm4, 128(%rax)
 ; AVX512BW-FCP-NEXT:    vzeroupper
 ; AVX512BW-FCP-NEXT:    retq
 ;
 ; AVX512DQ-BW-LABEL: store_i8_stride6_vf32:
 ; AVX512DQ-BW:       # %bb.0:
 ; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-NEXT:    vmovdqa (%rdi), %ymm3
-; AVX512DQ-BW-NEXT:    vmovdqa (%rsi), %ymm5
-; AVX512DQ-BW-NEXT:    vmovdqa (%rdx), %ymm2
-; AVX512DQ-BW-NEXT:    vmovdqa (%rcx), %ymm4
-; AVX512DQ-BW-NEXT:    vmovdqa (%r8), %ymm0
-; AVX512DQ-BW-NEXT:    vmovdqa (%r9), %ymm1
-; AVX512DQ-BW-NEXT:    vmovdqa (%rsi), %xmm9
-; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} xmm6 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm6, %xmm9, %xmm7
-; AVX512DQ-BW-NEXT:    vmovdqa (%rdi), %xmm10
-; AVX512DQ-BW-NEXT:    vpshufb %xmm6, %xmm10, %xmm6
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm6[8],xmm7[8],xmm6[9],xmm7[9],xmm6[10],xmm7[10],xmm6[11],xmm7[11],xmm6[12],xmm7[12],xmm6[13],xmm7[13],xmm6[14],xmm7[14],xmm6[15],xmm7[15]
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[0,0,0,1]
-; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
-; AVX512DQ-BW-NEXT:    vmovdqa (%rcx), %xmm11
-; AVX512DQ-BW-NEXT:    vpshufb %xmm6, %xmm11, %xmm8
-; AVX512DQ-BW-NEXT:    vmovdqa (%rdx), %xmm12
-; AVX512DQ-BW-NEXT:    vpshufb %xmm6, %xmm12, %xmm13
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3],xmm13[4],xmm8[4],xmm13[5],xmm8[5],xmm13[6],xmm8[6],xmm13[7],xmm8[7]
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,0,1]
-; AVX512DQ-BW-NEXT:    movw $18724, %cx # imm = 0x4924
+; AVX512DQ-BW-NEXT:    vmovdqa (%rdi), %ymm9
+; AVX512DQ-BW-NEXT:    vmovdqa (%rsi), %ymm10
+; AVX512DQ-BW-NEXT:    vmovdqa (%rdx), %ymm11
+; AVX512DQ-BW-NEXT:    vmovdqa (%rcx), %ymm12
+; AVX512DQ-BW-NEXT:    vmovdqa (%r8), %ymm7
+; AVX512DQ-BW-NEXT:    vmovdqa (%r9), %ymm8
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm9[0],ymm10[0],ymm9[1],ymm10[1],ymm9[2],ymm10[2],ymm9[3],ymm10[3],ymm9[4],ymm10[4],ymm9[5],ymm10[5],ymm9[6],ymm10[6],ymm9[7],ymm10[7],ymm9[16],ymm10[16],ymm9[17],ymm10[17],ymm9[18],ymm10[18],ymm9[19],ymm10[19],ymm9[20],ymm10[20],ymm9[21],ymm10[21],ymm9[22],ymm10[22],ymm9[23],ymm10[23]
+; AVX512DQ-BW-NEXT:    vmovdqa (%rsi), %xmm1
+; AVX512DQ-BW-NEXT:    vmovdqa (%rdi), %xmm2
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm3, %zmm0
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7,24,27,26,25,24,27,26,25,24,27,26,25,28,29,30,29]
+; AVX512DQ-BW-NEXT:    vpermw %zmm0, %zmm3, %zmm0
+; AVX512DQ-BW-NEXT:    vmovdqa (%rcx), %xmm3
+; AVX512DQ-BW-NEXT:    vmovdqa (%rdx), %xmm4
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm6 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7]
+; AVX512DQ-BW-NEXT:    vpermw %ymm5, %ymm6, %ymm5
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm6 = ymm11[0],ymm12[0],ymm11[1],ymm12[1],ymm11[2],ymm12[2],ymm11[3],ymm12[3],ymm11[4],ymm12[4],ymm11[5],ymm12[5],ymm11[6],ymm12[6],ymm11[7],ymm12[7],ymm11[16],ymm12[16],ymm11[17],ymm12[17],ymm11[18],ymm12[18],ymm11[19],ymm12[19],ymm11[20],ymm12[20],ymm11[21],ymm12[21],ymm11[22],ymm12[22],ymm11[23],ymm12[23]
+; AVX512DQ-BW-NEXT:    vprold $16, %ymm6, %ymm6
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,2,2,3]
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm5
+; AVX512DQ-BW-NEXT:    movl $613566756, %ecx # imm = 0x24924924
 ; AVX512DQ-BW-NEXT:    kmovd %ecx, %k1
-; AVX512DQ-BW-NEXT:    vmovdqu16 %ymm7, %ymm8 {%k1}
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm7
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3],xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm13 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5]
-; AVX512DQ-BW-NEXT:    vpermw %ymm8, %ymm13, %ymm8
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3],xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]
-; AVX512DQ-BW-NEXT:    vprold $16, %xmm13, %xmm13
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,0,1]
-; AVX512DQ-BW-NEXT:    movw $9362, %cx # imm = 0x2492
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm5, %zmm0 {%k1}
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm13 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[16],ymm8[16],ymm7[17],ymm8[17],ymm7[18],ymm8[18],ymm7[19],ymm8[19],ymm7[20],ymm8[20],ymm7[21],ymm8[21],ymm7[22],ymm8[22],ymm7[23],ymm8[23]
+; AVX512DQ-BW-NEXT:    vmovdqa (%r9), %xmm5
+; AVX512DQ-BW-NEXT:    vmovdqa (%r8), %xmm6
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm14 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15]
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm13, %zmm14, %zmm13
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,26,25,24,27,26,25,24,27,26,25,24,27,28,28,28,28]
+; AVX512DQ-BW-NEXT:    movl $1227133513, %ecx # imm = 0x49249249
 ; AVX512DQ-BW-NEXT:    kmovd %ecx, %k2
-; AVX512DQ-BW-NEXT:    vmovdqu16 %ymm13, %ymm8 {%k2}
-; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm8[0,1,2,3],zmm7[4,5,6,7]
-; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
-; AVX512DQ-BW-NEXT:    vmovdqa (%r9), %xmm13
-; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm13, %xmm14
-; AVX512DQ-BW-NEXT:    vmovdqa (%r8), %xmm15
-; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm15, %xmm16
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm16[0],xmm14[0],xmm16[1],xmm14[1],xmm16[2],xmm14[2],xmm16[3],xmm14[3],xmm16[4],xmm14[4],xmm16[5],xmm14[5],xmm16[6],xmm14[6],xmm16[7],xmm14[7]
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,0,0,1]
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm16 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3],xmm15[4],xmm13[4],xmm15[5],xmm13[5],xmm15[6],xmm13[6],xmm15[7],xmm13[7]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm17 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4]
-; AVX512DQ-BW-NEXT:    vpermw %ymm16, %ymm17, %ymm16
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm14, %zmm16, %zmm14
-; AVX512DQ-BW-NEXT:    movl $613566756, %ecx # imm = 0x24924924
-; AVX512DQ-BW-NEXT:    kmovd %ecx, %k3
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm14, %zmm7 {%k3}
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm14 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[4],ymm5[4],ymm3[5],ymm5[5],ymm3[6],ymm5[6],ymm3[7],ymm5[7],ymm3[16],ymm5[16],ymm3[17],ymm5[17],ymm3[18],ymm5[18],ymm3[19],ymm5[19],ymm3[20],ymm5[20],ymm3[21],ymm5[21],ymm3[22],ymm5[22],ymm3[23],ymm5[23]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm16 = [8,11,10,9,8,11,10,9,8,11,10,9,12,13,14,13]
-; AVX512DQ-BW-NEXT:    vpermw %ymm14, %ymm16, %ymm14
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm16 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[16],ymm4[16],ymm2[17],ymm4[17],ymm2[18],ymm4[18],ymm2[19],ymm4[19],ymm2[20],ymm4[20],ymm2[21],ymm4[21],ymm2[22],ymm4[22],ymm2[23],ymm4[23]
-; AVX512DQ-BW-NEXT:    vprold $16, %ymm16, %ymm16
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm16 = ymm16[2,2,2,3]
-; AVX512DQ-BW-NEXT:    vmovdqu16 %ymm16, %ymm14 {%k2}
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm14, %zmm0, %zmm14
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm11 = xmm12[8],xmm11[8],xmm12[9],xmm11[9],xmm12[10],xmm11[10],xmm12[11],xmm11[11],xmm12[12],xmm11[12],xmm12[13],xmm11[13],xmm12[14],xmm11[14],xmm12[15],xmm11[15]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm10 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7]
+; AVX512DQ-BW-NEXT:    vpermw %zmm13, %zmm14, %zmm0 {%k2}
+; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} ymm13 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm13, %ymm10, %ymm14
+; AVX512DQ-BW-NEXT:    vpshufb %ymm13, %ymm9, %ymm13
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm13 = ymm13[0],ymm14[0],ymm13[1],ymm14[1],ymm13[2],ymm14[2],ymm13[3],ymm14[3],ymm13[4],ymm14[4],ymm13[5],ymm14[5],ymm13[6],ymm14[6],ymm13[7],ymm14[7],ymm13[16],ymm14[16],ymm13[17],ymm14[17],ymm13[18],ymm14[18],ymm13[19],ymm14[19],ymm13[20],ymm14[20],ymm13[21],ymm14[21],ymm13[22],ymm14[22],ymm13[23],ymm14[23]
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[2,2,2,3]
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm14 = ymm11[8],ymm12[8],ymm11[9],ymm12[9],ymm11[10],ymm12[10],ymm11[11],ymm12[11],ymm11[12],ymm12[12],ymm11[13],ymm12[13],ymm11[14],ymm12[14],ymm11[15],ymm12[15],ymm11[24],ymm12[24],ymm11[25],ymm12[25],ymm11[26],ymm12[26],ymm11[27],ymm12[27],ymm11[28],ymm12[28],ymm11[29],ymm12[29],ymm11[30],ymm12[30],ymm11[31],ymm12[31]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm15 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
+; AVX512DQ-BW-NEXT:    vpermw %ymm14, %ymm15, %ymm14
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm14, %zmm13, %zmm13
+; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} ymm14 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm14, %ymm12, %ymm12
+; AVX512DQ-BW-NEXT:    vpshufb %ymm14, %ymm11, %ymm11
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm11 = ymm11[0],ymm12[0],ymm11[1],ymm12[1],ymm11[2],ymm12[2],ymm11[3],ymm12[3],ymm11[4],ymm12[4],ymm11[5],ymm12[5],ymm11[6],ymm12[6],ymm11[7],ymm12[7],ymm11[16],ymm12[16],ymm11[17],ymm12[17],ymm11[18],ymm12[18],ymm11[19],ymm12[19],ymm11[20],ymm12[20],ymm11[21],ymm12[21],ymm11[22],ymm12[22],ymm11[23],ymm12[23]
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,2,2,3]
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm9[8],ymm10[8],ymm9[9],ymm10[9],ymm9[10],ymm10[10],ymm9[11],ymm10[11],ymm9[12],ymm10[12],ymm9[13],ymm10[13],ymm9[14],ymm10[14],ymm9[15],ymm10[15],ymm9[24],ymm10[24],ymm9[25],ymm10[25],ymm9[26],ymm10[26],ymm9[27],ymm10[27],ymm9[28],ymm10[28],ymm9[29],ymm10[29],ymm9[30],ymm10[30],ymm9[31],ymm10[31]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm10 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
 ; AVX512DQ-BW-NEXT:    vpermw %ymm9, %ymm10, %ymm9
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm10 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7]
-; AVX512DQ-BW-NEXT:    vpermw %ymm11, %ymm10, %ymm9 {%k1}
-; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm9 = zmm9[0,1,2,3],zmm14[4,5,6,7]
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm10 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm11 = xmm15[8],xmm13[8],xmm15[9],xmm13[9],xmm15[10],xmm13[10],xmm15[11],xmm13[11],xmm15[12],xmm13[12],xmm15[13],xmm13[13],xmm15[14],xmm13[14],xmm15[15],xmm13[15]
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm10, %zmm11, %zmm10
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,26,25,24,27,26,25,24,27,26,25,24,27,28,28,28,28]
-; AVX512DQ-BW-NEXT:    movl $1227133513, %ecx # imm = 0x49249249
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm9, %zmm11, %zmm9
+; AVX512DQ-BW-NEXT:    movl $1227114788, %ecx # imm = 0x49244924
 ; AVX512DQ-BW-NEXT:    kmovd %ecx, %k2
-; AVX512DQ-BW-NEXT:    vpermw %zmm10, %zmm11, %zmm9 {%k2}
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm2[8],ymm4[8],ymm2[9],ymm4[9],ymm2[10],ymm4[10],ymm2[11],ymm4[11],ymm2[12],ymm4[12],ymm2[13],ymm4[13],ymm2[14],ymm4[14],ymm2[15],ymm4[15],ymm2[24],ymm4[24],ymm2[25],ymm4[25],ymm2[26],ymm4[26],ymm2[27],ymm4[27],ymm2[28],ymm4[28],ymm2[29],ymm4[29],ymm2[30],ymm4[30],ymm2[31],ymm4[31]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm11 = ymm3[8],ymm5[8],ymm3[9],ymm5[9],ymm3[10],ymm5[10],ymm3[11],ymm5[11],ymm3[12],ymm5[12],ymm3[13],ymm5[13],ymm3[14],ymm5[14],ymm3[15],ymm5[15],ymm3[24],ymm5[24],ymm3[25],ymm5[25],ymm3[26],ymm5[26],ymm3[27],ymm5[27],ymm3[28],ymm5[28],ymm3[29],ymm5[29],ymm3[30],ymm5[30],ymm3[31],ymm5[31]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm12 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
-; AVX512DQ-BW-NEXT:    vpermw %ymm11, %ymm12, %ymm11
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm12 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
-; AVX512DQ-BW-NEXT:    vpermw %ymm10, %ymm12, %ymm11 {%k1}
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm10
-; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} ymm11 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
-; AVX512DQ-BW-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[4],ymm5[4],ymm3[5],ymm5[5],ymm3[6],ymm5[6],ymm3[7],ymm5[7],ymm3[16],ymm5[16],ymm3[17],ymm5[17],ymm3[18],ymm5[18],ymm3[19],ymm5[19],ymm3[20],ymm5[20],ymm3[21],ymm5[21],ymm3[22],ymm5[22],ymm3[23],ymm5[23]
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm6, %ymm4, %ymm4
-; AVX512DQ-BW-NEXT:    vpshufb %ymm6, %ymm2, %ymm2
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[16],ymm4[16],ymm2[17],ymm4[17],ymm2[18],ymm4[18],ymm2[19],ymm4[19],ymm2[20],ymm4[20],ymm2[21],ymm4[21],ymm2[22],ymm4[22],ymm2[23],ymm4[23]
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]
-; AVX512DQ-BW-NEXT:    vmovdqu16 %ymm3, %ymm2 {%k1}
-; AVX512DQ-BW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm10[4,5,6,7]
-; AVX512DQ-BW-NEXT:    vpshufb %ymm8, %ymm1, %ymm3
-; AVX512DQ-BW-NEXT:    vpshufb %ymm8, %ymm0, %ymm4
-; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[16],ymm3[16],ymm4[17],ymm3[17],ymm4[18],ymm3[18],ymm4[19],ymm3[19],ymm4[20],ymm3[20],ymm4[21],ymm3[21],ymm4[22],ymm3[22],ymm4[23],ymm3[23]
-; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
-; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
-; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [10,13,12,11,10,13,12,11,10,13,12,11,14,13,14,15]
-; AVX512DQ-BW-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm3, %zmm0
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm13, %zmm9 {%k2}
+; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} ymm10 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
+; AVX512DQ-BW-NEXT:    vpshufb %ymm10, %ymm8, %ymm11
+; AVX512DQ-BW-NEXT:    vpshufb %ymm10, %ymm7, %ymm12
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} ymm11 = ymm12[0],ymm11[0],ymm12[1],ymm11[1],ymm12[2],ymm11[2],ymm12[3],ymm11[3],ymm12[4],ymm11[4],ymm12[5],ymm11[5],ymm12[6],ymm11[6],ymm12[7],ymm11[7],ymm12[16],ymm11[16],ymm12[17],ymm11[17],ymm12[18],ymm11[18],ymm12[19],ymm11[19],ymm12[20],ymm11[20],ymm12[21],ymm11[21],ymm12[22],ymm11[22],ymm12[23],ymm11[23]
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,2,2,3]
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15],ymm7[24],ymm8[24],ymm7[25],ymm8[25],ymm7[26],ymm8[26],ymm7[27],ymm8[27],ymm7[28],ymm8[28],ymm7[29],ymm8[29],ymm7[30],ymm8[30],ymm7[31],ymm8[31]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm8 = [10,13,12,11,10,13,12,11,10,13,12,11,14,13,14,15]
+; AVX512DQ-BW-NEXT:    vpermw %ymm7, %ymm8, %ymm7
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm7, %zmm11, %zmm7
 ; AVX512DQ-BW-NEXT:    movl $-1840700270, %ecx # imm = 0x92492492
-; AVX512DQ-BW-NEXT:    kmovd %ecx, %k1
-; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm0, %zmm2 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm2, 128(%rax)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm9, 64(%rax)
+; AVX512DQ-BW-NEXT:    kmovd %ecx, %k2
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm7, %zmm9 {%k2}
+; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm3, %xmm7
+; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm4, %xmm8
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3],xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,0,0,1]
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5]
+; AVX512DQ-BW-NEXT:    vpermw %ymm8, %ymm11, %ymm8
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm7, %zmm8, %zmm7
+; AVX512DQ-BW-NEXT:    vpbroadcastq {{.*#+}} xmm8 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm1, %xmm1
+; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm2, %xmm2
+; AVX512DQ-BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; AVX512DQ-BW-NEXT:    vprold $16, %xmm2, %xmm2
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[0,0,0,1,4,4,4,5]
+; AVX512DQ-BW-NEXT:    movl $1227105426, %ecx # imm = 0x49242492
+; AVX512DQ-BW-NEXT:    kmovd %ecx, %k2
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm1, %zmm7 {%k2}
+; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm5, %xmm1
+; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm6, %xmm2
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512DQ-BW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,0,1]
+; AVX512DQ-BW-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
+; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm3 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4]
+; AVX512DQ-BW-NEXT:    vpermw %ymm2, %ymm3, %ymm2
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512DQ-BW-NEXT:    vmovdqu16 %zmm1, %zmm7 {%k1}
 ; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm7, (%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm9, 128(%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, 64(%rax)
 ; AVX512DQ-BW-NEXT:    vzeroupper
 ; AVX512DQ-BW-NEXT:    retq
 ;
 ; AVX512DQ-BW-FCP-LABEL: store_i8_stride6_vf32:
 ; AVX512DQ-BW-FCP:       # %bb.0:
 ; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rsi), %ymm5
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdx), %ymm2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rcx), %ymm4
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %ymm2
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rsi), %ymm3
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdx), %ymm5
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rcx), %ymm6
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r8), %ymm0
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r9), %ymm1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rcx), %xmm9
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdx), %xmm10
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3],xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rsi), %xmm11
+; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm7
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm4
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm7[0],ymm4[1],ymm7[1],ymm4[2],ymm7[2],ymm4[3],ymm7[3],ymm4[4],ymm7[4],ymm4[5],ymm7[5],ymm4[6],ymm7[6],ymm4[7],ymm7[7],ymm4[16],ymm7[16],ymm4[17],ymm7[17],ymm4[18],ymm7[18],ymm4[19],ymm7[19],ymm4[20],ymm7[20],ymm4[21],ymm7[21],ymm4[22],ymm7[22],ymm4[23],ymm7[23]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm5[8],ymm6[8],ymm5[9],ymm6[9],ymm5[10],ymm6[10],ymm5[11],ymm6[11],ymm5[12],ymm6[12],ymm5[13],ymm6[13],ymm5[14],ymm6[14],ymm5[15],ymm6[15],ymm5[24],ymm6[24],ymm5[25],ymm6[25],ymm5[26],ymm6[26],ymm5[27],ymm6[27],ymm5[28],ymm6[28],ymm5[29],ymm6[29],ymm5[30],ymm6[30],ymm5[31],ymm6[31]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm8 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm7, %ymm8, %ymm7
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm4, %zmm7
+; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm8, %ymm6, %ymm4
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm8, %ymm5, %ymm9
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm9[0],ymm4[0],ymm9[1],ymm4[1],ymm9[2],ymm4[2],ymm9[3],ymm4[3],ymm9[4],ymm4[4],ymm9[5],ymm4[5],ymm9[6],ymm4[6],ymm9[7],ymm4[7],ymm9[16],ymm4[16],ymm9[17],ymm4[17],ymm9[18],ymm4[18],ymm9[19],ymm4[19],ymm9[20],ymm4[20],ymm9[21],ymm4[21],ymm9[22],ymm4[22],ymm9[23],ymm4[23]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm9 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm10 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm9, %ymm10, %ymm9
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm4, %zmm4
+; AVX512DQ-BW-FCP-NEXT:    movl $1227114788, %r10d # imm = 0x49244924
+; AVX512DQ-BW-FCP-NEXT:    kmovd %r10d, %k1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm7, %zmm4 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm7, %ymm1, %ymm9
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm7, %ymm0, %ymm10
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm9 = ymm10[0],ymm9[0],ymm10[1],ymm9[1],ymm10[2],ymm9[2],ymm10[3],ymm9[3],ymm10[4],ymm9[4],ymm10[5],ymm9[5],ymm10[6],ymm9[6],ymm10[7],ymm9[7],ymm10[16],ymm9[16],ymm10[17],ymm9[17],ymm10[18],ymm9[18],ymm10[19],ymm9[19],ymm10[20],ymm9[20],ymm10[21],ymm9[21],ymm10[22],ymm9[22],ymm10[23],ymm9[23]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [10,13,12,11,10,13,12,11,10,13,12,11,14,13,14,15]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm10, %ymm11, %ymm10
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm9, %zmm9
+; AVX512DQ-BW-FCP-NEXT:    movl $-1840700270, %r10d # imm = 0x92492492
+; AVX512DQ-BW-FCP-NEXT:    kmovd %r10d, %k1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm9, %zmm4 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rsi), %xmm9
+; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm10 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm10, %xmm9, %xmm11
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %xmm12
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3],xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm8 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm7, %ymm8, %ymm8
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm7 = [1,0,3,2,1,0,3,2,1,0,3,2,5,4,7,6]
-; AVX512DQ-BW-FCP-NEXT:    movw $9362, %cx # imm = 0x2492
-; AVX512DQ-BW-FCP-NEXT:    kmovd %ecx, %k2
-; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm6, %ymm7, %ymm8 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} xmm6 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm6, %xmm11, %xmm7
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm6, %xmm12, %xmm6
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm6[8],xmm7[8],xmm6[9],xmm7[9],xmm6[10],xmm7[10],xmm6[11],xmm7[11],xmm6[12],xmm7[12],xmm6[13],xmm7[13],xmm6[14],xmm7[14],xmm6[15],xmm7[15]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm7 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm7, %xmm9, %xmm13
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm7, %xmm10, %xmm14
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3],xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,0,1]
-; AVX512DQ-BW-FCP-NEXT:    movw $18724, %cx # imm = 0x4924
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm10, %xmm12, %xmm10
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm10[8],xmm11[8],xmm10[9],xmm11[9],xmm10[10],xmm11[10],xmm10[11],xmm11[11],xmm10[12],xmm11[12],xmm10[13],xmm11[13],xmm10[14],xmm11[14],xmm10[15],xmm11[15]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rcx), %xmm11
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdx), %xmm13
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm13[0],xmm11[0],xmm13[1],xmm11[1],xmm13[2],xmm11[2],xmm13[3],xmm11[3],xmm13[4],xmm11[4],xmm13[5],xmm11[5],xmm13[6],xmm11[6],xmm13[7],xmm11[7]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm15 = [1,0,3,2,1,0,3,2,1,0,3,2,5,4,7,6]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm14, %ymm15, %ymm14
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm14, %zmm10
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm8, %xmm11, %xmm14
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm8, %xmm13, %xmm8
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm8[0],xmm14[0],xmm8[1],xmm14[1],xmm8[2],xmm14[2],xmm8[3],xmm14[3],xmm8[4],xmm14[4],xmm8[5],xmm14[5],xmm8[6],xmm14[6],xmm8[7],xmm14[7]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm12[0],xmm9[0],xmm12[1],xmm9[1],xmm12[2],xmm9[2],xmm12[3],xmm9[3],xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm15 = [0,3,2,1,0,3,2,1,0,3,2,1,4,5,6,5]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm14, %ymm15, %ymm14
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm8, %zmm14, %zmm8
+; AVX512DQ-BW-FCP-NEXT:    movl $1227105426, %ecx # imm = 0x49242492
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm6, %ymm13 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm0, %zmm6
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm8[0,1,2,3],zmm6[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm8 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r9), %xmm13
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm8, %xmm13, %xmm14
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm10, %zmm8 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r9), %xmm10
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm7, %xmm10, %xmm14
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r8), %xmm15
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm8, %xmm15, %xmm16
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm16[0],xmm14[0],xmm16[1],xmm14[1],xmm16[2],xmm14[2],xmm16[3],xmm14[3],xmm16[4],xmm14[4],xmm16[5],xmm14[5],xmm16[6],xmm14[6],xmm16[7],xmm14[7]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,0,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm16 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3],xmm15[4],xmm13[4],xmm15[5],xmm13[5],xmm15[6],xmm13[6],xmm15[7],xmm13[7]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm17 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm16, %ymm17, %ymm16
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm14, %zmm16, %zmm14
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm7, %xmm15, %xmm7
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm7[0],xmm14[0],xmm7[1],xmm14[1],xmm7[2],xmm14[2],xmm7[3],xmm14[3],xmm7[4],xmm14[4],xmm7[5],xmm14[5],xmm7[6],xmm14[6],xmm7[7],xmm14[7]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,0,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm15[0],xmm10[0],xmm15[1],xmm10[1],xmm15[2],xmm10[2],xmm15[3],xmm10[3],xmm15[4],xmm10[4],xmm15[5],xmm10[5],xmm15[6],xmm10[6],xmm15[7],xmm10[7]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm16 = [2,1,0,3,2,1,0,3,2,1,0,3,4,4,4,4]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm14, %ymm16, %ymm14
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm14, %zmm7
 ; AVX512DQ-BW-FCP-NEXT:    movl $613566756, %ecx # imm = 0x24924924
-; AVX512DQ-BW-FCP-NEXT:    kmovd %ecx, %k3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm14, %zmm6 {%k3}
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm14 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[16],ymm4[16],ymm2[17],ymm4[17],ymm2[18],ymm4[18],ymm2[19],ymm4[19],ymm2[20],ymm4[20],ymm2[21],ymm4[21],ymm2[22],ymm4[22],ymm2[23],ymm4[23]
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm16 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[4],ymm5[4],ymm3[5],ymm5[5],ymm3[6],ymm5[6],ymm3[7],ymm5[7],ymm3[16],ymm5[16],ymm3[17],ymm5[17],ymm3[18],ymm5[18],ymm3[19],ymm5[19],ymm3[20],ymm5[20],ymm3[21],ymm5[21],ymm3[22],ymm5[22],ymm3[23],ymm5[23]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm17 = [8,11,10,9,8,11,10,9,8,11,10,9,12,13,14,13]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm16, %ymm17, %ymm16
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm17 = [9,8,11,10,9,8,11,10,9,8,11,10,13,12,15,14]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm14, %ymm17, %ymm16 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm16, %zmm0, %zmm14
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm12[8],xmm11[8],xmm12[9],xmm11[9],xmm12[10],xmm11[10],xmm12[11],xmm11[11],xmm12[12],xmm11[12],xmm12[13],xmm11[13],xmm12[14],xmm11[14],xmm12[15],xmm11[15]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm10, %ymm11, %ymm10
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm9, %ymm11, %ymm10 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm9 = zmm10[0,1,2,3],zmm14[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm10 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm11 = xmm15[8],xmm13[8],xmm15[9],xmm13[9],xmm15[10],xmm13[10],xmm15[11],xmm13[11],xmm15[12],xmm13[12],xmm15[13],xmm13[13],xmm15[14],xmm13[14],xmm15[15],xmm13[15]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm11, %zmm10
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,26,25,24,27,26,25,24,27,26,25,24,27,28,28,28,28]
+; AVX512DQ-BW-FCP-NEXT:    kmovd %ecx, %k1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm7, %zmm8 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm5 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[4],ymm6[4],ymm5[5],ymm6[5],ymm5[6],ymm6[6],ymm5[7],ymm6[7],ymm5[16],ymm6[16],ymm5[17],ymm6[17],ymm5[18],ymm6[18],ymm5[19],ymm6[19],ymm5[20],ymm6[20],ymm5[21],ymm6[21],ymm5[22],ymm6[22],ymm5[23],ymm6[23]
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm13[8],xmm11[8],xmm13[9],xmm11[9],xmm13[10],xmm11[10],xmm13[11],xmm11[11],xmm13[12],xmm11[12],xmm13[13],xmm11[13],xmm13[14],xmm11[14],xmm13[15],xmm11[15]
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm6, %zmm5
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[16],ymm3[16],ymm2[17],ymm3[17],ymm2[18],ymm3[18],ymm2[19],ymm3[19],ymm2[20],ymm3[20],ymm2[21],ymm3[21],ymm2[22],ymm3[22],ymm2[23],ymm3[23]
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm12[8],xmm9[8],xmm12[9],xmm9[9],xmm12[10],xmm9[10],xmm12[11],xmm9[11],xmm12[12],xmm9[12],xmm12[13],xmm9[13],xmm12[14],xmm9[14],xmm12[15],xmm9[15]
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [4,3,6,5,4,3,6,5,4,3,6,5,7,7,7,7,24,27,26,25,24,27,26,25,24,27,26,25,28,29,30,29]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm2, %zmm3, %zmm2
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [5,4,3,6,5,4,3,6,5,4,3,6,7,7,7,7,25,24,27,26,25,24,27,26,25,24,27,26,29,28,31,30]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm5, %zmm3, %zmm2 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm15[8],xmm10[8],xmm15[9],xmm10[9],xmm15[10],xmm10[10],xmm15[11],xmm10[11],xmm15[12],xmm10[12],xmm15[13],xmm10[13],xmm15[14],xmm10[14],xmm15[15],xmm10[15]
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [2,5,4,3,2,5,4,3,2,5,4,3,6,5,6,7,26,25,24,27,26,25,24,27,26,25,24,27,28,28,28,28]
 ; AVX512DQ-BW-FCP-NEXT:    movl $1227133513, %ecx # imm = 0x49249249
-; AVX512DQ-BW-FCP-NEXT:    kmovd %ecx, %k2
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm10, %zmm11, %zmm9 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm10 = ymm2[8],ymm4[8],ymm2[9],ymm4[9],ymm2[10],ymm4[10],ymm2[11],ymm4[11],ymm2[12],ymm4[12],ymm2[13],ymm4[13],ymm2[14],ymm4[14],ymm2[15],ymm4[15],ymm2[24],ymm4[24],ymm2[25],ymm4[25],ymm2[26],ymm4[26],ymm2[27],ymm4[27],ymm2[28],ymm4[28],ymm2[29],ymm4[29],ymm2[30],ymm4[30],ymm2[31],ymm4[31]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm11 = ymm3[8],ymm5[8],ymm3[9],ymm5[9],ymm3[10],ymm5[10],ymm3[11],ymm5[11],ymm3[12],ymm5[12],ymm3[13],ymm5[13],ymm3[14],ymm5[14],ymm3[15],ymm5[15],ymm3[24],ymm5[24],ymm3[25],ymm5[25],ymm3[26],ymm5[26],ymm3[27],ymm5[27],ymm3[28],ymm5[28],ymm3[29],ymm5[29],ymm3[30],ymm5[30],ymm3[31],ymm5[31]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm12 = [12,11,14,13,12,11,14,13,12,11,14,13,15,15,15,15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm11, %ymm12, %ymm11
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm12 = [13,12,11,14,13,12,11,14,13,12,11,14,15,15,15,15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm10, %ymm12, %ymm11 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm10
-; AVX512DQ-BW-FCP-NEXT:    vpbroadcastq {{.*#+}} ymm11 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[4],ymm5[4],ymm3[5],ymm5[5],ymm3[6],ymm5[6],ymm3[7],ymm5[7],ymm3[16],ymm5[16],ymm3[17],ymm5[17],ymm3[18],ymm5[18],ymm3[19],ymm5[19],ymm3[20],ymm5[20],ymm3[21],ymm5[21],ymm3[22],ymm5[22],ymm3[23],ymm5[23]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm7, %ymm4, %ymm4
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm7, %ymm2, %ymm2
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[16],ymm4[16],ymm2[17],ymm4[17],ymm2[18],ymm4[18],ymm2[19],ymm4[19],ymm2[20],ymm4[20],ymm2[21],ymm4[21],ymm2[22],ymm4[22],ymm2[23],ymm4[23]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %ymm3, %ymm2 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm10[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm8, %ymm1, %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm8, %ymm0, %ymm4
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[16],ymm3[16],ymm4[17],ymm3[17],ymm4[18],ymm3[18],ymm4[19],ymm3[19],ymm4[20],ymm3[20],ymm4[21],ymm3[21],ymm4[22],ymm3[22],ymm4[23],ymm3[23]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,2,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm1 = [10,13,12,11,10,13,12,11,10,13,12,11,14,13,14,15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm3, %zmm0
-; AVX512DQ-BW-FCP-NEXT:    movl $-1840700270, %ecx # imm = 0x92492492
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu16 %zmm0, %zmm2 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm2, 128(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm9, 64(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm6, (%rax)
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm0, %zmm1, %zmm2 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm2, 64(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm8, (%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm4, 128(%rax)
 ; AVX512DQ-BW-FCP-NEXT:    vzeroupper
 ; AVX512DQ-BW-FCP-NEXT:    retq
   %in.vec0 = load <32 x i8>, ptr %in.vecptr0, align 64
-- 
GitLab


From d06ba376833553f38b5cbc606e479ed7936e5f5b Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal@paszkowski.org>
Date: Tue, 26 Mar 2024 03:04:49 -0700
Subject: [PATCH 257/404] [SPIR-V] Support extension toggling and enabling all
 (#85503)

---
 llvm/lib/Target/SPIRV/CMakeLists.txt          |   1 +
 llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp    | 101 ++++++++++++++++++
 llvm/lib/Target/SPIRV/SPIRVCommandLine.h      |  38 +++++++
 llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp      |  82 ++------------
 llvm/test/CodeGen/SPIRV/LinkOnceODR.ll        |   4 +-
 llvm/test/CodeGen/SPIRV/LinkOnceODRFun.ll     |   4 +-
 llvm/test/CodeGen/SPIRV/assume.ll             |   4 +-
 .../SPIRV/exec_mode_float_control_khr.ll      |   2 +-
 llvm/test/CodeGen/SPIRV/expect.ll             |   4 +-
 .../atomicrmw_faddfsub_double.ll              |   2 +-
 .../atomicrmw_faddfsub_float.ll               |   2 +-
 .../atomicrmw_faddfsub_half.ll                |   4 +-
 .../atomicrmw_fminfmax_double.ll              |   2 +-
 .../atomicrmw_fminfmax_float.ll               |   2 +-
 .../atomicrmw_fminfmax_half.ll                |   2 +-
 .../SPV_INTEL_arbitrary_precision_integers.ll |   2 +-
 .../bfloat16-conv-negative1.ll                |   2 +-
 .../bfloat16-conv-negative2.ll                |   2 +-
 .../bfloat16-conv-negative3.ll                |   2 +-
 .../bfloat16-conv-negative4.ll                |   2 +-
 .../bfloat16-conv.ll                          |   4 +-
 .../SPV_INTEL_function_pointers/fp_const.ll   |   2 +-
 .../fp_two_calls.ll                           |   2 +-
 .../SPIRV/extensions/SPV_INTEL_optnone.ll     |   2 +-
 .../cl_intel_sub_groups.ll                    |   2 +-
 .../intel-usm-addrspaces.ll                   |   4 +-
 .../SPV_INTEL_variable_length_array/vararr.ll |   4 +-
 .../vararr_spec_const.ll                      |   4 +-
 .../extensions/SPV_KHR_bit_instructions.ll    |   2 +-
 .../SPV_KHR_no_integer_wrap_decoration.ll     |   2 +-
 .../subgroup-rotate.ll                        |   4 +-
 .../uniform-group-instructions.ll             |   2 +-
 ...both-allowed-disallowed-extension-error.ll |   7 ++
 .../enable-all-extensions-but-one.ll          |   9 ++
 .../SPIRV/extensions/enable-all-extensions.ll |   7 ++
 .../invalid-extension-list-format.ll          |   6 ++
 .../extensions/unknown-extension-name.ll      |   6 ++
 .../llvm-intrinsics/llvm-vector-reduce/add.ll |   2 +-
 .../llvm-intrinsics/llvm-vector-reduce/and.ll |   2 +-
 .../llvm-vector-reduce/fadd.ll                |   2 +-
 .../llvm-vector-reduce/fmax.ll                |   2 +-
 .../llvm-vector-reduce/fmaximum.ll            |   2 +-
 .../llvm-vector-reduce/fmin.ll                |   2 +-
 .../llvm-vector-reduce/fminimum.ll            |   2 +-
 .../llvm-vector-reduce/fmul.ll                |   2 +-
 .../llvm-intrinsics/llvm-vector-reduce/mul.ll |   2 +-
 .../llvm-intrinsics/llvm-vector-reduce/or.ll  |   2 +-
 .../llvm-vector-reduce/smax.ll                |   2 +-
 .../llvm-vector-reduce/smin.ll                |   2 +-
 .../llvm-vector-reduce/umax.ll                |   2 +-
 .../llvm-vector-reduce/umin.ll                |   2 +-
 .../llvm-intrinsics/llvm-vector-reduce/xor.ll |   2 +-
 .../SPIRV/transcoding/NoSignedUnsignedWrap.ll |   2 +-
 53 files changed, 235 insertions(+), 130 deletions(-)
 create mode 100644 llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
 create mode 100644 llvm/lib/Target/SPIRV/SPIRVCommandLine.h
 create mode 100644 llvm/test/CodeGen/SPIRV/extensions/both-allowed-disallowed-extension-error.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/extensions/enable-all-extensions-but-one.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/extensions/enable-all-extensions.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/extensions/invalid-extension-list-format.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/extensions/unknown-extension-name.ll

diff --git a/llvm/lib/Target/SPIRV/CMakeLists.txt b/llvm/lib/Target/SPIRV/CMakeLists.txt
index afc26dda4c68..7001ac382f41 100644
--- a/llvm/lib/Target/SPIRV/CMakeLists.txt
+++ b/llvm/lib/Target/SPIRV/CMakeLists.txt
@@ -17,6 +17,7 @@ add_llvm_target(SPIRVCodeGen
   SPIRVAsmPrinter.cpp
   SPIRVBuiltins.cpp
   SPIRVCallLowering.cpp
+  SPIRVCommandLine.cpp
   SPIRVDuplicatesTracker.cpp
   SPIRVEmitIntrinsics.cpp
   SPIRVGlobalRegistry.cpp
diff --git a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
new file mode 100644
index 000000000000..691e6ee0e582
--- /dev/null
+++ b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
@@ -0,0 +1,101 @@
+//===--- SPIRVCommandLine.cpp ---- Command Line Options ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains definitions of classes and functions needed for
+// processing, parsing, and using CLI options for the SPIR-V backend.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SPIRVCommandLine.h"
+#include "llvm/ADT/StringRef.h"
+#include <algorithm>
+#include <map>
+
+#define DEBUG_TYPE "spirv-commandline"
+
+using namespace llvm;
+
+static const std::map<std::string, SPIRV::Extension::Extension>
+    SPIRVExtensionMap = {
+        {"SPV_EXT_shader_atomic_float_add",
+         SPIRV::Extension::Extension::SPV_EXT_shader_atomic_float_add},
+        {"SPV_EXT_shader_atomic_float16_add",
+         SPIRV::Extension::Extension::SPV_EXT_shader_atomic_float16_add},
+        {"SPV_EXT_shader_atomic_float_min_max",
+         SPIRV::Extension::Extension::SPV_EXT_shader_atomic_float_min_max},
+        {"SPV_INTEL_arbitrary_precision_integers",
+         SPIRV::Extension::Extension::SPV_INTEL_arbitrary_precision_integers},
+        {"SPV_INTEL_optnone", SPIRV::Extension::Extension::SPV_INTEL_optnone},
+        {"SPV_INTEL_usm_storage_classes",
+         SPIRV::Extension::Extension::SPV_INTEL_usm_storage_classes},
+        {"SPV_INTEL_subgroups",
+         SPIRV::Extension::Extension::SPV_INTEL_subgroups},
+        {"SPV_KHR_uniform_group_instructions",
+         SPIRV::Extension::Extension::SPV_KHR_uniform_group_instructions},
+        {"SPV_KHR_no_integer_wrap_decoration",
+         SPIRV::Extension::Extension::SPV_KHR_no_integer_wrap_decoration},
+        {"SPV_KHR_float_controls",
+         SPIRV::Extension::Extension::SPV_KHR_float_controls},
+        {"SPV_KHR_expect_assume",
+         SPIRV::Extension::Extension::SPV_KHR_expect_assume},
+        {"SPV_KHR_bit_instructions",
+         SPIRV::Extension::Extension::SPV_KHR_bit_instructions},
+        {"SPV_KHR_linkonce_odr",
+         SPIRV::Extension::Extension::SPV_KHR_linkonce_odr},
+        {"SPV_INTEL_bfloat16_conversion",
+         SPIRV::Extension::Extension::SPV_INTEL_bfloat16_conversion},
+        {"SPV_KHR_subgroup_rotate",
+         SPIRV::Extension::Extension::SPV_KHR_subgroup_rotate},
+        {"SPV_INTEL_variable_length_array",
+         SPIRV::Extension::Extension::SPV_INTEL_variable_length_array},
+        {"SPV_INTEL_function_pointers",
+         SPIRV::Extension::Extension::SPV_INTEL_function_pointers},
+};
+
+bool SPIRVExtensionsParser::parse(cl::Option &O, llvm::StringRef ArgName,
+                                  llvm::StringRef ArgValue,
+                                  std::set<SPIRV::Extension::Extension> &Vals) {
+  llvm::SmallVector<llvm::StringRef, 10> Tokens;
+  ArgValue.split(Tokens, ",", -1, false);
+  std::sort(Tokens.begin(), Tokens.end());
+
+  std::set<SPIRV::Extension::Extension> EnabledExtensions;
+
+  for (const auto &Token : Tokens) {
+    if (Token == "all") {
+      for (const auto &[ExtensionName, ExtensionEnum] : SPIRVExtensionMap)
+        EnabledExtensions.insert(ExtensionEnum);
+
+      continue;
+    }
+
+    if (Token.empty() || (!Token.starts_with("+") && !Token.starts_with("-")))
+      return O.error("Invalid extension list format: " + Token.str());
+
+    llvm::StringRef ExtensionName = Token.substr(1);
+    auto NameValuePair = SPIRVExtensionMap.find(ExtensionName.str());
+
+    if (NameValuePair == SPIRVExtensionMap.end())
+      return O.error("Unknown SPIR-V extension: " + Token.str());
+
+    if (Token.starts_with("+")) {
+      EnabledExtensions.insert(NameValuePair->second);
+    } else if (EnabledExtensions.count(NameValuePair->second)) {
+      if (std::find(Tokens.begin(), Tokens.end(), "+" + ExtensionName.str()) !=
+          Tokens.end())
+        return O.error(
+            "Extension cannot be allowed and disallowed at the same time: " +
+            ExtensionName.str());
+
+      EnabledExtensions.erase(NameValuePair->second);
+    }
+  }
+
+  Vals = std::move(EnabledExtensions);
+  return false;
+}
diff --git a/llvm/lib/Target/SPIRV/SPIRVCommandLine.h b/llvm/lib/Target/SPIRV/SPIRVCommandLine.h
new file mode 100644
index 000000000000..741d829b2ab8
--- /dev/null
+++ b/llvm/lib/Target/SPIRV/SPIRVCommandLine.h
@@ -0,0 +1,38 @@
+//===--- SPIRVCommandLine.h ---- Command Line Options -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains classes and functions needed for processing, parsing, and
+// using CLI options for the SPIR-V backend.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_SPIRV_COMMANDLINE_H
+#define LLVM_LIB_TARGET_SPIRV_COMMANDLINE_H
+
+#include "MCTargetDesc/SPIRVBaseInfo.h"
+#include "llvm/Support/CommandLine.h"
+#include <set>
+
+namespace llvm {
+
+/// Command line parser for toggling SPIR-V extensions.
+struct SPIRVExtensionsParser
+    : public cl::parser<std::set<SPIRV::Extension::Extension>> {
+public:
+  SPIRVExtensionsParser(cl::Option &O)
+      : cl::parser<std::set<SPIRV::Extension::Extension>>(O) {}
+
+  /// Parses SPIR-V extension name from CLI arguments.
+  ///
+  /// \return Returns true on error.
+  bool parse(cl::Option &O, StringRef ArgName, StringRef ArgValue,
+             std::set<SPIRV::Extension::Extension> &Vals);
+};
+
+} // namespace llvm
+#endif // LLVM_LIB_TARGET_SPIRV_COMMANDLINE_H
diff --git a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp
index 09a029a35a74..f3864b56e1e9 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp
@@ -12,6 +12,7 @@
 
 #include "SPIRVSubtarget.h"
 #include "SPIRV.h"
+#include "SPIRVCommandLine.h"
 #include "SPIRVGlobalRegistry.h"
 #include "SPIRVLegalizerInfo.h"
 #include "SPIRVRegisterBankInfo.h"
@@ -32,80 +33,10 @@ static cl::opt<bool>
                         cl::desc("SPIR-V Translator compatibility mode"),
                         cl::Optional, cl::init(false));
 
-cl::list<SPIRV::Extension::Extension> Extensions(
-    "spirv-extensions", cl::desc("SPIR-V extensions"), cl::ZeroOrMore,
-    cl::Hidden,
-    cl::values(
-        clEnumValN(SPIRV::Extension::SPV_EXT_shader_atomic_float_add,
-                   "SPV_EXT_shader_atomic_float_add",
-                   "Adds atomic add instruction on floating-point numbers."),
-        clEnumValN(
-            SPIRV::Extension::SPV_EXT_shader_atomic_float16_add,
-            "SPV_EXT_shader_atomic_float16_add",
-            "Extends the SPV_EXT_shader_atomic_float_add extension to support "
-            "atomically adding to 16-bit floating-point numbers in memory."),
-        clEnumValN(
-            SPIRV::Extension::SPV_EXT_shader_atomic_float_min_max,
-            "SPV_EXT_shader_atomic_float_min_max",
-            "Adds atomic min and max instruction on floating-point numbers."),
-        clEnumValN(SPIRV::Extension::SPV_INTEL_arbitrary_precision_integers,
-                   "SPV_INTEL_arbitrary_precision_integers",
-                   "Allows generating arbitrary width integer types."),
-        clEnumValN(SPIRV::Extension::SPV_INTEL_optnone, "SPV_INTEL_optnone",
-                   "Adds OptNoneINTEL value for Function Control mask that "
-                   "indicates a request to not optimize the function."),
-        clEnumValN(SPIRV::Extension::SPV_INTEL_usm_storage_classes,
-                   "SPV_INTEL_usm_storage_classes",
-                   "Introduces two new storage classes that are sub classes of "
-                   "the CrossWorkgroup storage class "
-                   "that provides additional information that can enable "
-                   "optimization."),
-        clEnumValN(SPIRV::Extension::SPV_INTEL_subgroups, "SPV_INTEL_subgroups",
-                   "Allows work items in a subgroup to share data without the "
-                   "use of local memory and work group barriers, and to "
-                   "utilize specialized hardware to load and store blocks of "
-                   "data from images or buffers."),
-        clEnumValN(SPIRV::Extension::SPV_KHR_uniform_group_instructions,
-                   "SPV_KHR_uniform_group_instructions",
-                   "Allows support for additional group operations within "
-                   "uniform control flow."),
-        clEnumValN(SPIRV::Extension::SPV_KHR_no_integer_wrap_decoration,
-                   "SPV_KHR_no_integer_wrap_decoration",
-                   "Adds decorations to indicate that a given instruction does "
-                   "not cause integer wrapping."),
-        clEnumValN(
-            SPIRV::Extension::SPV_KHR_float_controls, "SPV_KHR_float_controls",
-            "Provides new execution modes to control floating-point "
-            "computations by overriding an implementation’s default behavior "
-            "for rounding modes, denormals, signed zero, and infinities."),
-        clEnumValN(SPIRV::Extension::SPV_KHR_expect_assume,
-                   "SPV_KHR_expect_assume",
-                   "Provides additional information to a compiler, similar to "
-                   "the llvm.assume and llvm.expect intrinsics."),
-        clEnumValN(SPIRV::Extension::SPV_KHR_bit_instructions,
-                   "SPV_KHR_bit_instructions",
-                   "This enables bit instructions to be used by SPIR-V modules "
-                   "without requiring the Shader capability."),
-        clEnumValN(
-            SPIRV::Extension::SPV_KHR_linkonce_odr, "SPV_KHR_linkonce_odr",
-            "Allows to use the LinkOnceODR linkage type that is to let "
-            "a function or global variable to be merged with other functions "
-            "or global variables of the same name when linkage occurs."),
-        clEnumValN(SPIRV::Extension::SPV_INTEL_bfloat16_conversion,
-                   "SPV_INTEL_bfloat16_conversion",
-                   "Adds instructions to convert between single-precision "
-                   "32-bit floating-point values and 16-bit bfloat16 values."),
-        clEnumValN(SPIRV::Extension::SPV_KHR_subgroup_rotate,
-                   "SPV_KHR_subgroup_rotate",
-                   "Adds a new instruction that enables rotating values across "
-                   "invocations within a subgroup."),
-        clEnumValN(SPIRV::Extension::SPV_INTEL_variable_length_array,
-                   "SPV_INTEL_variable_length_array",
-                   "Allows to allocate local arrays whose number of elements "
-                   "is unknown at compile time."),
-        clEnumValN(SPIRV::Extension::SPV_INTEL_function_pointers,
-                   "SPV_INTEL_function_pointers",
-                   "Allows translation of function pointers.")));
+static cl::opt<std::set<SPIRV::Extension::Extension>, false,
+               SPIRVExtensionsParser>
+    Extensions("spirv-ext",
+               cl::desc("Specify list of enabled SPIR-V extensions"));
 
 // Compare version numbers, but allow 0 to mean unspecified.
 static bool isAtLeastVer(uint32_t Target, uint32_t VerToCompareTo) {
@@ -172,8 +103,7 @@ void SPIRVSubtarget::initAvailableExtensions() {
   if (!isOpenCLEnv())
     return;
 
-  for (auto Extension : Extensions)
-    AvailableExtensions.insert(Extension);
+  AvailableExtensions.insert(Extensions.begin(), Extensions.end());
 }
 
 // TODO: use command line args for this rather than just defaults.
diff --git a/llvm/test/CodeGen/SPIRV/LinkOnceODR.ll b/llvm/test/CodeGen/SPIRV/LinkOnceODR.ll
index 3dfdeac7adaa..ec660b77368e 100644
--- a/llvm/test/CodeGen/SPIRV/LinkOnceODR.ll
+++ b/llvm/test/CodeGen/SPIRV/LinkOnceODR.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_KHR_linkonce_odr %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV-EXT
-; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_KHR_linkonce_odr %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_KHR_linkonce_odr %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV-EXT
+; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_KHR_linkonce_odr %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-SPIRV-EXT: Capability Linkage
 ; CHECK-SPIRV-EXT: Extension "SPV_KHR_linkonce_odr"
diff --git a/llvm/test/CodeGen/SPIRV/LinkOnceODRFun.ll b/llvm/test/CodeGen/SPIRV/LinkOnceODRFun.ll
index 7505c3fc277e..42170dce5d13 100644
--- a/llvm/test/CodeGen/SPIRV/LinkOnceODRFun.ll
+++ b/llvm/test/CodeGen/SPIRV/LinkOnceODRFun.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_KHR_linkonce_odr %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV-EXT
-; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_KHR_linkonce_odr %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_KHR_linkonce_odr %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV-EXT
+; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_KHR_linkonce_odr %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-SPIRV-EXT: Capability Linkage
 ; CHECK-SPIRV-EXT: Extension "SPV_KHR_linkonce_odr"
diff --git a/llvm/test/CodeGen/SPIRV/assume.ll b/llvm/test/CodeGen/SPIRV/assume.ll
index 6099955e4afb..fbf12ef184a8 100644
--- a/llvm/test/CodeGen/SPIRV/assume.ll
+++ b/llvm/test/CodeGen/SPIRV/assume.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_KHR_expect_assume < %s | FileCheck --check-prefixes=EXT,CHECK %s
-; RUN: llc -mtriple=spirv64-unknown-unknown --spirv-extensions=SPV_KHR_expect_assume < %s | FileCheck --check-prefixes=EXT,CHECK %s
+; RUN: llc -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_KHR_expect_assume < %s | FileCheck --check-prefixes=EXT,CHECK %s
+; RUN: llc -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_expect_assume < %s | FileCheck --check-prefixes=EXT,CHECK %s
 ; RUN: llc -mtriple=spirv32-unknown-unknown < %s | FileCheck --check-prefixes=NOEXT,CHECK %s
 ; RUN: llc -mtriple=spirv64-unknown-unknown < %s | FileCheck --check-prefixes=NOEXT,CHECK %s
 
diff --git a/llvm/test/CodeGen/SPIRV/exec_mode_float_control_khr.ll b/llvm/test/CodeGen/SPIRV/exec_mode_float_control_khr.ll
index 473794a1ac97..721e825a1c98 100644
--- a/llvm/test/CodeGen/SPIRV/exec_mode_float_control_khr.ll
+++ b/llvm/test/CodeGen/SPIRV/exec_mode_float_control_khr.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefixes=SPV
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s --mattr=+spirv1.3 --spirv-extensions=SPV_KHR_float_controls -o - | FileCheck %s --check-prefixes=SPVEXT
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s --mattr=+spirv1.3 --spirv-ext=+SPV_KHR_float_controls -o - | FileCheck %s --check-prefixes=SPVEXT
 
 define dso_local dllexport spir_kernel void @k_float_controls_0(i32 %ibuf, i32 %obuf) local_unnamed_addr {
 entry:
diff --git a/llvm/test/CodeGen/SPIRV/expect.ll b/llvm/test/CodeGen/SPIRV/expect.ll
index 51555cd15552..82c1ec7dc916 100644
--- a/llvm/test/CodeGen/SPIRV/expect.ll
+++ b/llvm/test/CodeGen/SPIRV/expect.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_KHR_expect_assume < %s | FileCheck --check-prefixes=CHECK,EXT %s
-; RUN: llc -mtriple=spirv64-unknown-unknown --spirv-extensions=SPV_KHR_expect_assume < %s | FileCheck --check-prefixes=CHECK,EXT %s
+; RUN: llc -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_KHR_expect_assume < %s | FileCheck --check-prefixes=CHECK,EXT %s
+; RUN: llc -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_expect_assume < %s | FileCheck --check-prefixes=CHECK,EXT %s
 ; RUN: llc -mtriple=spirv32-unknown-unknown < %s | FileCheck --check-prefixes=CHECK,NOEXT %s
 ; RUN: llc -mtriple=spirv64-unknown-unknown < %s | FileCheck --check-prefixes=CHECK,NOEXT %s
 
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_add/atomicrmw_faddfsub_double.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_add/atomicrmw_faddfsub_double.ll
index 1bfa556affce..e7b66798d3b3 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_add/atomicrmw_faddfsub_double.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_add/atomicrmw_faddfsub_double.ll
@@ -1,6 +1,6 @@
 ; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
 
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_EXT_shader_atomic_float_add %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_EXT_shader_atomic_float_add %s -o - | FileCheck %s
 
 ; CHECK-ERROR: LLVM ERROR: The atomic float instruction requires the following SPIR-V extension: SPV_EXT_shader_atomic_float_add
 
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_add/atomicrmw_faddfsub_float.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_add/atomicrmw_faddfsub_float.ll
index 627b59fee1a6..4fb99d9dfc76 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_add/atomicrmw_faddfsub_float.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_add/atomicrmw_faddfsub_float.ll
@@ -1,6 +1,6 @@
 ; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
 
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_EXT_shader_atomic_float_add %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_EXT_shader_atomic_float_add %s -o - | FileCheck %s
 
 ; CHECK-ERROR: LLVM ERROR: The atomic float instruction requires the following SPIR-V extension: SPV_EXT_shader_atomic_float_add
 
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_add/atomicrmw_faddfsub_half.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_add/atomicrmw_faddfsub_half.ll
index fffda4bd1128..2f536dc53843 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_add/atomicrmw_faddfsub_half.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_add/atomicrmw_faddfsub_half.ll
@@ -1,7 +1,7 @@
 ; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR1
-; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_EXT_shader_atomic_float_add %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR2
+; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_EXT_shader_atomic_float_add %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR2
 
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_EXT_shader_atomic_float_add --spirv-extensions=SPV_EXT_shader_atomic_float16_add %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_EXT_shader_atomic_float_add,+SPV_EXT_shader_atomic_float16_add %s -o - | FileCheck %s
 
 ; CHECK-ERROR1: LLVM ERROR: The atomic float instruction requires the following SPIR-V extension: SPV_EXT_shader_atomic_float_add
 ; CHECK-ERROR2: LLVM ERROR: The atomic float instruction requires the following SPIR-V extension: SPV_EXT_shader_atomic_float16_add
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_double.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_double.ll
index 3c6fa276cb76..7654c3618fd4 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_double.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_double.ll
@@ -1,6 +1,6 @@
 ; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
 
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_EXT_shader_atomic_float_min_max %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_EXT_shader_atomic_float_min_max %s -o - | FileCheck %s
 
 ; CHECK-ERROR: LLVM ERROR: The atomic float instruction requires the following SPIR-V extension: SPV_EXT_shader_atomic_float_min_max
 
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_float.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_float.ll
index cc52e4c72055..8a3599002822 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_float.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_float.ll
@@ -1,6 +1,6 @@
 ; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
 
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_EXT_shader_atomic_float_min_max %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_EXT_shader_atomic_float_min_max %s -o - | FileCheck %s
 
 ; CHECK-ERROR: LLVM ERROR: The atomic float instruction requires the following SPIR-V extension: SPV_EXT_shader_atomic_float_min_max
 
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_half.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_half.ll
index b406aee528b2..45baaa8e9749 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_half.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_half.ll
@@ -1,6 +1,6 @@
 ; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
 
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_EXT_shader_atomic_float_min_max %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_EXT_shader_atomic_float_min_max %s -o - | FileCheck %s
 
 ; CHECK-ERROR: LLVM ERROR: The atomic float instruction requires the following SPIR-V extension: SPV_EXT_shader_atomic_float_min_max
 
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_arbitrary_precision_integers.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_arbitrary_precision_integers.ll
index b68fb363ad85..f49367c50e0e 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_arbitrary_precision_integers.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_arbitrary_precision_integers.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_arbitrary_precision_integers %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_arbitrary_precision_integers %s -o - | FileCheck %s
 
 define i6 @getConstantI6() {
   ret i6 2
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv-negative1.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv-negative1.ll
index 2f3c859db346..4326d8df8820 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv-negative1.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv-negative1.ll
@@ -1,4 +1,4 @@
-; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_bfloat16_conversion %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_bfloat16_conversion %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
 ; CHECK-ERROR: result and argument must have the same number of components
 
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv-negative2.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv-negative2.ll
index 8ab84d63d485..57f52b97c4cc 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv-negative2.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv-negative2.ll
@@ -1,4 +1,4 @@
-; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_bfloat16_conversion %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_bfloat16_conversion %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
 ; CHECK-ERROR: result and argument must have the same number of components
 
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv-negative3.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv-negative3.ll
index 20a8042ad9c2..2cb229eb7ef7 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv-negative3.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv-negative3.ll
@@ -1,4 +1,4 @@
-; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_bfloat16_conversion %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_bfloat16_conversion %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
 ; CHECK-ERROR: result and argument must have the same number of components
 
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv-negative4.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv-negative4.ll
index efbd50b77dc6..eb5a2c729102 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv-negative4.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv-negative4.ll
@@ -1,4 +1,4 @@
-; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_bfloat16_conversion %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_bfloat16_conversion %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
 ; CHECK-ERROR: result and argument must have the same number of components
 
 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64"
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv.ll
index 2bd59b22322f..91fa340e4611 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_bfloat16_conversion/bfloat16-conv.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_bfloat16_conversion %s -o - | FileCheck %s
-; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-extensions=SPV_INTEL_bfloat16_conversion %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_bfloat16_conversion %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_bfloat16_conversion %s -o - -filetype=obj | spirv-val %}
 
 ; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
 ; CHECK-ERROR: the builtin requires the following SPIR-V extension: SPV_INTEL_bfloat16_conversion
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fp_const.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fp_const.ll
index 0bd1b5d776a9..5f073e95cb68 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fp_const.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fp_const.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_function_pointers %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
 ; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: OpCapability Int8
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fp_two_calls.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fp_two_calls.ll
index 89de098dead9..b7fecefe9a58 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fp_two_calls.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_function_pointers/fp_two_calls.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_function_pointers %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
 ; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG: OpCapability Int8
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_optnone.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_optnone.ll
index afbcaec9c5c5..a611be8eb6ee 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_optnone.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_optnone.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_optnone %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-EXTENSION
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_optnone %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-EXTENSION
 ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-NO-EXTENSION
 
 ; CHECK-EXTENSION: OpCapability OptNoneINTEL
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_subgroups/cl_intel_sub_groups.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_subgroups/cl_intel_sub_groups.ll
index 0e0b2a4dd6ec..df17ec435ad3 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_subgroups/cl_intel_sub_groups.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_subgroups/cl_intel_sub_groups.ll
@@ -37,7 +37,7 @@
 
 ; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
 
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_subgroups %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_subgroups %s -o - | FileCheck %s
 
 ; CHECK-ERROR: LLVM ERROR: intel_sub_group_shuffle: the builtin requires the following SPIR-V extension: SPV_INTEL_subgroups
 
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_usm_storage_classes/intel-usm-addrspaces.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_usm_storage_classes/intel-usm-addrspaces.ll
index 30c16350bf2b..b5df462bd8fa 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_usm_storage_classes/intel-usm-addrspaces.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_usm_storage_classes/intel-usm-addrspaces.ll
@@ -1,7 +1,7 @@
 ; Modified from: https://github.com/KhronosGroup/SPIRV-LLVM-Translator/test/extensions/INTEL/SPV_INTEL_usm_storage_classes/intel_usm_addrspaces.ll
 
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_usm_storage_classes %s -o - | FileCheck %s --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-EXT
-; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-extensions=SPV_INTEL_usm_storage_classes %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_usm_storage_classes %s -o - | FileCheck %s --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-EXT
+; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_usm_storage_classes %s -o - -filetype=obj | spirv-val %}
 ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-WITHOUT
 ; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_variable_length_array/vararr.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_variable_length_array/vararr.ll
index 897aab70852d..8a54d22a539d 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_variable_length_array/vararr.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_variable_length_array/vararr.ll
@@ -1,8 +1,8 @@
 ; Modified from: https://github.com/KhronosGroup/SPIRV-LLVM-Translator/test/extensions/INTEL/SPV_INTEL_variable_length_array/basic.ll
 
 ; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_variable_length_array %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
-; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_variable_length_array %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_variable_length_array %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_variable_length_array %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-ERROR: LLVM ERROR: array allocation: this instruction requires the following SPIR-V extension: SPV_INTEL_variable_length_array
 
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_variable_length_array/vararr_spec_const.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_variable_length_array/vararr_spec_const.ll
index fbac43e51f39..7b9f75d74db9 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_variable_length_array/vararr_spec_const.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_variable_length_array/vararr_spec_const.ll
@@ -1,7 +1,7 @@
 ; Modified from: https://github.com/KhronosGroup/SPIRV-LLVM-Translator/test/extensions/INTEL/SPV_INTEL_variable_length_array/vla_spec_const.ll
 
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_variable_length_array %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
-; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_variable_length_array %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_variable_length_array %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_variable_length_array %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-SPIRV: Capability VariableLengthArrayINTEL
 ; CHECK-SPIRV: Extension "SPV_INTEL_variable_length_array"
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_bit_instructions.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_bit_instructions.ll
index 95395d5efb55..100f02faba85 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_bit_instructions.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_bit_instructions.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s --spirv-extensions=SPV_KHR_bit_instructions -o - | FileCheck %s --check-prefix=CHECK-EXTENSION
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s --spirv-ext=+SPV_KHR_bit_instructions -o - | FileCheck %s --check-prefix=CHECK-EXTENSION
 ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-NO-EXTENSION
 
 ; CHECK-EXTENSION:      OpCapability BitInstructions
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_no_integer_wrap_decoration.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_no_integer_wrap_decoration.ll
index e74dd99617f9..0d9ab4ab65ce 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_no_integer_wrap_decoration.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_no_integer_wrap_decoration.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_KHR_no_integer_wrap_decoration %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_KHR_no_integer_wrap_decoration %s -o - | FileCheck %s
 
 ; CHECK-DAG: OpExtension "SPV_KHR_no_integer_wrap_decoration"
 
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_subgroup_rotate/subgroup-rotate.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_subgroup_rotate/subgroup-rotate.ll
index b1d6a09c7fe3..63aade4f7f8d 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_subgroup_rotate/subgroup-rotate.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_subgroup_rotate/subgroup-rotate.ll
@@ -1,6 +1,6 @@
 ; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_KHR_subgroup_rotate %s -o - | FileCheck %s
-; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-extensions=SPV_KHR_subgroup_rotate %s -o - -filetype=obj | spirv-val %}
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_KHR_subgroup_rotate %s -o - | FileCheck %s
+; TODO: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_KHR_subgroup_rotate %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-ERROR: LLVM ERROR: OpGroupNonUniformRotateKHR instruction requires the following SPIR-V extension: SPV_KHR_subgroup_rotate
 
diff --git a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_uniform_group_instructions/uniform-group-instructions.ll b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_uniform_group_instructions/uniform-group-instructions.ll
index 39bf63ddae4f..0de654be8ed7 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_uniform_group_instructions/uniform-group-instructions.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/SPV_KHR_uniform_group_instructions/uniform-group-instructions.ll
@@ -1,6 +1,6 @@
 ; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
 
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_KHR_uniform_group_instructions %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_KHR_uniform_group_instructions %s -o - | FileCheck %s
 
 ; CHECK-ERROR: LLVM ERROR: __spirv_GroupBitwiseAndKHR: the builtin requires the following SPIR-V extension: SPV_KHR_uniform_group_instructions
 
diff --git a/llvm/test/CodeGen/SPIRV/extensions/both-allowed-disallowed-extension-error.ll b/llvm/test/CodeGen/SPIRV/extensions/both-allowed-disallowed-extension-error.ll
new file mode 100644
index 000000000000..fc07cca4dd24
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/both-allowed-disallowed-extension-error.ll
@@ -0,0 +1,7 @@
+; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_arbitrary_precision_integers,-SPV_INTEL_arbitrary_precision_integers %s -o %t.spvt 2>&1 | FileCheck %s
+; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=-SPV_INTEL_arbitrary_precision_integers,+SPV_INTEL_arbitrary_precision_integers %s -o %t.spvt 2>&1 | FileCheck %s
+; CHECK: Extension cannot be allowed and disallowed at the same time: SPV_INTEL_arbitrary_precision_integers
+
+define i8 @foo() {
+  ret i8 2
+}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/enable-all-extensions-but-one.ll b/llvm/test/CodeGen/SPIRV/extensions/enable-all-extensions-but-one.ll
new file mode 100644
index 000000000000..973a5e6f6056
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/enable-all-extensions-but-one.ll
@@ -0,0 +1,9 @@
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=all,-SPV_INTEL_arbitrary_precision_integers %s -o - | FileCheck %s
+
+define i6 @foo() {
+  %call = tail call i32 @llvm.bitreverse.i32(i32 42)
+  ret i6 2
+}
+
+; CHECK-NOT: OpExtension "SPV_INTEL_arbitrary_precision_integers"
+; CHECK-DAG: OpExtension "SPV_KHR_bit_instructions"
diff --git a/llvm/test/CodeGen/SPIRV/extensions/enable-all-extensions.ll b/llvm/test/CodeGen/SPIRV/extensions/enable-all-extensions.ll
new file mode 100644
index 000000000000..a5b979469b93
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/enable-all-extensions.ll
@@ -0,0 +1,7 @@
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=all %s -o - | FileCheck %s
+
+define i6 @getConstantI6() {
+  ret i6 2
+}
+
+; CHECK: OpExtension "SPV_INTEL_arbitrary_precision_integers"
diff --git a/llvm/test/CodeGen/SPIRV/extensions/invalid-extension-list-format.ll b/llvm/test/CodeGen/SPIRV/extensions/invalid-extension-list-format.ll
new file mode 100644
index 000000000000..207ed4bf2572
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/invalid-extension-list-format.ll
@@ -0,0 +1,6 @@
+; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=UNKNOWN_EXTENSION %s -o %t.spvt 2>&1 | FileCheck %s
+; CHECK: Invalid extension list format: UNKNOWN_EXTENSION
+
+define i8 @foo() {
+  ret i8 2
+}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/unknown-extension-name.ll b/llvm/test/CodeGen/SPIRV/extensions/unknown-extension-name.ll
new file mode 100644
index 000000000000..f4f54247c2ad
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/unknown-extension-name.ll
@@ -0,0 +1,6 @@
+; RUN: not llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+UNKNOWN_EXTENSION %s -o %t.spvt 2>&1 | FileCheck %s
+; CHECK: Unknown SPIR-V extension: +UNKNOWN_EXTENSION
+
+define i8 @foo() {
+  ret i8 2
+}
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/add.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/add.ll
index 93190f972d9b..e0c84ee3a3f1 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/add.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/add.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_function_pointers %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 target triple = "spir64-unknown-unknown"
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/and.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/and.ll
index aa879b22e057..12a4a86fa4a8 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/and.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/and.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_function_pointers %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 target triple = "spir64-unknown-unknown"
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fadd.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fadd.ll
index a0d18d5a750f..459bc6bdcdaf 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fadd.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fadd.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_function_pointers %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 target triple = "spir64-unknown-unknown"
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fmax.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fmax.ll
index 247ebcca2dd8..4f9cd29cd05d 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fmax.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fmax.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_function_pointers %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 target triple = "spir64-unknown-unknown"
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fmaximum.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fmaximum.ll
index 13f44108733c..837bea0fbe62 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fmaximum.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fmaximum.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_function_pointers %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 target triple = "spir64-unknown-unknown"
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fmin.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fmin.ll
index 13ef11862166..475da2e1ec31 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fmin.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fmin.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_function_pointers %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 target triple = "spir64-unknown-unknown"
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fminimum.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fminimum.ll
index 93ef79affdda..b525c849c3b5 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fminimum.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fminimum.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_function_pointers %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 target triple = "spir64-unknown-unknown"
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fmul.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fmul.ll
index afe30d5b51e0..0985be992ca7 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fmul.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/fmul.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_function_pointers %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 target triple = "spir64-unknown-unknown"
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/mul.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/mul.ll
index 9b397aee30cb..1a700577e46b 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/mul.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/mul.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_function_pointers %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 target triple = "spir64-unknown-unknown"
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/or.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/or.ll
index a62bb0c376f3..90c6cf5562a9 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/or.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/or.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_function_pointers %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 target triple = "spir64-unknown-unknown"
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/smax.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/smax.ll
index 3fc2bcc86054..4551fa31681d 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/smax.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/smax.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_function_pointers %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 target triple = "spir64-unknown-unknown"
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/smin.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/smin.ll
index 9459946a408f..a0d257bb1318 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/smin.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/smin.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_function_pointers %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 target triple = "spir64-unknown-unknown"
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/umax.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/umax.ll
index cce7189c173c..ba5dba76aeec 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/umax.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/umax.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_function_pointers %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 target triple = "spir64-unknown-unknown"
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/umin.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/umin.ll
index bcc49c50d480..e16bde88ef50 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/umin.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/umin.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_function_pointers %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 target triple = "spir64-unknown-unknown"
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/xor.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/xor.ll
index 26bc96b41a98..cf887bb358ac 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/xor.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/xor.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_INTEL_function_pointers %s -o - | FileCheck %s
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_INTEL_function_pointers %s -o - | FileCheck %s
 ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 target triple = "spir64-unknown-unknown"
diff --git a/llvm/test/CodeGen/SPIRV/transcoding/NoSignedUnsignedWrap.ll b/llvm/test/CodeGen/SPIRV/transcoding/NoSignedUnsignedWrap.ll
index 02d1250399f9..e405ef0ed58a 100644
--- a/llvm/test/CodeGen/SPIRV/transcoding/NoSignedUnsignedWrap.ll
+++ b/llvm/test/CodeGen/SPIRV/transcoding/NoSignedUnsignedWrap.ll
@@ -7,7 +7,7 @@
 ;;
 ;; Positive tests:
 ;;
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-extensions=SPV_KHR_no_integer_wrap_decoration %s -o - | FileCheck %s --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-NEGATIVE
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=+SPV_KHR_no_integer_wrap_decoration %s -o - | FileCheck %s --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-NEGATIVE
 ;;
 ;; Negative tests:
 ;;
-- 
GitLab


From f5e1cd5625bccef71b7c914b39fcc5d547ebb610 Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal@paszkowski.org>
Date: Tue, 26 Mar 2024 03:06:49 -0700
Subject: [PATCH 258/404] [SPIR-V] Extend SPIRVUsage.rst document (#84744)

---
 llvm/docs/SPIRVUsage.rst | 334 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 327 insertions(+), 7 deletions(-)

diff --git a/llvm/docs/SPIRVUsage.rst b/llvm/docs/SPIRVUsage.rst
index 5ca035b68392..a183877c48cb 100644
--- a/llvm/docs/SPIRVUsage.rst
+++ b/llvm/docs/SPIRVUsage.rst
@@ -14,10 +14,43 @@ Introduction
 The SPIR-V target provides code generation for the SPIR-V binary format described
 in  `the official SPIR-V specification <https://www.khronos.org/registry/SPIR-V/>`_.
 
+Usage
+=====
+
+The SPIR-V backend can be invoked either from LLVM's Static Compiler (llc) or Clang, 
+allowing developers to compile LLVM intermediate language (IL) files or OpenCL kernel 
+sources directly to SPIR-V. This section outlines the usage of various commands to 
+leverage the SPIR-V backend for different purposes.
+
+Static Compiler Commands
+------------------------
+
+1. **Basic SPIR-V Compilation**
+   Command: `llc -mtriple=spirv32-unknown-unknown input.ll -o output.spvt`
+   Description: This command compiles an LLVM IL file (`input.ll`) to a SPIR-V binary (`output.spvt`) for a 32-bit architecture.
+
+2. **Compilation with Extensions and Optimization**
+   Command: `llc -O1 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_arbitrary_precision_integers input.ll -o output.spvt`
+   Description: Compiles an LLVM IL file to SPIR-V with (`-O1`) optimizations, targeting a 64-bit architecture. It enables the SPV_INTEL_arbitrary_precision_integers extension.
+
+3. **SPIR-V Binary Generation**
+   Command: `llc -O0 -mtriple=spirv64-unknown-unknown -filetype=obj input.ll -o output.spvt`
+   Description: Generates a SPIR-V object file (`output.spvt`) from an LLVM module, targeting a 64-bit SPIR-V architecture with no optimizations.
+
+Clang Commands
+--------------
+
+1. **SPIR-V Generation**
+   Command: `clang –target=spirv64 input.cl`
+   Description: Generates a SPIR-V file directly from an OpenCL kernel source file (`input.cl`).
+
+Compiler Options
+================
+
 .. _spirv-target-triples:
 
 Target Triples
-==============
+--------------
 
 For cross-compilation into SPIR-V use option
 
@@ -32,6 +65,7 @@ to specify the target triple:
      ============ ==============================================================
      ``spirv32``   SPIR-V with 32-bit pointer width.
      ``spirv64``   SPIR-V with 64-bit pointer width.
+     ``spirv``     SPIR-V with logical memory layout.
      ============ ==============================================================
 
   .. table:: SPIR-V Subarchitectures
@@ -39,13 +73,14 @@ to specify the target triple:
      =============== ==============================================================
      Subarchitecture Description
      =============== ==============================================================
-     *<empty>*        SPIR-V version deduced by tools based on the compiled input.
+     *<empty>*        SPIR-V version deduced by backend based on the input.
      ``v1.0``         SPIR-V version 1.0.
      ``v1.1``         SPIR-V version 1.1.
      ``v1.2``         SPIR-V version 1.2.
      ``v1.3``         SPIR-V version 1.3.
      ``v1.4``         SPIR-V version 1.4.
      ``v1.5``         SPIR-V version 1.5.
+     ``v1.6``         SPIR-V version 1.6.
      =============== ==============================================================
 
   .. table:: SPIR-V Vendors
@@ -62,6 +97,9 @@ to specify the target triple:
      OS                    Description
      ===================== ============================================================
      *<empty>*/``unknown``  Defaults to the OpenCL runtime.
+     ``vulkan``             Vulkan shader runtime.
+     ``vulkan1.2``          Vulkan 1.2 runtime, corresponding to SPIR-V 1.5.
+     ``vulkan1.3``          Vulkan 1.3 runtime, corresponding to SPIR-V 1.6.
      ===================== ============================================================
 
   .. table:: SPIR-V Environments
@@ -69,20 +107,95 @@ to specify the target triple:
      ===================== ==============================================================
      Environment           Description
      ===================== ==============================================================
-     *<empty>*/``unknown``  Defaults to the OpenCL environment.
+     *<empty>*/``unknown``  OpenCL environment or deduced by backend based on the input.
      ===================== ==============================================================
 
 Example:
 
 ``-target spirv64v1.0`` can be used to compile for SPIR-V version 1.0 with 64-bit pointer width.
 
-.. _spirv-types:
+.. _spirv-extensions:
+
+Extensions
+----------
+
+The SPIR-V backend supports a variety of `extensions <https://github.com/KhronosGroup/SPIRV-Registry/tree/main/extensions>`_ 
+that enable or enhance features beyond the core SPIR-V specification. 
+These extensions can be enabled using the ``-spirv-extensions`` option 
+followed by the name of the extension(s) you wish to enable. Below is a 
+list of supported SPIR-V extensions, sorted alphabetically by their extension names:
+
+.. list-table:: Supported SPIR-V Extensions
+   :widths: 50 150
+   :header-rows: 1
+
+   * - Extension Name
+     - Description
+   * - ``SPV_EXT_shader_atomic_float16_add``
+     - Extends the SPV_EXT_shader_atomic_float_add extension to support atomically adding to 16-bit floating-point numbers in memory.
+   * - ``SPV_EXT_shader_atomic_float_add``
+     - Adds atomic add instruction on floating-point numbers.
+   * - ``SPV_EXT_shader_atomic_float_min_max``
+     - Adds atomic min and max instruction on floating-point numbers.
+   * - ``SPV_INTEL_arbitrary_precision_integers``
+     - Allows generating arbitrary width integer types.
+   * - ``SPV_INTEL_bfloat16_conversion``
+     - Adds instructions to convert between single-precision 32-bit floating-point values and 16-bit bfloat16 values.
+   * - ``SPV_INTEL_function_pointers``
+     - Allows translation of function pointers.
+   * - ``SPV_INTEL_optnone``
+     - Adds OptNoneINTEL value for Function Control mask that indicates a request to not optimize the function.
+   * - ``SPV_INTEL_subgroups``
+     - Allows work items in a subgroup to share data without the use of local memory and work group barriers, and to utilize specialized hardware to load and store blocks of data from images or buffers.
+   * - ``SPV_INTEL_usm_storage_classes``
+     - Introduces two new storage classes that are subclasses of the CrossWorkgroup storage class that provides additional information that can enable optimization.
+   * - ``SPV_INTEL_variable_length_array``
+     - Allows to allocate local arrays whose number of elements is unknown at compile time.
+   * - ``SPV_KHR_bit_instructions``
+     - Enables bit instructions to be used by SPIR-V modules without requiring the Shader capability.
+   * - ``SPV_KHR_expect_assume``
+     - Provides additional information to a compiler, similar to the llvm.assume and llvm.expect intrinsics.
+   * - ``SPV_KHR_float_controls``
+     - Provides new execution modes to control floating-point computations by overriding an implementation’s default behavior for rounding modes, denormals, signed zero, and infinities.
+   * - ``SPV_KHR_linkonce_odr``
+     - Allows to use the LinkOnceODR linkage type that lets a function or global variable to be merged with other functions or global variables of the same name when linkage occurs.
+   * - ``SPV_KHR_no_integer_wrap_decoration``
+     - Adds decorations to indicate that a given instruction does not cause integer wrapping.
+   * - ``SPV_KHR_subgroup_rotate``
+     - Adds a new instruction that enables rotating values across invocations within a subgroup.
+   * - ``SPV_KHR_uniform_group_instructions``
+     - Allows support for additional group operations within uniform control flow.
 
-Representing special types in SPIR-V
-====================================
+To enable multiple extensions, list them separated by spaces. For example, to enable support for atomic operations on floating-point numbers and arbitrary precision integers, use:
+
+``-spirv-ext=+SPV_EXT_shader_atomic_float_add,+SPV_INTEL_arbitrary_precision_integers``
+
+To enable all extensions, use the following option:
+``-spirv-ext=all``
+
+To enable all extensions except specified, specify ``all`` followed by a list of disallowed extensions. For example:
+``-spirv-ext=all,-SPV_INTEL_arbitrary_precision_integers``
+
+SPIR-V representation in LLVM IR
+================================
+
+SPIR-V is intentionally designed for seamless integration with various Intermediate 
+Representations (IRs), including LLVM IR, facilitating straightforward mappings for 
+most of its entities. The development of the SPIR-V backend has been guided by a 
+principle of compatibility with the `Khronos Group SPIR-V LLVM Translator <https://github.com/KhronosGroup/SPIRV-LLVM-Translator>`_.
+Consequently, the input representation accepted by the SPIR-V backend aligns closely 
+with that detailed in `the SPIR-V Representation in LLVM document <https://github.com/KhronosGroup/SPIRV-LLVM-Translator/blob/main/docs/SPIRVRepresentationInLLVM.rst>`_. 
+This document, along with the sections that follow, delineate the main points and focus 
+on any differences between the LLVM IR that this backend processes and the conventions 
+used by other tools.
+
+.. _spirv-special-types:
+
+Special types
+-------------
 
 SPIR-V specifies several kinds of opaque types. These types are represented
-using target extension types. These types are represented as follows:
+using target extension types and are represented as follows:
 
   .. table:: SPIR-V Opaque Types
 
@@ -108,3 +221,210 @@ dimensionality parameter as ``1`` meaning 2D. Sampled image types include the
 parameters of its underlying image type, so that a sampled image for the
 previous type has the representation
 ``target("spirv.SampledImage, void, 1, 1, 0, 0, 0, 0, 0)``.
+
+.. _spirv-intrinsics:
+
+Target Intrinsics
+-----------------
+
+The SPIR-V backend employs several LLVM IR intrinsics that facilitate various low-level 
+operations essential for generating correct and efficient SPIR-V code. These intrinsics 
+cover a range of functionalities from type assignment and memory management to control 
+flow and atomic operations. Below is a detailed table of selected intrinsics used in the 
+SPIR-V backend, along with their descriptions and argument details.
+
+.. list-table:: LLVM IR Intrinsics for SPIR-V
+   :widths: 25 15 20 40
+   :header-rows: 1
+
+   * - Intrinsic ID
+     - Return Type
+     - Argument Types
+     - Description
+   * - `int_spv_assign_type`
+     - None
+     - `[Type, Metadata]`
+     - Associates a type with metadata, crucial for maintaining type information in SPIR-V structures. Not emitted directly but supports the type system internally.
+   * - `int_spv_assign_ptr_type`
+     - None
+     - `[Type, Metadata, Integer]`
+     - Similar to `int_spv_assign_type`, but for pointer types with an additional integer specifying the storage class. Supports SPIR-V's detailed pointer type system. Not emitted directly.
+   * - `int_spv_assign_name`
+     - None
+     - `[Type, Vararg]`
+     - Assigns names to types or values, enhancing readability and debuggability of SPIR-V code. Not emitted directly but used for metadata enrichment.
+   * - `int_spv_track_constant`
+     - Type
+     - `[Type, Metadata]`
+     - Tracks constants in the SPIR-V module. Essential for optimizing and reducing redundancy. Emitted for internal use only.
+   * - `int_spv_init_global`
+     - None
+     - `[Type, Type]`
+     - Initializes global variables, a necessary step for ensuring correct global state management in SPIR-V. Emitted for internal use only.
+   * - `int_spv_unref_global`
+     - None
+     - `[Type]`
+     - Manages the lifetime of global variables by marking them as unreferenced, thus enabling optimizations related to global variable usage. Emitted for internal use only.
+   * - `int_spv_gep`
+     - Pointer
+     - `[Boolean, Type, Vararg]`
+     - Computes the address of a sub-element of an aggregate type. Critical for accessing array elements and structure fields. Supports conditionally addressing elements in a generic way.
+   * - `int_spv_load`
+     - 32-bit Integer
+     - `[Pointer, 16-bit Integer, 8-bit Integer]`
+     - Loads a value from a memory location. The additional integers specify memory access and alignment details, vital for ensuring correct and efficient memory operations.
+   * - `int_spv_store`
+     - None
+     - `[Type, Pointer, 16-bit Integer, 8-bit Integer]`
+     - Stores a value to a memory location. Like `int_spv_load`, it includes specifications for memory access and alignment, essential for memory operations.
+   * - `int_spv_extractv`
+     - Type
+     - `[32-bit Integer, Vararg]`
+     - Extracts a value from a vector, allowing for vector operations within SPIR-V. Enables manipulation of vector components.
+   * - `int_spv_insertv`
+     - 32-bit Integer
+     - `[32-bit Integer, Type, Vararg]`
+     - Inserts a value into a vector. Complementary to `int_spv_extractv`, it facilitates the construction and manipulation of vectors.
+   * - `int_spv_extractelt`
+     - Type
+     - `[Type, Any Integer]`
+     - Extracts an element from an aggregate type based on an index. Essential for operations on arrays and vectors.
+   * - `int_spv_insertelt`
+     - Type
+     - `[Type, Type, Any Integer]`
+     - Inserts an element into an aggregate type at a specified index. Allows for building and modifying arrays and vectors.
+   * - `int_spv_const_composite`
+     - 32-bit Integer
+     - `[Vararg]`
+     - Constructs a composite type from given elements. Key for creating arrays, structs, and vectors from individual components.
+   * - `int_spv_bitcast`
+     - Type
+     - `[Type]`
+     - Performs a bit-wise cast between types. Critical for type conversions that do not change the bit representation.
+   * - `int_spv_ptrcast`
+     - Type
+     - `[Type, Metadata, Integer]`
+     - Casts pointers between different types. Similar to `int_spv_bitcast` but specifically for pointers, taking into account SPIR-V's strict type system.
+   * - `int_spv_switch`
+     - None
+     - `[Type, Vararg]`
+     - Implements a multi-way branch based on a value. Enables complex control flow structures, similar to the switch statement in high-level languages.
+   * - `int_spv_cmpxchg`
+     - 32-bit Integer
+     - `[Type, Vararg]`
+     - Performs an atomic compare-and-exchange operation. Crucial for synchronization and concurrency control in compute shaders.
+   * - `int_spv_unreachable`
+     - None
+     - `[]`
+     - Marks a point in the code that should never be reached, enabling optimizations by indicating unreachable code paths.
+   * - `int_spv_alloca`
+     - Type
+     - `[]`
+     - Allocates memory on the stack. Fundamental for local variable storage in functions.
+   * - `int_spv_alloca_array`
+     - Type
+     - `[Any Integer]`
+     - Allocates an array on the stack. Extends `int_spv_alloca` to support array allocations, essential for temporary arrays.
+   * - `int_spv_undef`
+     - 32-bit Integer
+     - `[]`
+     - Generates an undefined value. Useful for optimizations and indicating uninitialized variables.
+   * - `int_spv_assume`
+     - None
+     - `[1-bit Integer]`
+     - Provides hints to the optimizer about assumptions that can be made about program state. Improves optimization potential.
+   * - `int_spv_expect`
+     - Any Integer Type
+     - `[Type, Type]`
+     - Guides branch prediction by indicating expected branch paths. Enhances performance by optimizing common code paths.
+   * - `int_spv_thread_id`
+     - 32-bit Integer
+     - `[32-bit Integer]`
+     - Retrieves the thread ID within a workgroup. Essential for identifying execution context in parallel compute operations.
+   * - `int_spv_create_handle`
+     - Pointer
+     - `[8-bit Integer]`
+     - Creates a resource handle for graphics or compute resources. Facilitates the management and use of resources in shaders.
+
+.. _spirv-builtin-functions:
+
+Builtin Functions
+-----------------
+
+The following section highlights the representation of SPIR-V builtins in LLVM IR, 
+emphasizing builtins that do not have direct counterparts in LLVM.
+
+Instructions as Function Calls
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+SPIR-V builtins without direct LLVM counterparts are represented as LLVM function calls. 
+These functions, termed SPIR-V builtin functions, follow an IA64 mangling scheme with 
+SPIR-V-specific extensions. Parsing non-mangled calls to builtins is supported in some cases, 
+but not tested extensively. The general format is:
+
+.. code-block:: c
+
+  __spirv_{OpCodeName}{_OptionalPostfixes}
+
+Where `{OpCodeName}` is the SPIR-V opcode name sans the "Op" prefix, and 
+`{OptionalPostfixes}` are decoration-specific postfixes, if any. The mangling and 
+postfixes allow for the representation of SPIR-V's rich instruction set within LLVM's 
+framework.
+
+Extended Instruction Sets
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+SPIR-V defines several extended instruction sets for additional functionalities, such as 
+OpenCL-specific operations. In LLVM IR, these are represented by function calls to 
+mangled builtins and selected based on the environment. For example:
+
+.. code-block:: c
+
+  acos_f32
+
+represents the `acos` function from the OpenCL extended instruction set for a float32 
+input.
+
+Builtin Variables
+~~~~~~~~~~~~~~~~~
+
+SPIR-V builtin variables, which provide access to special hardware or execution model 
+properties, are mapped to either LLVM function calls or LLVM global variables. The 
+representation follows the naming convention:
+
+.. code-block:: c
+
+  __spirv_BuiltIn{VariableName}
+
+For instance, the SPIR-V builtin `GlobalInvocationId` is accessible in LLVM IR as 
+`__spirv_BuiltInGlobalInvocationId`.
+
+Vector Load and Store Builtins
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+SPIR-V's capabilities for loading and storing vectors are represented in LLVM IR using 
+functions that mimic the SPIR-V instructions. These builtins handle cases that LLVM's 
+native instructions do not directly support, enabling fine-grained control over memory 
+operations.
+
+Atomic Operations
+~~~~~~~~~~~~~~~~~
+
+SPIR-V's atomic operations, especially those operating on floating-point data, are 
+represented in LLVM IR with corresponding function calls. These builtins ensure 
+atomicity in operations where LLVM might not have direct support, essential for parallel 
+execution and synchronization.
+
+Image Operations
+~~~~~~~~~~~~~~~~
+
+SPIR-V provides extensive support for image and sampler operations, which LLVM 
+represents through function calls to builtins. These include image reads, writes, and 
+queries, allowing detailed manipulation of image data and parameters.
+
+Group and Subgroup Operations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For workgroup and subgroup operations, LLVM uses function calls to represent SPIR-V's 
+group-based instructions. These builtins facilitate group synchronization, data sharing, 
+and collective operations essential for efficient parallel computation.
-- 
GitLab


From 5d7e7abc82ec87fe0cca3bc2afe67b70be5d0151 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 25 Mar 2024 21:41:53 +0000
Subject: [PATCH 259/404] [X86] ICX - vector XMM splat use Port 1 or 5 when
 boradcasting the shift amount

Noticed while trying to compare splat vs per-element shift perf stats for #39424

Confirmed with uops.info
---
 llvm/lib/Target/X86/X86SchedIceLake.td        |  6 +--
 .../X86/IceLakeServer/resources-avx1.s        | 34 ++++++++---------
 .../X86/IceLakeServer/resources-avx512bwvl.s  | 38 +++++++++----------
 .../X86/IceLakeServer/resources-sse2.s        | 34 ++++++++---------
 4 files changed, 56 insertions(+), 56 deletions(-)

diff --git a/llvm/lib/Target/X86/X86SchedIceLake.td b/llvm/lib/Target/X86/X86SchedIceLake.td
index 3981279abc36..2d296771b1c0 100644
--- a/llvm/lib/Target/X86/X86SchedIceLake.td
+++ b/llvm/lib/Target/X86/X86SchedIceLake.td
@@ -413,9 +413,9 @@ defm : ICXWriteResPair<WritePHMINPOS, [ICXPort0], 4, [1], 1, 6>; // Vector PHMIN
 
 // Vector integer shifts.
 defm : ICXWriteResPair<WriteVecShift, [ICXPort0], 1, [1], 1, 5>;
-defm : X86WriteRes<WriteVecShiftX,    [ICXPort5,ICXPort01],  2, [1,1], 2>;
-defm : X86WriteRes<WriteVecShiftY,    [ICXPort5,ICXPort01],  4, [1,1], 2>;
-defm : X86WriteRes<WriteVecShiftZ,    [ICXPort5,ICXPort0],   4, [1,1], 2>;
+defm : X86WriteRes<WriteVecShiftX,    [ICXPort01,ICXPort15], 2, [1,1], 2>;
+defm : X86WriteRes<WriteVecShiftY,    [ICXPort01,ICXPort5],  4, [1,1], 2>;
+defm : X86WriteRes<WriteVecShiftZ,    [ICXPort0,ICXPort5],   4, [1,1], 2>;
 defm : X86WriteRes<WriteVecShiftXLd,  [ICXPort01,ICXPort23], 7, [1,1], 2>;
 defm : X86WriteRes<WriteVecShiftYLd,  [ICXPort01,ICXPort23], 8, [1,1], 2>;
 defm : X86WriteRes<WriteVecShiftZLd,  [ICXPort0,ICXPort23],  8, [1,1], 2>;
diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s
index f184d5579d06..bd7a4894b45d 100644
--- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s
+++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s
@@ -1563,30 +1563,30 @@ vzeroupper
 # CHECK-NEXT:  1      1     0.50                        vpsignw	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   vpsignw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpslld	$1, %xmm0, %xmm2
-# CHECK-NEXT:  2      2     1.00                        vpslld	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      2     0.67                        vpslld	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   vpslld	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpslldq	$1, %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpsllq	$1, %xmm0, %xmm2
-# CHECK-NEXT:  2      2     1.00                        vpsllq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      2     0.67                        vpsllq	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   vpsllq	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpsllw	$1, %xmm0, %xmm2
-# CHECK-NEXT:  2      2     1.00                        vpsllw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      2     0.67                        vpsllw	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   vpsllw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpsrad	$1, %xmm0, %xmm2
-# CHECK-NEXT:  2      2     1.00                        vpsrad	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      2     0.67                        vpsrad	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   vpsrad	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpsraw	$1, %xmm0, %xmm2
-# CHECK-NEXT:  2      2     1.00                        vpsraw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      2     0.67                        vpsraw	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   vpsraw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpsrld	$1, %xmm0, %xmm2
-# CHECK-NEXT:  2      2     1.00                        vpsrld	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      2     0.67                        vpsrld	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   vpsrld	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpsrldq	$1, %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpsrlq	$1, %xmm0, %xmm2
-# CHECK-NEXT:  2      2     1.00                        vpsrlq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      2     0.67                        vpsrlq	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   vpsrlq	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpsrlw	$1, %xmm0, %xmm2
-# CHECK-NEXT:  2      2     1.00                        vpsrlw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      2     0.67                        vpsrlw	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   vpsrlw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.33                        vpsubb	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   vpsubb	(%rax), %xmm1, %xmm2
@@ -1738,7 +1738,7 @@ vzeroupper
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -     126.00 322.92 233.92 160.50 160.50 19.00  295.92 6.25   19.00  19.00  19.00
+# CHECK-NEXT:  -     126.00 322.92 237.92 160.50 160.50 19.00  291.92 6.25   19.00  19.00  19.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -2275,30 +2275,30 @@ vzeroupper
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     vpsignw	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsignw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     vpslld	$1, %xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpslld	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     vpslld	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpslld	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -      -      -     vpslldq	$1, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     vpsllq	$1, %xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpsllq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     vpsllq	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsllq	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     vpsllw	$1, %xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpsllw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     vpsllw	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsllw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     vpsrad	$1, %xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpsrad	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     vpsrad	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsrad	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     vpsraw	$1, %xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpsraw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     vpsraw	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsraw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     vpsrld	$1, %xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpsrld	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     vpsrld	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsrld	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -      -      -     vpsrldq	$1, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     vpsrlq	$1, %xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpsrlq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     vpsrlq	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsrlq	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     vpsrlw	$1, %xmm0, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpsrlw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     vpsrlw	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsrlw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -      -      -     vpsubb	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -      -      -     vpsubb	(%rax), %xmm1, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512bwvl.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512bwvl.s
index a0715cef8b00..ee095f0d3bfc 100644
--- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512bwvl.s
+++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx512bwvl.s
@@ -1768,11 +1768,11 @@ vpunpcklwd         (%rax), %ymm17, %ymm19 {z}{k1}
 # CHECK-NEXT:  2      8     0.50    *                   vpsllw	$0, (%rax), %ymm19 {%k1}
 # CHECK-NEXT:  1      1     0.50                        vpsllw	$0, %ymm16, %ymm19 {%k1} {z}
 # CHECK-NEXT:  2      8     0.50    *                   vpsllw	$0, (%rax), %ymm19 {%k1} {z}
-# CHECK-NEXT:  2      2     1.00                        vpsllw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      2     0.67                        vpsllw	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  2      7     0.50    *                   vpsllw	(%rax), %xmm17, %xmm19
-# CHECK-NEXT:  2      2     1.00                        vpsllw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      2     0.67                        vpsllw	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  2      7     0.50    *                   vpsllw	(%rax), %xmm17, %xmm19 {%k1}
-# CHECK-NEXT:  2      2     1.00                        vpsllw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      2     0.67                        vpsllw	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      7     0.50    *                   vpsllw	(%rax), %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      4     1.00                        vpsllw	%xmm16, %ymm17, %ymm19
 # CHECK-NEXT:  2      8     0.50    *                   vpsllw	(%rax), %ymm17, %ymm19
@@ -1804,11 +1804,11 @@ vpunpcklwd         (%rax), %ymm17, %ymm19 {z}{k1}
 # CHECK-NEXT:  2      8     0.50    *                   vpsraw	$0, (%rax), %ymm19 {%k1}
 # CHECK-NEXT:  1      1     0.50                        vpsraw	$0, %ymm16, %ymm19 {%k1} {z}
 # CHECK-NEXT:  2      8     0.50    *                   vpsraw	$0, (%rax), %ymm19 {%k1} {z}
-# CHECK-NEXT:  2      2     1.00                        vpsraw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      2     0.67                        vpsraw	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  2      7     0.50    *                   vpsraw	(%rax), %xmm17, %xmm19
-# CHECK-NEXT:  2      2     1.00                        vpsraw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      2     0.67                        vpsraw	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  2      7     0.50    *                   vpsraw	(%rax), %xmm17, %xmm19 {%k1}
-# CHECK-NEXT:  2      2     1.00                        vpsraw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      2     0.67                        vpsraw	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      7     0.50    *                   vpsraw	(%rax), %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      4     1.00                        vpsraw	%xmm16, %ymm17, %ymm19
 # CHECK-NEXT:  2      8     0.50    *                   vpsraw	(%rax), %ymm17, %ymm19
@@ -1844,11 +1844,11 @@ vpunpcklwd         (%rax), %ymm17, %ymm19 {z}{k1}
 # CHECK-NEXT:  2      8     0.50    *                   vpsrlw	$0, (%rax), %ymm19 {%k1}
 # CHECK-NEXT:  1      1     0.50                        vpsrlw	$0, %ymm16, %ymm19 {%k1} {z}
 # CHECK-NEXT:  2      8     0.50    *                   vpsrlw	$0, (%rax), %ymm19 {%k1} {z}
-# CHECK-NEXT:  2      2     1.00                        vpsrlw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      2     0.67                        vpsrlw	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  2      7     0.50    *                   vpsrlw	(%rax), %xmm17, %xmm19
-# CHECK-NEXT:  2      2     1.00                        vpsrlw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      2     0.67                        vpsrlw	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  2      7     0.50    *                   vpsrlw	(%rax), %xmm17, %xmm19 {%k1}
-# CHECK-NEXT:  2      2     1.00                        vpsrlw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      2     0.67                        vpsrlw	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      7     0.50    *                   vpsrlw	(%rax), %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      4     1.00                        vpsrlw	%xmm16, %ymm17, %ymm19
 # CHECK-NEXT:  2      8     0.50    *                   vpsrlw	(%rax), %ymm17, %ymm19
@@ -2025,7 +2025,7 @@ vpunpcklwd         (%rax), %ymm17, %ymm19 {z}{k1}
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -      -     255.33 303.33 216.00 216.00 10.00  451.33  -     10.00  10.00  10.00
+# CHECK-NEXT:  -      -     255.33 307.83 216.00 216.00 10.00  446.83  -     10.00  10.00  10.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -2705,11 +2705,11 @@ vpunpcklwd         (%rax), %ymm17, %ymm19 {z}{k1}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsllw	$0, (%rax), %ymm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     vpsllw	$0, %ymm16, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsllw	$0, (%rax), %ymm19 {%k1} {z}
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpsllw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     vpsllw	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsllw	(%rax), %xmm17, %xmm19
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpsllw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     vpsllw	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsllw	(%rax), %xmm17, %xmm19 {%k1}
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpsllw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     vpsllw	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsllw	(%rax), %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpsllw	%xmm16, %ymm17, %ymm19
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsllw	(%rax), %ymm17, %ymm19
@@ -2741,11 +2741,11 @@ vpunpcklwd         (%rax), %ymm17, %ymm19 {z}{k1}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsraw	$0, (%rax), %ymm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     vpsraw	$0, %ymm16, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsraw	$0, (%rax), %ymm19 {%k1} {z}
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpsraw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     vpsraw	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsraw	(%rax), %xmm17, %xmm19
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpsraw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     vpsraw	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsraw	(%rax), %xmm17, %xmm19 {%k1}
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpsraw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     vpsraw	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsraw	(%rax), %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpsraw	%xmm16, %ymm17, %ymm19
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsraw	(%rax), %ymm17, %ymm19
@@ -2781,11 +2781,11 @@ vpunpcklwd         (%rax), %ymm17, %ymm19 {z}{k1}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsrlw	$0, (%rax), %ymm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     vpsrlw	$0, %ymm16, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsrlw	$0, (%rax), %ymm19 {%k1} {z}
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpsrlw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     vpsrlw	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsrlw	(%rax), %xmm17, %xmm19
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpsrlw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     vpsrlw	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsrlw	(%rax), %xmm17, %xmm19 {%k1}
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpsrlw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     vpsrlw	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsrlw	(%rax), %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     vpsrlw	%xmm16, %ymm17, %ymm19
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpsrlw	(%rax), %ymm17, %ymm19
diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse2.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse2.s
index a4a51280b8ca..a1bf6c1e9862 100644
--- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse2.s
+++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse2.s
@@ -595,30 +595,30 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pshuflw	$1, %xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   pshuflw	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pslld	$1, %xmm2
-# CHECK-NEXT:  2      2     1.00                        pslld	%xmm0, %xmm2
+# CHECK-NEXT:  2      2     0.67                        pslld	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   pslld	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pslldq	$1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        psllq	$1, %xmm2
-# CHECK-NEXT:  2      2     1.00                        psllq	%xmm0, %xmm2
+# CHECK-NEXT:  2      2     0.67                        psllq	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   psllq	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        psllw	$1, %xmm2
-# CHECK-NEXT:  2      2     1.00                        psllw	%xmm0, %xmm2
+# CHECK-NEXT:  2      2     0.67                        psllw	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   psllw	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        psrad	$1, %xmm2
-# CHECK-NEXT:  2      2     1.00                        psrad	%xmm0, %xmm2
+# CHECK-NEXT:  2      2     0.67                        psrad	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   psrad	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        psraw	$1, %xmm2
-# CHECK-NEXT:  2      2     1.00                        psraw	%xmm0, %xmm2
+# CHECK-NEXT:  2      2     0.67                        psraw	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   psraw	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        psrld	$1, %xmm2
-# CHECK-NEXT:  2      2     1.00                        psrld	%xmm0, %xmm2
+# CHECK-NEXT:  2      2     0.67                        psrld	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   psrld	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        psrldq	$1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        psrlq	$1, %xmm2
-# CHECK-NEXT:  2      2     1.00                        psrlq	%xmm0, %xmm2
+# CHECK-NEXT:  2      2     0.67                        psrlq	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   psrlq	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        psrlw	$1, %xmm2
-# CHECK-NEXT:  2      2     1.00                        psrlw	%xmm0, %xmm2
+# CHECK-NEXT:  2      2     0.67                        psrlw	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   psrlw	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.33                        psubb	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   psubb	(%rax), %xmm2
@@ -691,7 +691,7 @@ xorpd       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -     40.00  106.92 98.92  58.50  58.50  7.50   76.42  1.75   8.00   8.00   7.50
+# CHECK-NEXT:  -     40.00  106.92 102.92 58.50  58.50  7.50   72.42  1.75   8.00   8.00   7.50
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -883,30 +883,30 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -      -      -     pshuflw	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     pshuflw	$1, (%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     pslld	$1, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     pslld	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     pslld	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     pslld	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -      -      -     pslldq	$1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     psllq	$1, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     psllq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     psllq	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     psllq	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     psllw	$1, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     psllw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     psllw	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     psllw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     psrad	$1, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     psrad	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     psrad	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     psrad	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     psraw	$1, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     psraw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     psraw	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     psraw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     psrld	$1, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     psrld	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     psrld	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     psrld	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -      -      -     psrldq	$1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     psrlq	$1, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     psrlq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     psrlq	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     psrlq	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     psrlw	$1, %xmm2
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     1.00    -      -      -      -     psrlw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     0.50   1.00    -      -      -     0.50    -      -      -      -     psrlw	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     psrlw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -      -      -     psubb	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -      -      -     psubb	(%rax), %xmm2
-- 
GitLab


From 5fc619b5ee227ab4d22319dad8d2ba6fc1660091 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 26 Mar 2024 10:41:07 +0000
Subject: [PATCH 260/404] [DAG] Update ISD::AVG folds to use hasOperation to
 allow Custom matching prior to legalization

Fixes issue where AVX1 targets weren't matching 256-bit AVGCEILU cases.
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 59 ++++++------
 llvm/test/CodeGen/X86/avgceilu.ll             | 92 ++++++-------------
 2 files changed, 57 insertions(+), 94 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 6f1b1080cc88..6f7a2a8a4c14 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -609,6 +609,9 @@ namespace {
                            SDValue &CC, bool MatchStrict = false) const;
     bool isOneUseSetCC(SDValue N) const;
 
+    SDValue foldAddToAvg(SDNode *N, const SDLoc &DL);
+    SDValue foldSubToAvg(SDNode *N, const SDLoc &DL);
+
     SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
                                          unsigned HiOp);
     SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
@@ -2530,26 +2533,22 @@ static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG) {
 }
 
 // Attempt to form avgceil(A, B) from (A | B) - ((A ^ B) >> 1)
-static SDValue combineFixedwidthToAVGCEIL(SDNode *N, SelectionDAG &DAG) {
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+SDValue DAGCombiner::foldSubToAvg(SDNode *N, const SDLoc &DL) {
   SDValue N0 = N->getOperand(0);
   EVT VT = N0.getValueType();
-  SDLoc DL(N);
   SDValue A, B;
 
-  if (TLI.isOperationLegal(ISD::AVGCEILU, VT)) {
-    if (sd_match(N, m_Sub(m_Or(m_Value(A), m_Value(B)),
-                          m_Srl(m_Xor(m_Deferred(A), m_Deferred(B)),
-                                m_SpecificInt(1))))) {
-      return DAG.getNode(ISD::AVGCEILU, DL, VT, A, B);
-    }
+  if (hasOperation(ISD::AVGCEILU, VT) &&
+      sd_match(N, m_Sub(m_Or(m_Value(A), m_Value(B)),
+                        m_Srl(m_Xor(m_Deferred(A), m_Deferred(B)),
+                              m_SpecificInt(1))))) {
+    return DAG.getNode(ISD::AVGCEILU, DL, VT, A, B);
   }
-  if (TLI.isOperationLegal(ISD::AVGCEILS, VT)) {
-    if (sd_match(N, m_Sub(m_Or(m_Value(A), m_Value(B)),
-                          m_Sra(m_Xor(m_Deferred(A), m_Deferred(B)),
-                                m_SpecificInt(1))))) {
-      return DAG.getNode(ISD::AVGCEILS, DL, VT, A, B);
-    }
+  if (hasOperation(ISD::AVGCEILS, VT) &&
+      sd_match(N, m_Sub(m_Or(m_Value(A), m_Value(B)),
+                        m_Sra(m_Xor(m_Deferred(A), m_Deferred(B)),
+                              m_SpecificInt(1))))) {
+    return DAG.getNode(ISD::AVGCEILS, DL, VT, A, B);
   }
   return SDValue();
 }
@@ -2846,26 +2845,22 @@ SDValue DAGCombiner::visitADDLike(SDNode *N) {
 }
 
 // Attempt to form avgfloor(A, B) from (A & B) + ((A ^ B) >> 1)
-static SDValue combineFixedwidthToAVGFLOOR(SDNode *N, SelectionDAG &DAG) {
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+SDValue DAGCombiner::foldAddToAvg(SDNode *N, const SDLoc &DL) {
   SDValue N0 = N->getOperand(0);
   EVT VT = N0.getValueType();
-  SDLoc DL(N);
   SDValue A, B;
 
-  if (TLI.isOperationLegal(ISD::AVGFLOORU, VT)) {
-    if (sd_match(N, m_Add(m_And(m_Value(A), m_Value(B)),
-                          m_Srl(m_Xor(m_Deferred(A), m_Deferred(B)),
-                                m_SpecificInt(1))))) {
-      return DAG.getNode(ISD::AVGFLOORU, DL, VT, A, B);
-    }
+  if (hasOperation(ISD::AVGFLOORU, VT) &&
+      sd_match(N, m_Add(m_And(m_Value(A), m_Value(B)),
+                        m_Srl(m_Xor(m_Deferred(A), m_Deferred(B)),
+                              m_SpecificInt(1))))) {
+    return DAG.getNode(ISD::AVGFLOORU, DL, VT, A, B);
   }
-  if (TLI.isOperationLegal(ISD::AVGFLOORS, VT)) {
-    if (sd_match(N, m_Add(m_And(m_Value(A), m_Value(B)),
-                          m_Sra(m_Xor(m_Deferred(A), m_Deferred(B)),
-                                m_SpecificInt(1))))) {
-      return DAG.getNode(ISD::AVGFLOORS, DL, VT, A, B);
-    }
+  if (hasOperation(ISD::AVGFLOORS, VT) &&
+      sd_match(N, m_Add(m_And(m_Value(A), m_Value(B)),
+                        m_Sra(m_Xor(m_Deferred(A), m_Deferred(B)),
+                              m_SpecificInt(1))))) {
+    return DAG.getNode(ISD::AVGFLOORS, DL, VT, A, B);
   }
 
   return SDValue();
@@ -2887,7 +2882,7 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
     return V;
 
   // Try to match AVGFLOOR fixedwidth pattern
-  if (SDValue V = combineFixedwidthToAVGFLOOR(N, DAG))
+  if (SDValue V = foldAddToAvg(N, DL))
     return V;
 
   // fold (a+b) -> (a|b) iff a and b share no bits.
@@ -3886,7 +3881,7 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
     return V;
 
   // Try to match AVGCEIL fixedwidth pattern
-  if (SDValue V = combineFixedwidthToAVGCEIL(N, DAG))
+  if (SDValue V = foldSubToAvg(N, DL))
     return V;
 
   if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, SDLoc(N)))
diff --git a/llvm/test/CodeGen/X86/avgceilu.ll b/llvm/test/CodeGen/X86/avgceilu.ll
index 3a74fca23773..dee1a5a720f9 100644
--- a/llvm/test/CodeGen/X86/avgceilu.ll
+++ b/llvm/test/CodeGen/X86/avgceilu.ll
@@ -319,18 +319,11 @@ define <32 x i8> @test_fixed_v32i8(<32 x i8> %a0, <32 x i8> %a1) {
 ;
 ; AVX1-LABEL: test_fixed_v32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm2
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm1
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
-; AVX1-NEXT:    vpsubb %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpavgb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpavgb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: test_fixed_v32i8:
@@ -392,15 +385,11 @@ define <16 x i16> @test_fixed_v16i16(<16 x i16> %a0, <16 x i16> %a1) {
 ;
 ; AVX1-LABEL: test_fixed_v16i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm2
-; AVX1-NEXT:    vxorps %ymm0, %ymm1, %ymm0
-; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm0
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
-; AVX1-NEXT:    vpsubw %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpavgw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpavgw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: test_fixed_v16i16:
@@ -959,29 +948,16 @@ define <64 x i8> @test_fixed_v64i8(<64 x i8> %a0, <64 x i8> %a1) {
 ;
 ; AVX1-LABEL: test_fixed_v64i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vorps %ymm3, %ymm1, %ymm4
-; AVX1-NEXT:    vorps %ymm2, %ymm0, %ymm5
-; AVX1-NEXT:    vxorps %ymm2, %ymm0, %ymm0
-; AVX1-NEXT:    vxorps %ymm3, %ymm1, %ymm1
-; AVX1-NEXT:    vpsrlw $1, %xmm1, %xmm2
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm3 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vpsrlw $1, %xmm1, %xmm1
-; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm6
-; AVX1-NEXT:    vpand %xmm3, %xmm6, %xmm6
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm3
-; AVX1-NEXT:    vpsubb %xmm0, %xmm3, %xmm0
-; AVX1-NEXT:    vpsubb %xmm6, %xmm5, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm3
-; AVX1-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpsubb %xmm2, %xmm4, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT:    vpavgb %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpavgb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpavgb %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpavgb %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: test_fixed_v64i8:
@@ -1054,24 +1030,16 @@ define <32 x i16> @test_fixed_v32i16(<32 x i16> %a0, <32 x i16> %a1) {
 ;
 ; AVX1-LABEL: test_fixed_v32i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vorps %ymm3, %ymm1, %ymm4
-; AVX1-NEXT:    vorps %ymm2, %ymm0, %ymm5
-; AVX1-NEXT:    vxorps %ymm0, %ymm2, %ymm0
-; AVX1-NEXT:    vxorps %ymm1, %ymm3, %ymm1
-; AVX1-NEXT:    vpsrlw $1, %xmm1, %xmm2
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT:    vpsrlw $1, %xmm1, %xmm1
-; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm3
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm0
-; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm6
-; AVX1-NEXT:    vpsubw %xmm0, %xmm6, %xmm0
-; AVX1-NEXT:    vpsubw %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm3
-; AVX1-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpsubw %xmm2, %xmm4, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT:    vpavgw %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpavgw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpavgw %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpavgw %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: test_fixed_v32i16:
-- 
GitLab


From 5b544b511c7133fcb26a5c563b746a4baefb38d6 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 26 Mar 2024 10:43:29 +0000
Subject: [PATCH 261/404] [Mips] ctpop.mir - regenerate checks to improve
 codegen diff in #86505

---
 .../Mips/GlobalISel/legalizer/ctpop.mir       | 126 +++++++++---------
 1 file changed, 64 insertions(+), 62 deletions(-)

diff --git a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/ctpop.mir b/llvm/test/CodeGen/Mips/GlobalISel/legalizer/ctpop.mir
index 136c03946f36..4c0b3c617721 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/ctpop.mir
+++ b/llvm/test/CodeGen/Mips/GlobalISel/legalizer/ctpop.mir
@@ -10,29 +10,30 @@ body:             |
 
     ; MIPS32-LABEL: name: ctpop_i32
     ; MIPS32: liveins: $a0
-    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
-    ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; MIPS32: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
-    ; MIPS32: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
-    ; MIPS32: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C1]]
-    ; MIPS32: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[AND]]
-    ; MIPS32: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; MIPS32: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C2]](s32)
-    ; MIPS32: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
-    ; MIPS32: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C3]]
-    ; MIPS32: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C3]]
-    ; MIPS32: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND1]], [[AND2]]
-    ; MIPS32: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; MIPS32: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C4]](s32)
-    ; MIPS32: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR2]], [[ADD]]
-    ; MIPS32: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
-    ; MIPS32: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C5]]
-    ; MIPS32: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
-    ; MIPS32: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[C6]]
-    ; MIPS32: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; MIPS32: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C7]](s32)
-    ; MIPS32: $v0 = COPY [[LSHR3]](s32)
-    ; MIPS32: RetRA implicit $v0
+    ; MIPS32-NEXT: {{  $}}
+    ; MIPS32-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; MIPS32-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; MIPS32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
+    ; MIPS32-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C1]]
+    ; MIPS32-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[AND]]
+    ; MIPS32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; MIPS32-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C2]](s32)
+    ; MIPS32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
+    ; MIPS32-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C3]]
+    ; MIPS32-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C3]]
+    ; MIPS32-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND1]], [[AND2]]
+    ; MIPS32-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; MIPS32-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C4]](s32)
+    ; MIPS32-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR2]], [[ADD]]
+    ; MIPS32-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
+    ; MIPS32-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C5]]
+    ; MIPS32-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
+    ; MIPS32-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[C6]]
+    ; MIPS32-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; MIPS32-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C7]](s32)
+    ; MIPS32-NEXT: $v0 = COPY [[LSHR3]](s32)
+    ; MIPS32-NEXT: RetRA implicit $v0
     %0:_(s32) = COPY $a0
     %1:_(s32) = G_CTPOP %0(s32)
     $v0 = COPY %1(s32)
@@ -49,45 +50,46 @@ body:             |
 
     ; MIPS32-LABEL: name: ctpop_i64
     ; MIPS32: liveins: $a0, $a1
-    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
-    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
-    ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
-    ; MIPS32: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
-    ; MIPS32: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
-    ; MIPS32: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C1]]
-    ; MIPS32: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[AND]]
-    ; MIPS32: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
-    ; MIPS32: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C2]](s32)
-    ; MIPS32: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
-    ; MIPS32: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C3]]
-    ; MIPS32: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C3]]
-    ; MIPS32: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND1]], [[AND2]]
-    ; MIPS32: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; MIPS32: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C4]](s32)
-    ; MIPS32: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR2]], [[ADD]]
-    ; MIPS32: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
-    ; MIPS32: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C5]]
-    ; MIPS32: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
-    ; MIPS32: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[C6]]
-    ; MIPS32: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; MIPS32: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C7]](s32)
-    ; MIPS32: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
-    ; MIPS32: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C1]]
-    ; MIPS32: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[COPY1]], [[AND4]]
-    ; MIPS32: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SUB1]], [[C2]](s32)
-    ; MIPS32: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C3]]
-    ; MIPS32: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SUB1]], [[C3]]
-    ; MIPS32: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[AND5]], [[AND6]]
-    ; MIPS32: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD2]], [[C4]](s32)
-    ; MIPS32: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[LSHR6]], [[ADD2]]
-    ; MIPS32: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ADD3]], [[C5]]
-    ; MIPS32: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND7]], [[C6]]
-    ; MIPS32: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[MUL1]], [[C7]](s32)
-    ; MIPS32: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[LSHR7]], [[LSHR3]]
-    ; MIPS32: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; MIPS32: $v0 = COPY [[ADD4]](s32)
-    ; MIPS32: $v1 = COPY [[C8]](s32)
-    ; MIPS32: RetRA implicit $v0, implicit $v1
+    ; MIPS32-NEXT: {{  $}}
+    ; MIPS32-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; MIPS32-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
+    ; MIPS32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1431655765
+    ; MIPS32-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C1]]
+    ; MIPS32-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[AND]]
+    ; MIPS32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; MIPS32-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[SUB]], [[C2]](s32)
+    ; MIPS32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 858993459
+    ; MIPS32-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C3]]
+    ; MIPS32-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C3]]
+    ; MIPS32-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND1]], [[AND2]]
+    ; MIPS32-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; MIPS32-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ADD]], [[C4]](s32)
+    ; MIPS32-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR2]], [[ADD]]
+    ; MIPS32-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 252645135
+    ; MIPS32-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C5]]
+    ; MIPS32-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16843009
+    ; MIPS32-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[C6]]
+    ; MIPS32-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; MIPS32-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[MUL]], [[C7]](s32)
+    ; MIPS32-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
+    ; MIPS32-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C1]]
+    ; MIPS32-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[COPY1]], [[AND4]]
+    ; MIPS32-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SUB1]], [[C2]](s32)
+    ; MIPS32-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C3]]
+    ; MIPS32-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SUB1]], [[C3]]
+    ; MIPS32-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[AND5]], [[AND6]]
+    ; MIPS32-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[ADD2]], [[C4]](s32)
+    ; MIPS32-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[LSHR6]], [[ADD2]]
+    ; MIPS32-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ADD3]], [[C5]]
+    ; MIPS32-NEXT: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND7]], [[C6]]
+    ; MIPS32-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[MUL1]], [[C7]](s32)
+    ; MIPS32-NEXT: [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[LSHR7]], [[LSHR3]]
+    ; MIPS32-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; MIPS32-NEXT: $v0 = COPY [[ADD4]](s32)
+    ; MIPS32-NEXT: $v1 = COPY [[C8]](s32)
+    ; MIPS32-NEXT: RetRA implicit $v0, implicit $v1
     %1:_(s32) = COPY $a0
     %2:_(s32) = COPY $a1
     %0:_(s64) = G_MERGE_VALUES %1(s32), %2(s32)
-- 
GitLab


From f01377d8ebcb1f52fe75a236cdf34f1b8b1e99fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Naz=C4=B1m=20Can=20Alt=C4=B1nova?= <canaltinova@gmail.com>
Date: Tue, 26 Mar 2024 12:32:52 +0100
Subject: [PATCH 262/404] [tsan] Mark `pthread_*_lock` functions as blocking
 (#84162)

Fixes #83561.

When a thread is blocked on a mutex and we send an async signal to that
mutex, it never arrives because tsan thinks that `pthread_mutex_lock` is
not a blocking function. This patch marks `pthread_*_lock` functions as
blocking so we can successfully deliver async signals like `SIGPROF`
when the thread is blocked on them.

See the issue also for more details. I also added a test, which is a
simplified version of the compiler explorer example I posted in the
issue.

Please let me know if you have any other ideas or things to improve!
Happy to work on them.

Also I filed #83844 which is more tricky because we don't have a libc
wrapper for `SYS_futex`. I'm not sure how to intercept this yet. Please
let me know if you have ideas on that as well. Thanks!
---
 .../lib/tsan/rtl/tsan_interceptors_posix.cpp  | 10 +--
 .../test/tsan/signal_in_mutex_lock.cpp        | 71 +++++++++++++++++++
 2 files changed, 76 insertions(+), 5 deletions(-)
 create mode 100644 compiler-rt/test/tsan/signal_in_mutex_lock.cpp

diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
index 8ffc703b05ea..26c2cfef4e77 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
@@ -1340,7 +1340,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
 TSAN_INTERCEPTOR(int, pthread_mutex_lock, void *m) {
   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_lock, m);
   MutexPreLock(thr, pc, (uptr)m);
-  int res = REAL(pthread_mutex_lock)(m);
+  int res = BLOCK_REAL(pthread_mutex_lock)(m);
   if (res == errno_EOWNERDEAD)
     MutexRepair(thr, pc, (uptr)m);
   if (res == 0 || res == errno_EOWNERDEAD)
@@ -1385,7 +1385,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_clocklock, void *m,
                  __sanitizer_clockid_t clock, void *abstime) {
   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_clocklock, m, clock, abstime);
   MutexPreLock(thr, pc, (uptr)m);
-  int res = REAL(pthread_mutex_clocklock)(m, clock, abstime);
+  int res = BLOCK_REAL(pthread_mutex_clocklock)(m, clock, abstime);
   if (res == errno_EOWNERDEAD)
     MutexRepair(thr, pc, (uptr)m);
   if (res == 0 || res == errno_EOWNERDEAD)
@@ -1403,7 +1403,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_clocklock, void *m,
 TSAN_INTERCEPTOR(int, __pthread_mutex_lock, void *m) {
   SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_lock, m);
   MutexPreLock(thr, pc, (uptr)m);
-  int res = REAL(__pthread_mutex_lock)(m);
+  int res = BLOCK_REAL(__pthread_mutex_lock)(m);
   if (res == errno_EOWNERDEAD)
     MutexRepair(thr, pc, (uptr)m);
   if (res == 0 || res == errno_EOWNERDEAD)
@@ -1446,7 +1446,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) {
 TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) {
   SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m);
   MutexPreLock(thr, pc, (uptr)m);
-  int res = REAL(pthread_spin_lock)(m);
+  int res = BLOCK_REAL(pthread_spin_lock)(m);
   if (res == 0) {
     MutexPostLock(thr, pc, (uptr)m);
   }
@@ -1521,7 +1521,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) {
 TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) {
   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m);
   MutexPreLock(thr, pc, (uptr)m);
-  int res = REAL(pthread_rwlock_wrlock)(m);
+  int res = BLOCK_REAL(pthread_rwlock_wrlock)(m);
   if (res == 0) {
     MutexPostLock(thr, pc, (uptr)m);
   }
diff --git a/compiler-rt/test/tsan/signal_in_mutex_lock.cpp b/compiler-rt/test/tsan/signal_in_mutex_lock.cpp
new file mode 100644
index 000000000000..ec99e2319840
--- /dev/null
+++ b/compiler-rt/test/tsan/signal_in_mutex_lock.cpp
@@ -0,0 +1,71 @@
+// RUN: %clang_tsan %s -lstdc++ -o %t && %run %t 2>&1 | FileCheck %s
+
+#include "test.h"
+#include <pthread.h>
+#include <signal.h>
+#include <stdio.h>
+
+#include <cassert>
+#include <condition_variable>
+#include <mutex>
+
+std::mutex sampler_mutex; //dummy mutex to lock in the thread we spawn.
+std::mutex done_mutex;    // guards the cv and done variables.
+std::condition_variable cv;
+bool done = false;
+
+void *ThreadFunc(void *x) {
+  while (true) {
+    // Lock the mutex
+    std::lock_guard<std::mutex> guard(sampler_mutex);
+    // Mutex is released at the end
+  }
+
+  return nullptr;
+}
+
+static void SigprofHandler(int signal, siginfo_t *info, void *context) {
+  // Assuming we did some work, change the variable to let the main thread
+  // know that we are done.
+  {
+    std::unique_lock<std::mutex> lck(done_mutex);
+    done = true;
+    cv.notify_one();
+  }
+}
+
+int main() {
+  alarm(60); // Kill the test if it hangs.
+
+  // Install the signal handler
+  struct sigaction sa;
+  sa.sa_sigaction = SigprofHandler;
+  sigemptyset(&sa.sa_mask);
+  sa.sa_flags = SA_RESTART | SA_SIGINFO;
+  if (sigaction(SIGPROF, &sa, 0) != 0) {
+    fprintf(stderr, "failed to install signal handler\n");
+    abort();
+  }
+
+  // Spawn a thread that will just loop and get the mutex lock:
+  pthread_t thread;
+  pthread_create(&thread, NULL, ThreadFunc, NULL);
+
+  // Lock the mutex before sending the signal
+  std::lock_guard<std::mutex> guard(sampler_mutex);
+  // From now on thread 1 will be waiting for the lock
+
+  // Send the SIGPROF signal to thread.
+  int r = pthread_kill(thread, SIGPROF);
+  assert(r == 0);
+
+  // Wait until signal handler sends the data.
+  std::unique_lock lk(done_mutex);
+  cv.wait(lk, [] { return done; });
+
+  // We got the done variable from the signal handler. Exiting successfully.
+  fprintf(stderr, "PASS\n");
+}
+
+// CHECK-NOT: WARNING: ThreadSanitizer:
+// CHECK: PASS
-- 
GitLab


From fe97a6148ebce0091c03abd12abd814cb55afe24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Naz=C4=B1m=20Can=20Alt=C4=B1nova?= <canaltinova@gmail.com>
Date: Tue, 26 Mar 2024 12:33:51 +0100
Subject: [PATCH 263/404] [tsan] Add callbacks for futex syscalls and mark them
 as blocking on tsan (#86537)

Fixes #83844.

This PR adds callbacks to mark futex syscalls as blocking. Unfortunately
we didn't have a mechanism before to mark syscalls as a blocking call,
so I had to implement it, but it mostly reuses the `BlockingCall`
implementation
[here](https://github.com/llvm/llvm-project/blob/96819daa3d095cf9f662e0229dc82eaaa25480e8/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp#L362-L380).

The issue includes some information but this issue was discovered
because Rust uses futexes directly. So most likely we need to update
Rust as well to use these callbacks.

Also see the latest comments in #85188 for some context.
I also sent another PR #84162 to mark `pthread_*_lock` calls as
blocking.
---
 .../include/sanitizer/linux_syscall_hooks.h   |  16 ++-
 .../sanitizer_common_syscalls.inc             |  26 ++++
 .../lib/tsan/rtl/tsan_interceptors_posix.cpp  |  22 ++++
 .../test/tsan/signal_in_futex_wait.cpp        | 116 ++++++++++++++++++
 4 files changed, 178 insertions(+), 2 deletions(-)
 create mode 100644 compiler-rt/test/tsan/signal_in_futex_wait.cpp

diff --git a/compiler-rt/include/sanitizer/linux_syscall_hooks.h b/compiler-rt/include/sanitizer/linux_syscall_hooks.h
index 3f3f1e78dfb8..5f262455cb94 100644
--- a/compiler-rt/include/sanitizer/linux_syscall_hooks.h
+++ b/compiler-rt/include/sanitizer/linux_syscall_hooks.h
@@ -1856,6 +1856,15 @@
   __sanitizer_syscall_pre_impl_sigaltstack((long)ss, (long)oss)
 #define __sanitizer_syscall_post_sigaltstack(res, ss, oss)                     \
   __sanitizer_syscall_post_impl_sigaltstack(res, (long)ss, (long)oss)
+#define __sanitizer_syscall_pre_futex(uaddr, futex_op, val, timeout, uaddr2,   \
+                                      val3)                                    \
+  __sanitizer_syscall_pre_impl_futex((long)uaddr, (long)futex_op, (long)val,   \
+                                     (long)timeout, (long)uaddr2, (long)val3)
+#define __sanitizer_syscall_post_futex(res, uaddr, futex_op, val, timeout,     \
+                                       uaddr2, val3)                           \
+  __sanitizer_syscall_post_impl_futex(res, (long)uaddr, (long)futex_op,        \
+                                      (long)val, (long)timeout, (long)uaddr2,  \
+                                      (long)val3)
 
 // And now a few syscalls we don't handle yet.
 #define __sanitizer_syscall_pre_afs_syscall(...)
@@ -1875,7 +1884,6 @@
 #define __sanitizer_syscall_pre_fchown32(...)
 #define __sanitizer_syscall_pre_ftime(...)
 #define __sanitizer_syscall_pre_ftruncate64(...)
-#define __sanitizer_syscall_pre_futex(...)
 #define __sanitizer_syscall_pre_getegid32(...)
 #define __sanitizer_syscall_pre_geteuid32(...)
 #define __sanitizer_syscall_pre_getgid32(...)
@@ -1954,7 +1962,6 @@
 #define __sanitizer_syscall_post_fchown32(res, ...)
 #define __sanitizer_syscall_post_ftime(res, ...)
 #define __sanitizer_syscall_post_ftruncate64(res, ...)
-#define __sanitizer_syscall_post_futex(res, ...)
 #define __sanitizer_syscall_post_getegid32(res, ...)
 #define __sanitizer_syscall_post_geteuid32(res, ...)
 #define __sanitizer_syscall_post_getgid32(res, ...)
@@ -3093,6 +3100,11 @@ void __sanitizer_syscall_post_impl_rt_sigaction(long res, long signum, long act,
                                                 long oldact, long sz);
 void __sanitizer_syscall_pre_impl_sigaltstack(long ss, long oss);
 void __sanitizer_syscall_post_impl_sigaltstack(long res, long ss, long oss);
+void __sanitizer_syscall_pre_impl_futex(long uaddr, long futex_op, long val,
+                                        long timeout, long uaddr2, long val3);
+void __sanitizer_syscall_post_impl_futex(long res, long uaddr, long futex_op,
+                                         long val, long timeout, long uaddr2,
+                                         long val3);
 #ifdef __cplusplus
 } // extern "C"
 #endif
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc
index b3161690f3ce..14615f9668de 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc
@@ -38,6 +38,10 @@
 //          Called before fork syscall.
 //   COMMON_SYSCALL_POST_FORK(long res)
 //          Called after fork syscall.
+//   COMMON_SYSCALL_BLOCKING_START()
+//         Called before blocking syscall.
+//   COMMON_SYSCALL_BLOCKING_END()
+//         Called after blocking syscall.
 //===----------------------------------------------------------------------===//
 
 #include "sanitizer_platform.h"
@@ -85,6 +89,16 @@
       {}
 #  endif
 
+#  ifndef COMMON_SYSCALL_BLOCKING_START
+#    define COMMON_SYSCALL_BLOCKING_START() \
+      {}
+#  endif
+
+#  ifndef COMMON_SYSCALL_BLOCKING_END
+#    define COMMON_SYSCALL_BLOCKING_END() \
+      {}
+#  endif
+
 // FIXME: do some kind of PRE_READ for all syscall arguments (int(s) and such).
 
 extern "C" {
@@ -3176,6 +3190,18 @@ POST_SYSCALL(sigaltstack)(long res, void *ss, void *oss) {
     }
   }
 }
+
+PRE_SYSCALL(futex)
+(void *uaddr, long futex_op, long val, void *timeout, void *uaddr2, long val3) {
+  COMMON_SYSCALL_BLOCKING_START();
+}
+
+POST_SYSCALL(futex)
+(long res, void *uaddr, long futex_op, long val, void *timeout, void *uaddr2,
+ long val3) {
+  COMMON_SYSCALL_BLOCKING_END();
+}
+
 }  // extern "C"
 
 #  undef PRE_SYSCALL
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
index 26c2cfef4e77..3103f4717c16 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
@@ -2673,6 +2673,25 @@ static USED void syscall_fd_release(uptr pc, int fd) {
   FdRelease(thr, pc, fd);
 }
 
+static USED void sycall_blocking_start() {
+  DPrintf("sycall_blocking_start()\n");
+  ThreadState *thr = cur_thread();
+  EnterBlockingFunc(thr);
+  // When we are in a "blocking call", we process signals asynchronously
+  // (right when they arrive). In this context we do not expect to be
+  // executing any user/runtime code. The known interceptor sequence when
+  // this is not true is: pthread_join -> munmap(stack). It's fine
+  // to ignore munmap in this case -- we handle stack shadow separately.
+  thr->ignore_interceptors++;
+}
+
+static USED void sycall_blocking_end() {
+  DPrintf("sycall_blocking_end()\n");
+  ThreadState *thr = cur_thread();
+  thr->ignore_interceptors--;
+  atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed);
+}
+
 static void syscall_pre_fork(uptr pc) { ForkBefore(cur_thread(), pc); }
 
 static void syscall_post_fork(uptr pc, int pid) {
@@ -2727,6 +2746,9 @@ static void syscall_post_fork(uptr pc, int pid) {
 #define COMMON_SYSCALL_POST_FORK(res) \
   syscall_post_fork(GET_CALLER_PC(), res)
 
+#define COMMON_SYSCALL_BLOCKING_START() sycall_blocking_start()
+#define COMMON_SYSCALL_BLOCKING_END() sycall_blocking_end()
+
 #include "sanitizer_common/sanitizer_common_syscalls.inc"
 #include "sanitizer_common/sanitizer_syscalls_netbsd.inc"
 
diff --git a/compiler-rt/test/tsan/signal_in_futex_wait.cpp b/compiler-rt/test/tsan/signal_in_futex_wait.cpp
new file mode 100644
index 000000000000..cf31e5467486
--- /dev/null
+++ b/compiler-rt/test/tsan/signal_in_futex_wait.cpp
@@ -0,0 +1,116 @@
+// RUN: %clang_tsan %s -lstdc++ -o %t && %run %t 2>&1 | FileCheck %s
+
+#include "test.h"
+#include <errno.h>
+#include <linux/futex.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdio.h>
+#include <sys/syscall.h>
+
+#include <cassert>
+#include <stdexcept>
+#include <thread>
+
+#include <sanitizer/linux_syscall_hooks.h>
+
+int futex(int *uaddr, int futex_op, int val, const struct timespec *timeout,
+          int *uaddr2, int val3) {
+  __sanitizer_syscall_pre_futex(uaddr, futex_op, val, timeout, uaddr2, val3);
+  int result = syscall(SYS_futex, uaddr, futex_op, val, timeout, uaddr2, val3);
+  __sanitizer_syscall_post_futex(result, uaddr, futex_op, val, timeout, uaddr2,
+                                 val3);
+  return result;
+}
+
+// Simple mutex implementation using futex.
+class Mutex {
+public:
+  Mutex() : value(0) {}
+
+  void lock() {
+    int c;
+    while ((c = __sync_val_compare_and_swap(&value, 0, 1)) != 0) {
+      if (c != 1)
+        continue;
+      int r = futex(&value, FUTEX_WAIT_PRIVATE, 1, nullptr, nullptr, 0);
+      if (r == -1 && errno != EAGAIN) {
+        fprintf(stderr, "futex wait error\n");
+        abort();
+      }
+    }
+  }
+
+  void unlock() {
+    value = 0;
+    int r = futex(&value, FUTEX_WAKE_PRIVATE, 1, nullptr, nullptr, 0);
+    if (r == -1) {
+      fprintf(stderr, "futex wake error\n");
+      abort();
+    }
+  }
+
+private:
+  int value;
+};
+
+Mutex mutex;
+
+void *Thread(void *x) {
+  // fprintf(stderr, "canova here thread 0\n");
+  // Waiting for the futex.
+  mutex.lock();
+  // fprintf(stderr, "canova here thread 1\n");
+  // Finished waiting.
+  return nullptr;
+}
+
+static void SigprofHandler(int signal, siginfo_t *info, void *context) {
+  // fprintf(stderr, "canova here sigprof handler\n");
+  // Unlock the futex.
+  mutex.unlock();
+}
+
+void InstallSignalHandler() {
+  struct sigaction sa;
+  sa.sa_sigaction = SigprofHandler;
+  sigemptyset(&sa.sa_mask);
+  sa.sa_flags = SA_RESTART | SA_SIGINFO;
+  if (sigaction(SIGPROF, &sa, 0) != 0) {
+    fprintf(stderr, "failed to install signal handler\n");
+    abort();
+  }
+}
+
+int main() {
+  alarm(60); // Kill the test if it hangs.
+
+  // Install the signal handler
+  InstallSignalHandler();
+
+  // Lock the futex at first so the other thread will wait for it.
+  mutex.lock();
+
+  // Create the thread to wait for the futex.
+  pthread_t thread;
+  pthread_create(&thread, NULL, Thread, NULL);
+
+  // Just waiting a bit to make sure the thead is at the FUTEX_WAIT_PRIVATE
+  // syscall.
+  std::this_thread::sleep_for(std::chrono::milliseconds(100));
+
+  // Send the signal to the other thread, which will send the futex wake
+  // syscall.
+  int r = pthread_kill(thread, SIGPROF);
+  assert(r == 0);
+
+  // Futex should be notified and the thread should be able to continue.
+  pthread_join(thread, NULL);
+
+  // Exiting successfully.
+  fprintf(stderr, "PASS\n");
+  return 0;
+}
+
+// CHECK-NOT: WARNING: ThreadSanitizer:
+// CHECK: PASS
-- 
GitLab


From da9ac43433ace54b262cd8552a9fafac21186381 Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas@arm.com>
Date: Tue, 26 Mar 2024 11:36:34 +0000
Subject: [PATCH 264/404] [FMV] Allow mixing target_version with target_clones.
 (#86493)

The latest ACLE allows it and further clarifies the following
in regards to the combination of the two attributes:

"If the `default` matches with another explicitly provided
 version in the same translation unit, then the compiler can
 emit only one function instead of the two. The explicitly
 provided version shall be preferred."

("default" refers to the default clone here)

https://github.com/ARM-software/acle/pull/310
---
 clang/include/clang/Basic/Attr.td             |  14 +
 clang/lib/AST/ASTContext.cpp                  |  15 +-
 clang/lib/CodeGen/CodeGenModule.cpp           | 105 ++++---
 clang/lib/Sema/SemaDecl.cpp                   | 190 ++++++++----
 .../CodeGen/aarch64-mixed-target-attributes.c | 278 ++++++++++++++++++
 .../test/CodeGen/attr-target-clones-aarch64.c |  94 +++---
 .../CodeGenCXX/attr-target-clones-aarch64.cpp |  28 +-
 clang/test/Sema/attr-target-clones-aarch64.c  |   2 +-
 8 files changed, 537 insertions(+), 189 deletions(-)
 create mode 100644 clang/test/CodeGen/aarch64-mixed-target-attributes.c

diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 3e03e5561264..318d4e5ac5ba 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -3088,6 +3088,20 @@ def TargetClones : InheritableAttr {
     StringRef getFeatureStr(unsigned Index) const {
       return *(featuresStrs_begin() + Index);
     }
+    bool isDefaultVersion(unsigned Index) const {
+      return getFeatureStr(Index) == "default";
+    }
+    void getFeatures(llvm::SmallVectorImpl<StringRef> &Out,
+                     unsigned Index) const {
+      if (isDefaultVersion(Index)) return;
+      StringRef Features = getFeatureStr(Index);
+      SmallVector<StringRef, 8> AttrFeatures;
+      Features.split(AttrFeatures, "+");
+      for (auto &Feature : AttrFeatures) {
+        Feature = Feature.trim();
+        Out.push_back(Feature);
+      }
+    }
     // Given an index into the 'featuresStrs' sequence, compute a unique
     // ID to be used with function name mangling for the associated variant.
     // This mapping is necessary due to a requirement that the mangling ID
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index fcf801adeaf5..0b5f20a572a7 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -13676,22 +13676,19 @@ void ASTContext::getFunctionFeatureMap(llvm::StringMap<bool> &FeatureMap,
     Target->initFeatureMap(FeatureMap, getDiagnostics(), TargetCPU, Features);
   } else if (const auto *TC = FD->getAttr<TargetClonesAttr>()) {
     std::vector<std::string> Features;
-    StringRef VersionStr = TC->getFeatureStr(GD.getMultiVersionIndex());
     if (Target->getTriple().isAArch64()) {
       // TargetClones for AArch64
-      if (VersionStr != "default") {
-        SmallVector<StringRef, 1> VersionFeatures;
-        VersionStr.split(VersionFeatures, "+");
-        for (auto &VFeature : VersionFeatures) {
-          VFeature = VFeature.trim();
+      llvm::SmallVector<StringRef, 8> Feats;
+      TC->getFeatures(Feats, GD.getMultiVersionIndex());
+      for (StringRef Feat : Feats)
+        if (Target->validateCpuSupports(Feat.str()))
           // Use '?' to mark features that came from AArch64 TargetClones.
-          Features.push_back((StringRef{"?"} + VFeature).str());
-        }
-      }
+          Features.push_back("?" + Feat.str());
       Features.insert(Features.begin(),
                       Target->getTargetOpts().FeaturesAsWritten.begin(),
                       Target->getTargetOpts().FeaturesAsWritten.end());
     } else {
+      StringRef VersionStr = TC->getFeatureStr(GD.getMultiVersionIndex());
       if (VersionStr.starts_with("arch="))
         TargetCPU = VersionStr.drop_front(sizeof("arch=") - 1);
       else if (VersionStr != "default")
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index da411a9fa579..3997da17bb28 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -3712,7 +3712,8 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) {
     // Forward declarations are emitted lazily on first use.
     if (!FD->doesThisDeclarationHaveABody()) {
       if (!FD->doesDeclarationForceExternallyVisibleDefinition() &&
-          !FD->isTargetVersionMultiVersion())
+          (!FD->isMultiVersion() ||
+           !FD->getASTContext().getTargetInfo().getTriple().isAArch64()))
         return;
 
       StringRef MangledName = getMangledName(GD);
@@ -3994,10 +3995,11 @@ void CodeGenModule::EmitMultiVersionFunctionDefinition(GlobalDecl GD,
     auto *Spec = FD->getAttr<CPUSpecificAttr>();
     for (unsigned I = 0; I < Spec->cpus_size(); ++I)
       EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr);
-  } else if (FD->isTargetClonesMultiVersion()) {
-    auto *Clone = FD->getAttr<TargetClonesAttr>();
-    for (unsigned I = 0; I < Clone->featuresStrs_size(); ++I)
-      if (Clone->isFirstOfVersion(I))
+  } else if (auto *TC = FD->getAttr<TargetClonesAttr>()) {
+    for (unsigned I = 0; I < TC->featuresStrs_size(); ++I)
+      // AArch64 favors the default target version over the clone if any.
+      if ((!TC->isDefaultVersion(I) || !getTarget().getTriple().isAArch64()) &&
+          TC->isFirstOfVersion(I))
         EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr);
     // Ensure that the resolver function is also emitted.
     GetOrCreateMultiVersionResolver(GD);
@@ -4137,57 +4139,49 @@ void CodeGenModule::emitMultiVersionFunctions() {
     };
 
     bool HasDefaultDecl = !FD->isTargetVersionMultiVersion();
-    bool ShouldEmitResolver = !FD->isTargetVersionMultiVersion();
+    bool ShouldEmitResolver =
+        !getContext().getTargetInfo().getTriple().isAArch64();
     SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
-    if (FD->isTargetMultiVersion()) {
-      getContext().forEachMultiversionedFunctionVersion(
-          FD, [&](const FunctionDecl *CurFD) {
-            llvm::SmallVector<StringRef, 8> Feats;
-            llvm::Function *Func = createFunction(CurFD);
 
-            if (const auto *TA = CurFD->getAttr<TargetAttr>()) {
-              TA->getAddedFeatures(Feats);
-              Options.emplace_back(Func, TA->getArchitecture(), Feats);
-            } else if (const auto *TVA = CurFD->getAttr<TargetVersionAttr>()) {
-              bool HasDefaultDef = TVA->isDefaultVersion() &&
-                                   CurFD->doesThisDeclarationHaveABody();
-              HasDefaultDecl |= TVA->isDefaultVersion();
-              ShouldEmitResolver |= (CurFD->isUsed() || HasDefaultDef);
-              TVA->getFeatures(Feats);
-              Options.emplace_back(Func, /*Architecture*/ "", Feats);
-            } else
-              llvm_unreachable("unexpected MultiVersionKind");
-          });
-    } else if (const auto *TC = FD->getAttr<TargetClonesAttr>()) {
-      for (unsigned I = 0; I < TC->featuresStrs_size(); ++I) {
-        if (!TC->isFirstOfVersion(I))
-          continue;
+    getContext().forEachMultiversionedFunctionVersion(
+        FD, [&](const FunctionDecl *CurFD) {
+          llvm::SmallVector<StringRef, 8> Feats;
 
-        llvm::Function *Func = createFunction(FD, I);
-        StringRef Version = TC->getFeatureStr(I);
-        StringRef Architecture;
-        llvm::SmallVector<StringRef, 1> Feature;
-
-        if (getTarget().getTriple().isAArch64()) {
-          if (Version != "default") {
-            llvm::SmallVector<StringRef, 8> VerFeats;
-            Version.split(VerFeats, "+");
-            for (auto &CurFeat : VerFeats)
-              Feature.push_back(CurFeat.trim());
-          }
-        } else {
-          if (Version.starts_with("arch="))
-            Architecture = Version.drop_front(sizeof("arch=") - 1);
-          else if (Version != "default")
-            Feature.push_back(Version);
-        }
-
-        Options.emplace_back(Func, Architecture, Feature);
-      }
-    } else {
-      assert(0 && "Expected a target or target_clones multiversion function");
-      continue;
-    }
+          if (const auto *TA = CurFD->getAttr<TargetAttr>()) {
+            TA->getAddedFeatures(Feats);
+            llvm::Function *Func = createFunction(CurFD);
+            Options.emplace_back(Func, TA->getArchitecture(), Feats);
+          } else if (const auto *TVA = CurFD->getAttr<TargetVersionAttr>()) {
+            bool HasDefaultDef = TVA->isDefaultVersion() &&
+                                 CurFD->doesThisDeclarationHaveABody();
+            HasDefaultDecl |= TVA->isDefaultVersion();
+            ShouldEmitResolver |= (CurFD->isUsed() || HasDefaultDef);
+            TVA->getFeatures(Feats);
+            llvm::Function *Func = createFunction(CurFD);
+            Options.emplace_back(Func, /*Architecture*/ "", Feats);
+          } else if (const auto *TC = CurFD->getAttr<TargetClonesAttr>()) {
+            ShouldEmitResolver |= CurFD->doesThisDeclarationHaveABody();
+            for (unsigned I = 0; I < TC->featuresStrs_size(); ++I) {
+              if (!TC->isFirstOfVersion(I))
+                continue;
+
+              llvm::Function *Func = createFunction(CurFD, I);
+              StringRef Architecture;
+              Feats.clear();
+              if (getTarget().getTriple().isAArch64())
+                TC->getFeatures(Feats, I);
+              else {
+                StringRef Version = TC->getFeatureStr(I);
+                if (Version.starts_with("arch="))
+                  Architecture = Version.drop_front(sizeof("arch=") - 1);
+                else if (Version != "default")
+                  Feats.push_back(Version);
+              }
+              Options.emplace_back(Func, Architecture, Feats);
+            }
+          } else
+            llvm_unreachable("unexpected MultiVersionKind");
+        });
 
     if (!ShouldEmitResolver)
       continue;
@@ -4378,7 +4372,7 @@ void CodeGenModule::AddDeferredMultiVersionResolverToEmit(GlobalDecl GD) {
   const auto *FD = cast<FunctionDecl>(GD.getDecl());
   assert(FD && "Not a FunctionDecl?");
 
-  if (FD->isTargetVersionMultiVersion()) {
+  if (FD->isTargetVersionMultiVersion() || FD->isTargetClonesMultiVersion()) {
     std::string MangledName =
         getMangledNameImpl(*this, GD, FD, /*OmitMultiVersionMangling=*/true);
     if (!DeferredResolversToEmit.insert(MangledName).second)
@@ -4489,7 +4483,8 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction(
 
     if (FD->isMultiVersion()) {
       UpdateMultiVersionNames(GD, FD, MangledName);
-      if (FD->isTargetVersionMultiVersion() && !FD->isUsed())
+      if (FD->getASTContext().getTargetInfo().getTriple().isAArch64() &&
+          !FD->isUsed())
         AddDeferredMultiVersionResolverToEmit(GD);
       else if (!IsForDefinition)
         return GetOrCreateMultiVersionResolver(GD);
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 73ea155053d7..9a286e0b26a4 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -11260,11 +11260,13 @@ static bool checkNonMultiVersionCompatAttributes(Sema &S,
         return Diagnose(S, A);
       break;
     case attr::TargetVersion:
-      if (MVKind != MultiVersionKind::TargetVersion)
+      if (MVKind != MultiVersionKind::TargetVersion &&
+          MVKind != MultiVersionKind::TargetClones)
         return Diagnose(S, A);
       break;
     case attr::TargetClones:
-      if (MVKind != MultiVersionKind::TargetClones)
+      if (MVKind != MultiVersionKind::TargetClones &&
+          MVKind != MultiVersionKind::TargetVersion)
         return Diagnose(S, A);
       break;
     default:
@@ -11469,6 +11471,22 @@ static bool PreviousDeclsHaveMultiVersionAttribute(const FunctionDecl *FD) {
   return false;
 }
 
+static void patchDefaultTargetVersion(FunctionDecl *From, FunctionDecl *To) {
+  if (!From->getASTContext().getTargetInfo().getTriple().isAArch64())
+    return;
+
+  MultiVersionKind MVKindFrom = From->getMultiVersionKind();
+  MultiVersionKind MVKindTo = To->getMultiVersionKind();
+
+  if (MVKindTo == MultiVersionKind::None &&
+      (MVKindFrom == MultiVersionKind::TargetVersion ||
+       MVKindFrom == MultiVersionKind::TargetClones)) {
+    To->setIsMultiVersion();
+    To->addAttr(TargetVersionAttr::CreateImplicit(
+        To->getASTContext(), "default", To->getSourceRange()));
+  }
+}
+
 static bool CheckTargetCausesMultiVersioning(Sema &S, FunctionDecl *OldFD,
                                              FunctionDecl *NewFD,
                                              bool &Redeclaration,
@@ -11479,10 +11497,7 @@ static bool CheckTargetCausesMultiVersioning(Sema &S, FunctionDecl *OldFD,
   // The definitions should be allowed in any order. If we have discovered
   // a new target version and the preceeding was the default, then add the
   // corresponding attribute to it.
-  if (OldFD->getMultiVersionKind() == MultiVersionKind::None &&
-      NewFD->getMultiVersionKind() == MultiVersionKind::TargetVersion)
-    OldFD->addAttr(TargetVersionAttr::CreateImplicit(S.Context, "default",
-                                                     OldFD->getSourceRange()));
+  patchDefaultTargetVersion(NewFD, OldFD);
 
   const auto *NewTA = NewFD->getAttr<TargetAttr>();
   const auto *NewTVA = NewFD->getAttr<TargetVersionAttr>();
@@ -11583,36 +11598,60 @@ static bool CheckTargetCausesMultiVersioning(Sema &S, FunctionDecl *OldFD,
   return false;
 }
 
-static bool MultiVersionTypesCompatible(MultiVersionKind Old,
-                                        MultiVersionKind New) {
-  if (Old == New || Old == MultiVersionKind::None ||
-      New == MultiVersionKind::None)
+static bool MultiVersionTypesCompatible(FunctionDecl *Old, FunctionDecl *New) {
+  MultiVersionKind OldKind = Old->getMultiVersionKind();
+  MultiVersionKind NewKind = New->getMultiVersionKind();
+
+  if (OldKind == NewKind || OldKind == MultiVersionKind::None ||
+      NewKind == MultiVersionKind::None)
     return true;
 
-  return (Old == MultiVersionKind::CPUDispatch &&
-          New == MultiVersionKind::CPUSpecific) ||
-         (Old == MultiVersionKind::CPUSpecific &&
-          New == MultiVersionKind::CPUDispatch);
+  if (Old->getASTContext().getTargetInfo().getTriple().isAArch64()) {
+    switch (OldKind) {
+    case MultiVersionKind::TargetVersion:
+      return NewKind == MultiVersionKind::TargetClones;
+    case MultiVersionKind::TargetClones:
+      return NewKind == MultiVersionKind::TargetVersion;
+    default:
+      return false;
+    }
+  } else {
+    switch (OldKind) {
+    case MultiVersionKind::CPUDispatch:
+      return NewKind == MultiVersionKind::CPUSpecific;
+    case MultiVersionKind::CPUSpecific:
+      return NewKind == MultiVersionKind::CPUDispatch;
+    default:
+      return false;
+    }
+  }
 }
 
 /// Check the validity of a new function declaration being added to an existing
 /// multiversioned declaration collection.
 static bool CheckMultiVersionAdditionalDecl(
     Sema &S, FunctionDecl *OldFD, FunctionDecl *NewFD,
-    MultiVersionKind NewMVKind, const CPUDispatchAttr *NewCPUDisp,
-    const CPUSpecificAttr *NewCPUSpec, const TargetClonesAttr *NewClones,
-    bool &Redeclaration, NamedDecl *&OldDecl, LookupResult &Previous) {
-  const auto *NewTA = NewFD->getAttr<TargetAttr>();
-  const auto *NewTVA = NewFD->getAttr<TargetVersionAttr>();
-  MultiVersionKind OldMVKind = OldFD->getMultiVersionKind();
+    const CPUDispatchAttr *NewCPUDisp, const CPUSpecificAttr *NewCPUSpec,
+    const TargetClonesAttr *NewClones, bool &Redeclaration, NamedDecl *&OldDecl,
+    LookupResult &Previous) {
+
   // Disallow mixing of multiversioning types.
-  if (!MultiVersionTypesCompatible(OldMVKind, NewMVKind)) {
+  if (!MultiVersionTypesCompatible(OldFD, NewFD)) {
     S.Diag(NewFD->getLocation(), diag::err_multiversion_types_mixed);
     S.Diag(OldFD->getLocation(), diag::note_previous_declaration);
     NewFD->setInvalidDecl();
     return true;
   }
 
+  // Add the default target_version attribute if it's missing.
+  patchDefaultTargetVersion(OldFD, NewFD);
+  patchDefaultTargetVersion(NewFD, OldFD);
+
+  const auto *NewTA = NewFD->getAttr<TargetAttr>();
+  const auto *NewTVA = NewFD->getAttr<TargetVersionAttr>();
+  MultiVersionKind NewMVKind = NewFD->getMultiVersionKind();
+  MultiVersionKind OldMVKind = OldFD->getMultiVersionKind();
+
   ParsedTargetAttr NewParsed;
   if (NewTA) {
     NewParsed = S.getASTContext().getTargetInfo().parseTargetAttr(
@@ -11641,19 +11680,6 @@ static bool CheckMultiVersionAdditionalDecl(
         S.IsOverload(NewFD, CurFD, UseMemberUsingDeclRules))
       continue;
 
-    if (NewMVKind == MultiVersionKind::None &&
-        OldMVKind == MultiVersionKind::TargetVersion) {
-      NewFD->addAttr(TargetVersionAttr::CreateImplicit(
-          S.Context, "default", NewFD->getSourceRange()));
-      NewFD->setIsMultiVersion();
-      NewMVKind = MultiVersionKind::TargetVersion;
-      if (!NewTVA) {
-        NewTVA = NewFD->getAttr<TargetVersionAttr>();
-        NewTVA->getFeatures(NewFeats);
-        llvm::sort(NewFeats);
-      }
-    }
-
     switch (NewMVKind) {
     case MultiVersionKind::None:
       assert(OldMVKind == MultiVersionKind::TargetClones &&
@@ -11681,43 +11707,81 @@ static bool CheckMultiVersionAdditionalDecl(
       break;
     }
     case MultiVersionKind::TargetVersion: {
-      const auto *CurTVA = CurFD->getAttr<TargetVersionAttr>();
-      if (CurTVA->getName() == NewTVA->getName()) {
-        NewFD->setIsMultiVersion();
-        Redeclaration = true;
-        OldDecl = ND;
-        return false;
-      }
-      llvm::SmallVector<StringRef, 8> CurFeats;
-      if (CurTVA) {
+      if (const auto *CurTVA = CurFD->getAttr<TargetVersionAttr>()) {
+        if (CurTVA->getName() == NewTVA->getName()) {
+          NewFD->setIsMultiVersion();
+          Redeclaration = true;
+          OldDecl = ND;
+          return false;
+        }
+        llvm::SmallVector<StringRef, 8> CurFeats;
         CurTVA->getFeatures(CurFeats);
         llvm::sort(CurFeats);
-      }
-      if (CurFeats == NewFeats) {
-        S.Diag(NewFD->getLocation(), diag::err_multiversion_duplicate);
-        S.Diag(CurFD->getLocation(), diag::note_previous_declaration);
-        NewFD->setInvalidDecl();
-        return true;
+
+        if (CurFeats == NewFeats) {
+          S.Diag(NewFD->getLocation(), diag::err_multiversion_duplicate);
+          S.Diag(CurFD->getLocation(), diag::note_previous_declaration);
+          NewFD->setInvalidDecl();
+          return true;
+        }
+      } else if (const auto *CurClones = CurFD->getAttr<TargetClonesAttr>()) {
+        // Default
+        if (NewFeats.empty())
+          break;
+
+        for (unsigned I = 0; I < CurClones->featuresStrs_size(); ++I) {
+          llvm::SmallVector<StringRef, 8> CurFeats;
+          CurClones->getFeatures(CurFeats, I);
+          llvm::sort(CurFeats);
+
+          if (CurFeats == NewFeats) {
+            S.Diag(NewFD->getLocation(), diag::err_multiversion_duplicate);
+            S.Diag(CurFD->getLocation(), diag::note_previous_declaration);
+            NewFD->setInvalidDecl();
+            return true;
+          }
+        }
       }
       break;
     }
     case MultiVersionKind::TargetClones: {
-      const auto *CurClones = CurFD->getAttr<TargetClonesAttr>();
+      assert(NewClones && "MultiVersionKind does not match attribute type");
+      if (const auto *CurClones = CurFD->getAttr<TargetClonesAttr>()) {
+        if (CurClones->featuresStrs_size() != NewClones->featuresStrs_size() ||
+            !std::equal(CurClones->featuresStrs_begin(),
+                        CurClones->featuresStrs_end(),
+                        NewClones->featuresStrs_begin())) {
+          S.Diag(NewFD->getLocation(), diag::err_target_clone_doesnt_match);
+          S.Diag(CurFD->getLocation(), diag::note_previous_declaration);
+          NewFD->setInvalidDecl();
+          return true;
+        }
+      } else if (const auto *CurTVA = CurFD->getAttr<TargetVersionAttr>()) {
+        llvm::SmallVector<StringRef, 8> CurFeats;
+        CurTVA->getFeatures(CurFeats);
+        llvm::sort(CurFeats);
+
+        // Default
+        if (CurFeats.empty())
+          break;
+
+        for (unsigned I = 0; I < NewClones->featuresStrs_size(); ++I) {
+          NewFeats.clear();
+          NewClones->getFeatures(NewFeats, I);
+          llvm::sort(NewFeats);
+
+          if (CurFeats == NewFeats) {
+            S.Diag(NewFD->getLocation(), diag::err_multiversion_duplicate);
+            S.Diag(CurFD->getLocation(), diag::note_previous_declaration);
+            NewFD->setInvalidDecl();
+            return true;
+          }
+        }
+        break;
+      }
       Redeclaration = true;
       OldDecl = CurFD;
       NewFD->setIsMultiVersion();
-
-      if (CurClones && NewClones &&
-          (CurClones->featuresStrs_size() != NewClones->featuresStrs_size() ||
-           !std::equal(CurClones->featuresStrs_begin(),
-                       CurClones->featuresStrs_end(),
-                       NewClones->featuresStrs_begin()))) {
-        S.Diag(NewFD->getLocation(), diag::err_target_clone_doesnt_match);
-        S.Diag(CurFD->getLocation(), diag::note_previous_declaration);
-        NewFD->setInvalidDecl();
-        return true;
-      }
-
       return false;
     }
     case MultiVersionKind::CPUSpecific:
@@ -11913,7 +11977,7 @@ static bool CheckMultiVersionFunction(Sema &S, FunctionDecl *NewFD,
   // At this point, we have a multiversion function decl (in OldFD) AND an
   // appropriate attribute in the current function decl.  Resolve that these are
   // still compatible with previous declarations.
-  return CheckMultiVersionAdditionalDecl(S, OldFD, NewFD, MVKind, NewCPUDisp,
+  return CheckMultiVersionAdditionalDecl(S, OldFD, NewFD, NewCPUDisp,
                                          NewCPUSpec, NewClones, Redeclaration,
                                          OldDecl, Previous);
 }
diff --git a/clang/test/CodeGen/aarch64-mixed-target-attributes.c b/clang/test/CodeGen/aarch64-mixed-target-attributes.c
new file mode 100644
index 000000000000..aef6ce36ab1c
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-mixed-target-attributes.c
@@ -0,0 +1,278 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --check-attributes --check-globals --include-generated-funcs
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature -v9.5a -S -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature -fmv -S -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK-NOFMV
+
+// The following is guarded because in NOFMV we get an error for redefining the default.
+#ifdef __HAVE_FUNCTION_MULTI_VERSIONING
+int explicit_default(void) { return 0; }
+__attribute__((target_version("jscvt"))) int explicit_default(void) { return 1; }
+__attribute__((target_clones("dotprod", "lse"))) int explicit_default(void) { return 2; }
+__attribute__((target_version("rdma"))) int explicit_default(void) { return 3; }
+
+int foo(void) { return explicit_default(); }
+#endif
+
+__attribute__((target_version("jscvt"))) int implicit_default(void) { return 1; }
+__attribute__((target_clones("dotprod", "lse"))) int implicit_default(void) { return 2; }
+__attribute__((target_version("rdma"))) int implicit_default(void) { return 3; }
+
+int bar(void) { return implicit_default(); }
+
+// These shouldn't generate anything.
+int unused_version_declarations(void);
+__attribute__((target_clones("dotprod", "lse"))) int unused_version_declarations(void);
+__attribute__((target_version("jscvt"))) int unused_version_declarations(void);
+
+// These should generate the default (mangled) version and the resolver.
+int default_def_with_version_decls(void) { return 0; }
+__attribute__((target_clones("dotprod", "lse"))) int default_def_with_version_decls(void);
+__attribute__((target_version("jscvt"))) int default_def_with_version_decls(void);
+
+//.
+// CHECK: @__aarch64_cpu_features = external dso_local global { i64 }
+// CHECK: @explicit_default.ifunc = weak_odr alias i32 (), ptr @explicit_default
+// CHECK: @implicit_default.ifunc = weak_odr alias i32 (), ptr @implicit_default
+// CHECK: @default_def_with_version_decls.ifunc = weak_odr alias i32 (), ptr @default_def_with_version_decls
+// CHECK: @explicit_default = weak_odr ifunc i32 (), ptr @explicit_default.resolver
+// CHECK: @implicit_default = weak_odr ifunc i32 (), ptr @implicit_default.resolver
+// CHECK: @default_def_with_version_decls = weak_odr ifunc i32 (), ptr @default_def_with_version_decls.resolver
+//.
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@explicit_default.default
+// CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 0
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@explicit_default._Mjscvt
+// CHECK-SAME: () #[[ATTR1:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@explicit_default._Mdotprod
+// CHECK-SAME: () #[[ATTR2:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 2
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@explicit_default._Mlse
+// CHECK-SAME: () #[[ATTR3:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 2
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@explicit_default.resolver() comdat {
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1048576
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK:       resolver_return:
+// CHECK-NEXT:    ret ptr @explicit_default._Mjscvt
+// CHECK:       resolver_else:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 64
+// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 64
+// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK:       resolver_return1:
+// CHECK-NEXT:    ret ptr @explicit_default._Mrdm
+// CHECK:       resolver_else2:
+// CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], 16
+// CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 16
+// CHECK-NEXT:    [[TMP11:%.*]] = and i1 true, [[TMP10]]
+// CHECK-NEXT:    br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]]
+// CHECK:       resolver_return3:
+// CHECK-NEXT:    ret ptr @explicit_default._Mdotprod
+// CHECK:       resolver_else4:
+// CHECK-NEXT:    [[TMP12:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 128
+// CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 128
+// CHECK-NEXT:    [[TMP15:%.*]] = and i1 true, [[TMP14]]
+// CHECK-NEXT:    br i1 [[TMP15]], label [[RESOLVER_RETURN5:%.*]], label [[RESOLVER_ELSE6:%.*]]
+// CHECK:       resolver_return5:
+// CHECK-NEXT:    ret ptr @explicit_default._Mlse
+// CHECK:       resolver_else6:
+// CHECK-NEXT:    ret ptr @explicit_default.default
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@explicit_default._Mrdm
+// CHECK-SAME: () #[[ATTR4:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 3
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@foo
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CALL:%.*]] = call i32 @explicit_default()
+// CHECK-NEXT:    ret i32 [[CALL]]
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@implicit_default._Mjscvt
+// CHECK-SAME: () #[[ATTR1]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@implicit_default._Mdotprod
+// CHECK-SAME: () #[[ATTR2]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 2
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@implicit_default._Mlse
+// CHECK-SAME: () #[[ATTR3]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 2
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@implicit_default.resolver() comdat {
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1048576
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK:       resolver_return:
+// CHECK-NEXT:    ret ptr @implicit_default._Mjscvt
+// CHECK:       resolver_else:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 64
+// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 64
+// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK:       resolver_return1:
+// CHECK-NEXT:    ret ptr @implicit_default._Mrdm
+// CHECK:       resolver_else2:
+// CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], 16
+// CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 16
+// CHECK-NEXT:    [[TMP11:%.*]] = and i1 true, [[TMP10]]
+// CHECK-NEXT:    br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]]
+// CHECK:       resolver_return3:
+// CHECK-NEXT:    ret ptr @implicit_default._Mdotprod
+// CHECK:       resolver_else4:
+// CHECK-NEXT:    [[TMP12:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 128
+// CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 128
+// CHECK-NEXT:    [[TMP15:%.*]] = and i1 true, [[TMP14]]
+// CHECK-NEXT:    br i1 [[TMP15]], label [[RESOLVER_RETURN5:%.*]], label [[RESOLVER_ELSE6:%.*]]
+// CHECK:       resolver_return5:
+// CHECK-NEXT:    ret ptr @implicit_default._Mlse
+// CHECK:       resolver_else6:
+// CHECK-NEXT:    ret ptr @implicit_default.default
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@implicit_default._Mrdm
+// CHECK-SAME: () #[[ATTR4]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 3
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@bar
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CALL:%.*]] = call i32 @implicit_default()
+// CHECK-NEXT:    ret i32 [[CALL]]
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@default_def_with_version_decls.default
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 0
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define {{[^@]+}}@implicit_default.default
+// CHECK-SAME: () #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 2
+//
+//
+// CHECK-LABEL: define {{[^@]+}}@default_def_with_version_decls.resolver() comdat {
+// CHECK-NEXT:  resolver_entry:
+// CHECK-NEXT:    call void @__init_cpu_features_resolver()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 1048576
+// CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576
+// CHECK-NEXT:    [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT:    br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
+// CHECK:       resolver_return:
+// CHECK-NEXT:    ret ptr @default_def_with_version_decls._Mjscvt
+// CHECK:       resolver_else:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 16
+// CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 16
+// CHECK-NEXT:    [[TMP7:%.*]] = and i1 true, [[TMP6]]
+// CHECK-NEXT:    br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK:       resolver_return1:
+// CHECK-NEXT:    ret ptr @default_def_with_version_decls._Mdotprod
+// CHECK:       resolver_else2:
+// CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], 128
+// CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 128
+// CHECK-NEXT:    [[TMP11:%.*]] = and i1 true, [[TMP10]]
+// CHECK-NEXT:    br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]]
+// CHECK:       resolver_return3:
+// CHECK-NEXT:    ret ptr @default_def_with_version_decls._Mlse
+// CHECK:       resolver_else4:
+// CHECK-NEXT:    ret ptr @default_def_with_version_decls.default
+//
+//
+// CHECK-NOFMV: Function Attrs: noinline nounwind optnone
+// CHECK-NOFMV-LABEL: define {{[^@]+}}@implicit_default
+// CHECK-NOFMV-SAME: () #[[ATTR0:[0-9]+]] {
+// CHECK-NOFMV-NEXT:  entry:
+// CHECK-NOFMV-NEXT:    ret i32 2
+//
+//
+// CHECK-NOFMV: Function Attrs: noinline nounwind optnone
+// CHECK-NOFMV-LABEL: define {{[^@]+}}@bar
+// CHECK-NOFMV-SAME: () #[[ATTR0]] {
+// CHECK-NOFMV-NEXT:  entry:
+// CHECK-NOFMV-NEXT:    [[CALL:%.*]] = call i32 @implicit_default()
+// CHECK-NOFMV-NEXT:    ret i32 [[CALL]]
+//
+//
+// CHECK-NOFMV: Function Attrs: noinline nounwind optnone
+// CHECK-NOFMV-LABEL: define {{[^@]+}}@default_def_with_version_decls
+// CHECK-NOFMV-SAME: () #[[ATTR0]] {
+// CHECK-NOFMV-NEXT:  entry:
+// CHECK-NOFMV-NEXT:    ret i32 0
+//
+//.
+// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-v9.5a" }
+// CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+jsconv,+neon,-v9.5a" }
+// CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon,-v9.5a" }
+// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-v9.5a" }
+// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm,-v9.5a" }
+// CHECK: attributes #[[ATTR5:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon,-v9.5a" }
+// CHECK: attributes #[[ATTR6:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+jsconv,+neon,-v9.5a" }
+// CHECK: attributes #[[ATTR7:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-v9.5a" }
+// CHECK: attributes #[[ATTR8:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-v9.5a" }
+//.
+// CHECK-NOFMV: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" }
+//.
+// CHECK: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+// CHECK: [[META1:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+//.
+// CHECK-NOFMV: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+// CHECK-NOFMV: [[META1:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+//.
diff --git a/clang/test/CodeGen/attr-target-clones-aarch64.c b/clang/test/CodeGen/attr-target-clones-aarch64.c
index 94095f9aa3e1..8c8b951e9118 100644
--- a/clang/test/CodeGen/attr-target-clones-aarch64.c
+++ b/clang/test/CodeGen/attr-target-clones-aarch64.c
@@ -29,8 +29,8 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK: @ftc_def.ifunc = weak_odr alias i32 (), ptr @ftc_def
 // CHECK: @ftc_dup1.ifunc = weak_odr alias i32 (), ptr @ftc_dup1
 // CHECK: @ftc_dup2.ifunc = weak_odr alias i32 (), ptr @ftc_dup2
-// CHECK: @ftc_inline1.ifunc = weak_odr alias i32 (), ptr @ftc_inline1
 // CHECK: @ftc_inline2.ifunc = weak_odr alias i32 (), ptr @ftc_inline2
+// CHECK: @ftc_inline1.ifunc = weak_odr alias i32 (), ptr @ftc_inline1
 // CHECK: @ftc_inline3.ifunc = weak_odr alias i32 (), ptr @ftc_inline3
 // CHECK: @ftc = weak_odr ifunc i32 (), ptr @ftc.resolver
 // CHECK: @ftc_def = weak_odr ifunc i32 (), ptr @ftc_def.resolver
@@ -52,12 +52,6 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-NEXT:    ret i32 0
 //
 //
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: @ftc.default(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 0
-//
-//
 // CHECK-LABEL: @ftc.resolver(
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
@@ -92,12 +86,6 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-NEXT:    ret i32 1
 //
 //
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: @ftc_def.default(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
-//
-//
 // CHECK-LABEL: @ftc_def.resolver(
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
@@ -126,12 +114,6 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-NEXT:    ret i32 2
 //
 //
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: @ftc_dup1.default(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 2
-//
-//
 // CHECK-LABEL: @ftc_dup1.resolver(
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
@@ -158,12 +140,6 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 // CHECK-NEXT:    ret i32 3
 //
 //
-// CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: @ftc_dup2.default(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 3
-//
-//
 // CHECK-LABEL: @ftc_dup2.resolver(
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
@@ -192,6 +168,12 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: @ftc_inline2._Mfp16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 2
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
 // CHECK-LABEL: @ftc_direct(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 4
@@ -287,45 +269,63 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: @ftc_inline1._MrngMsimd(
+// CHECK-LABEL: @ftc_inline2._MfcmaMsve2-bitperm(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
+// CHECK-NEXT:    ret i32 2
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: @ftc_inline1._MpredresMrcpc(
+// CHECK-LABEL: @ftc.default(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
+// CHECK-NEXT:    ret i32 0
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: @ftc_inline1._Msve2-aesMwfxt(
+// CHECK-LABEL: @ftc_def.default(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 1
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: @ftc_inline1.default(
+// CHECK-LABEL: @ftc_dup1.default(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 1
+// CHECK-NEXT:    ret i32 2
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: @ftc_inline2._Mfp16(
+// CHECK-LABEL: @ftc_dup2.default(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 2
+// CHECK-NEXT:    ret i32 3
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: @ftc_inline2._MfcmaMsve2-bitperm(
+// CHECK-LABEL: @ftc_inline2.default(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 2
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: @ftc_inline2.default(
+// CHECK-LABEL: @ftc_inline1._MrngMsimd(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 2
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: @ftc_inline1._MpredresMrcpc(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: @ftc_inline1._Msve2-aesMwfxt(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: @ftc_inline1.default(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 1
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
@@ -406,16 +406,16 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default"))
 //.
 // CHECK: attributes #[[ATTR0:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+lse,+neon" }
 // CHECK: attributes #[[ATTR1:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2" }
-// CHECK: attributes #[[ATTR2:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
-// CHECK: attributes #[[ATTR3:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+sha2" }
-// CHECK: attributes #[[ATTR4:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+mte,+neon,+sha2" }
-// CHECK: attributes #[[ATTR5:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" }
-// CHECK: attributes #[[ATTR6:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,+dotprod,+fp-armv8,+neon" }
-// CHECK: attributes #[[ATTR7:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rand" }
-// CHECK: attributes #[[ATTR8:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+predres,+rcpc" }
-// CHECK: attributes #[[ATTR9:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-aes,+wfxt" }
-// CHECK: attributes #[[ATTR10:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" }
-// CHECK: attributes #[[ATTR11:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+complxnum,+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-bitperm" }
+// CHECK: attributes #[[ATTR2:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+sha2" }
+// CHECK: attributes #[[ATTR3:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+mte,+neon,+sha2" }
+// CHECK: attributes #[[ATTR4:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" }
+// CHECK: attributes #[[ATTR5:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,+dotprod,+fp-armv8,+neon" }
+// CHECK: attributes #[[ATTR6:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+// CHECK: attributes #[[ATTR7:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" }
+// CHECK: attributes #[[ATTR8:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+complxnum,+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-bitperm" }
+// CHECK: attributes #[[ATTR9:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rand" }
+// CHECK: attributes #[[ATTR10:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+predres,+rcpc" }
+// CHECK: attributes #[[ATTR11:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-aes,+wfxt" }
 // CHECK: attributes #[[ATTR12:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti" }
 // CHECK: attributes #[[ATTR13:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sb,+sve" }
 //.
diff --git a/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp b/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp
index 14963867798d..7953f902bf09 100644
--- a/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp
+++ b/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp
@@ -56,13 +56,6 @@ void run_foo_tml() {
 // CHECK-NEXT:    ret i32 1
 //
 //
-// CHECK-LABEL: @_Z7foo_ovli.default(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[DOTADDR:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4
-// CHECK-NEXT:    ret i32 1
-//
-//
 // CHECK-LABEL: @_Z7foo_ovli.resolver(
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
@@ -82,11 +75,6 @@ void run_foo_tml() {
 // CHECK-NEXT:    ret i32 2
 //
 //
-// CHECK-LABEL: @_Z7foo_ovlv.default(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    ret i32 2
-//
-//
 // CHECK-LABEL: @_Z7foo_ovlv.resolver(
 // CHECK-NEXT:  resolver_entry:
 // CHECK-NEXT:    call void @__init_cpu_features_resolver()
@@ -182,6 +170,18 @@ void run_foo_tml() {
 // CHECK-NEXT:    ret i32 4
 //
 //
+// CHECK-LABEL: @_Z7foo_ovli.default(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[DOTADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4
+// CHECK-NEXT:    ret i32 1
+//
+//
+// CHECK-LABEL: @_Z7foo_ovlv.default(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret i32 2
+//
+//
 // CHECK-LABEL: @_ZN7MyClassIssE7foo_tmlEv._Mfrintts(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
@@ -231,8 +231,8 @@ void run_foo_tml() {
 //
 //.
 // CHECK: attributes #[[ATTR0:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" }
-// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
-// CHECK: attributes #[[ATTR2:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ls64" }
+// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ls64" }
+// CHECK: attributes #[[ATTR2:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
 // CHECK: attributes #[[ATTR3:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint" }
 // CHECK: attributes #[[ATTR4:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme-f64f64" }
 //.
diff --git a/clang/test/Sema/attr-target-clones-aarch64.c b/clang/test/Sema/attr-target-clones-aarch64.c
index 0ce277f41884..bc3fceab8282 100644
--- a/clang/test/Sema/attr-target-clones-aarch64.c
+++ b/clang/test/Sema/attr-target-clones-aarch64.c
@@ -27,7 +27,7 @@ int __attribute__((target_clones("rng", "fp16fml+fp", "default"))) redecl4(void)
 int __attribute__((target_clones("dgh+memtag+rpres+ls64_v", "ebf16+dpb+sha1", "default"))) redecl4(void) { return 1; }
 
 int __attribute__((target_version("flagm2"))) redef2(void) { return 1; }
-// expected-error@+2 {{multiversioning attributes cannot be combined}}
+// expected-error@+2 {{multiversioned function redeclarations require identical target attributes}}
 // expected-note@-2 {{previous declaration is here}}
 int __attribute__((target_clones("flagm2", "default"))) redef2(void) { return 1; }
 
-- 
GitLab


From f914e8e77c4703814e2f2dcef1d4569b17837c92 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen@arm.com>
Date: Tue, 26 Mar 2024 11:40:31 +0000
Subject: [PATCH 265/404] [AArch64][SME] Add coalescer barrier for args/results
 in locally streaming functions. (#85388)

Similar to how we protected FP/fixed-vector arguments and results from
calls, we should do the same for arguments/results from locally-streaming
functions such that those are not spilled/filled as ZPR registers.

This may cause a small regression (additional spills/fills), which is
addressed by #85386.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  21 ++-
 .../lib/Target/AArch64/AArch64SMEInstrInfo.td |   2 +-
 .../sme-avoid-coalescing-locally-streaming.ll | 120 ++++++++++++++++++
 ...ing-body-streaming-compatible-interface.ll |  26 ++--
 .../CodeGen/AArch64/sme-streaming-body.ll     |  34 +++--
 5 files changed, 168 insertions(+), 35 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/sme-avoid-coalescing-locally-streaming.ll

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 043a617898a6..f552f9192920 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -6895,6 +6895,11 @@ AArch64TargetLowering::allocateLazySaveBuffer(SDValue &Chain, const SDLoc &DL,
   return TPIDR2Obj;
 }
 
+static bool isPassedInFPR(EVT VT) {
+  return VT.isFixedLengthVector() ||
+         (VT.isFloatingPoint() && !VT.isScalableVector());
+}
+
 SDValue AArch64TargetLowering::LowerFormalArguments(
     SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
@@ -7031,6 +7036,13 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
         // This will be the new Chain/Root node.
         ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, RegVT, Glue);
         Glue = ArgValue.getValue(2);
+        if (isPassedInFPR(ArgValue.getValueType())) {
+          ArgValue =
+              DAG.getNode(AArch64ISD::COALESCER_BARRIER, DL,
+                          DAG.getVTList(ArgValue.getValueType(), MVT::Glue),
+                          {ArgValue, Glue});
+          Glue = ArgValue.getValue(1);
+        }
       } else
         ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, RegVT);
 
@@ -7402,11 +7414,6 @@ void AArch64TargetLowering::saveVarArgRegisters(CCState &CCInfo,
   }
 }
 
-static bool isPassedInFPR(EVT VT) {
-  return VT.isFixedLengthVector() ||
-         (VT.isFloatingPoint() && !VT.isScalableVector());
-}
-
 /// LowerCallResult - Lower the result values of a call into the
 /// appropriate copies out of appropriate physical registers.
 SDValue AArch64TargetLowering::LowerCallResult(
@@ -8632,6 +8639,10 @@ AArch64TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
 
   SmallVector<SDValue, 4> RetOps(1, Chain);
   for (auto &RetVal : RetVals) {
+    if (FuncAttrs.hasStreamingBody() && !FuncAttrs.hasStreamingInterface() &&
+        isPassedInFPR(RetVal.second.getValueType()))
+      RetVal.second = DAG.getNode(AArch64ISD::COALESCER_BARRIER, DL,
+                                  RetVal.second.getValueType(), RetVal.second);
     Chain = DAG.getCopyToReg(Chain, DL, RetVal.first, RetVal.second, Glue);
     Glue = Chain.getValue(1);
     RetOps.push_back(
diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index b65c67e70a4e..2db0fa253434 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -29,7 +29,7 @@ def AArch64_save_zt : SDNode<"AArch64ISD::SAVE_ZT", SDTypeProfile<0, 2,
                              [SDTCisInt<0>, SDTCisPtrTy<1>]>,
                              [SDNPHasChain, SDNPSideEffect, SDNPMayStore]>;
 def AArch64CoalescerBarrier
-    : SDNode<"AArch64ISD::COALESCER_BARRIER", SDTypeProfile<1, 1, []>, []>;
+    : SDNode<"AArch64ISD::COALESCER_BARRIER", SDTypeProfile<1, 1, []>, [SDNPOptInGlue, SDNPOutGlue]>;
 
 //===----------------------------------------------------------------------===//
 // Instruction naming conventions.
diff --git a/llvm/test/CodeGen/AArch64/sme-avoid-coalescing-locally-streaming.ll b/llvm/test/CodeGen/AArch64/sme-avoid-coalescing-locally-streaming.ll
new file mode 100644
index 000000000000..cd5046a9a647
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme-avoid-coalescing-locally-streaming.ll
@@ -0,0 +1,120 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mattr=+sme -stop-after=finalize-isel < %s | FileCheck %s --check-prefix=CHECK-COALESCER-BARRIER
+; RUN: llc -mattr=+sme -stop-after=virtregrewriter < %s | FileCheck %s --check-prefix=CHECK-REGALLOC
+
+target triple = "aarch64"
+
+define void @dont_coalesce_args(<2 x i64> %a) "aarch64_pstate_sm_body" nounwind {
+  ; CHECK-COALESCER-BARRIER-LABEL: name: dont_coalesce_args
+  ; CHECK-COALESCER-BARRIER: bb.0 (%ir-block.0):
+  ; CHECK-COALESCER-BARRIER-NEXT:   liveins: $q0
+  ; CHECK-COALESCER-BARRIER-NEXT: {{  $}}
+  ; CHECK-COALESCER-BARRIER-NEXT:   [[COPY:%[0-9]+]]:fpr128 = COPY $q0
+  ; CHECK-COALESCER-BARRIER-NEXT:   [[COALESCER_BARRIER_FPR128_:%[0-9]+]]:fpr128 = COALESCER_BARRIER_FPR128 [[COPY]]
+  ; CHECK-COALESCER-BARRIER-NEXT:   MSRpstatesvcrImm1 1, 1, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg
+  ; CHECK-COALESCER-BARRIER-NEXT:   [[DEF:%[0-9]+]]:zpr = IMPLICIT_DEF
+  ; CHECK-COALESCER-BARRIER-NEXT:   [[INSERT_SUBREG:%[0-9]+]]:zpr = INSERT_SUBREG [[DEF]], [[COALESCER_BARRIER_FPR128_]], %subreg.zsub
+  ; CHECK-COALESCER-BARRIER-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-COALESCER-BARRIER-NEXT:   $z0 = COPY [[INSERT_SUBREG]]
+  ; CHECK-COALESCER-BARRIER-NEXT:   BL @scalable_args, csr_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit $z0, implicit-def $sp
+  ; CHECK-COALESCER-BARRIER-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-COALESCER-BARRIER-NEXT:   MSRpstatesvcrImm1 1, 0, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg
+  ; CHECK-COALESCER-BARRIER-NEXT:   RET_ReallyLR
+  ;
+  ; CHECK-REGALLOC-LABEL: name: dont_coalesce_args
+  ; CHECK-REGALLOC: bb.0 (%ir-block.0):
+  ; CHECK-REGALLOC-NEXT:   liveins: $q0
+  ; CHECK-REGALLOC-NEXT: {{  $}}
+  ; CHECK-REGALLOC-NEXT:   renamable $q0 = COALESCER_BARRIER_FPR128 killed renamable $q0
+  ; CHECK-REGALLOC-NEXT:   STRQui killed renamable $q0, %stack.0, 0 :: (store (s128) into %stack.0)
+  ; CHECK-REGALLOC-NEXT:   MSRpstatesvcrImm1 1, 1, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg
+  ; CHECK-REGALLOC-NEXT:   renamable $q0 = LDRQui %stack.0, 0 :: (load (s128) from %stack.0)
+  ; CHECK-REGALLOC-NEXT:   renamable $q0 = KILL killed renamable $q0, implicit-def $z0
+  ; CHECK-REGALLOC-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-REGALLOC-NEXT:   BL @scalable_args, csr_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit $z0, implicit-def $sp
+  ; CHECK-REGALLOC-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-REGALLOC-NEXT:   MSRpstatesvcrImm1 1, 0, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg
+  ; CHECK-REGALLOC-NEXT:   RET_ReallyLR
+  %sa = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64> poison, <2 x i64> %a, i64 0)
+  call void @scalable_args(<vscale x 2 x i64> %sa)
+  ret void
+}
+
+define <2 x i64> @dont_coalesce_res() "aarch64_pstate_sm_body" nounwind {
+  ; CHECK-COALESCER-BARRIER-LABEL: name: dont_coalesce_res
+  ; CHECK-COALESCER-BARRIER: bb.0 (%ir-block.0):
+  ; CHECK-COALESCER-BARRIER-NEXT:   MSRpstatesvcrImm1 1, 1, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg
+  ; CHECK-COALESCER-BARRIER-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-COALESCER-BARRIER-NEXT:   BL @scalable_res, csr_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def $z0
+  ; CHECK-COALESCER-BARRIER-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-COALESCER-BARRIER-NEXT:   [[COPY:%[0-9]+]]:zpr = COPY $z0
+  ; CHECK-COALESCER-BARRIER-NEXT:   [[COPY1:%[0-9]+]]:fpr128 = COPY [[COPY]].zsub
+  ; CHECK-COALESCER-BARRIER-NEXT:   [[COALESCER_BARRIER_FPR128_:%[0-9]+]]:fpr128 = COALESCER_BARRIER_FPR128 [[COPY1]]
+  ; CHECK-COALESCER-BARRIER-NEXT:   MSRpstatesvcrImm1 1, 0, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit-def $q0, implicit $vg, implicit-def $vg
+  ; CHECK-COALESCER-BARRIER-NEXT:   $q0 = COPY [[COALESCER_BARRIER_FPR128_]]
+  ; CHECK-COALESCER-BARRIER-NEXT:   RET_ReallyLR implicit $q0
+  ;
+  ; CHECK-REGALLOC-LABEL: name: dont_coalesce_res
+  ; CHECK-REGALLOC: bb.0 (%ir-block.0):
+  ; CHECK-REGALLOC-NEXT:   MSRpstatesvcrImm1 1, 1, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg
+  ; CHECK-REGALLOC-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-REGALLOC-NEXT:   BL @scalable_res, csr_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def $z0
+  ; CHECK-REGALLOC-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-REGALLOC-NEXT:   renamable $q0 = KILL renamable $q0, implicit killed $z0
+  ; CHECK-REGALLOC-NEXT:   renamable $q0 = COALESCER_BARRIER_FPR128 killed renamable $q0
+  ; CHECK-REGALLOC-NEXT:   STRQui killed renamable $q0, %stack.0, 0 :: (store (s128) into %stack.0)
+  ; CHECK-REGALLOC-NEXT:   MSRpstatesvcrImm1 1, 0, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit-def dead $q0, implicit $vg, implicit-def $vg
+  ; CHECK-REGALLOC-NEXT:   $q0 = LDRQui %stack.0, 0 :: (load (s128) from %stack.0)
+  ; CHECK-REGALLOC-NEXT:   RET_ReallyLR implicit $q0
+  %sa = call <vscale x 2 x i64> @scalable_res()
+  %res = call <2 x i64> @llvm.vector.extract.v2i64.nxv2i64(<vscale x 2 x i64> %sa, i64 0)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @dont_coalesce_arg_that_is_also_res(<2 x i64> %a) "aarch64_pstate_sm_body" nounwind {
+  ; CHECK-COALESCER-BARRIER-LABEL: name: dont_coalesce_arg_that_is_also_res
+  ; CHECK-COALESCER-BARRIER: bb.0 (%ir-block.0):
+  ; CHECK-COALESCER-BARRIER-NEXT:   liveins: $q0
+  ; CHECK-COALESCER-BARRIER-NEXT: {{  $}}
+  ; CHECK-COALESCER-BARRIER-NEXT:   [[COPY:%[0-9]+]]:fpr128 = COPY $q0
+  ; CHECK-COALESCER-BARRIER-NEXT:   [[COALESCER_BARRIER_FPR128_:%[0-9]+]]:fpr128 = COALESCER_BARRIER_FPR128 [[COPY]]
+  ; CHECK-COALESCER-BARRIER-NEXT:   MSRpstatesvcrImm1 1, 1, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg
+  ; CHECK-COALESCER-BARRIER-NEXT:   [[DEF:%[0-9]+]]:zpr = IMPLICIT_DEF
+  ; CHECK-COALESCER-BARRIER-NEXT:   [[INSERT_SUBREG:%[0-9]+]]:zpr = INSERT_SUBREG [[DEF]], [[COALESCER_BARRIER_FPR128_]], %subreg.zsub
+  ; CHECK-COALESCER-BARRIER-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-COALESCER-BARRIER-NEXT:   $z0 = COPY [[INSERT_SUBREG]]
+  ; CHECK-COALESCER-BARRIER-NEXT:   BL @scalable_args, csr_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit $z0, implicit-def $sp
+  ; CHECK-COALESCER-BARRIER-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-COALESCER-BARRIER-NEXT:   [[COALESCER_BARRIER_FPR128_1:%[0-9]+]]:fpr128 = COALESCER_BARRIER_FPR128 [[COALESCER_BARRIER_FPR128_]]
+  ; CHECK-COALESCER-BARRIER-NEXT:   MSRpstatesvcrImm1 1, 0, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit-def $q0, implicit $vg, implicit-def $vg
+  ; CHECK-COALESCER-BARRIER-NEXT:   $q0 = COPY [[COALESCER_BARRIER_FPR128_1]]
+  ; CHECK-COALESCER-BARRIER-NEXT:   RET_ReallyLR implicit $q0
+  ;
+  ; CHECK-REGALLOC-LABEL: name: dont_coalesce_arg_that_is_also_res
+  ; CHECK-REGALLOC: bb.0 (%ir-block.0):
+  ; CHECK-REGALLOC-NEXT:   liveins: $q0
+  ; CHECK-REGALLOC-NEXT: {{  $}}
+  ; CHECK-REGALLOC-NEXT:   renamable $q0 = COALESCER_BARRIER_FPR128 killed renamable $q0
+  ; CHECK-REGALLOC-NEXT:   STRQui killed renamable $q0, %stack.0, 0 :: (store (s128) into %stack.0)
+  ; CHECK-REGALLOC-NEXT:   MSRpstatesvcrImm1 1, 1, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg
+  ; CHECK-REGALLOC-NEXT:   renamable $q0 = LDRQui %stack.0, 0 :: (load (s128) from %stack.0)
+  ; CHECK-REGALLOC-NEXT:   renamable $q0 = KILL killed renamable $q0, implicit-def $z0
+  ; CHECK-REGALLOC-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-REGALLOC-NEXT:   BL @scalable_args, csr_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit $z0, implicit-def $sp
+  ; CHECK-REGALLOC-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-REGALLOC-NEXT:   renamable $q0 = LDRQui %stack.0, 0 :: (load (s128) from %stack.0)
+  ; CHECK-REGALLOC-NEXT:   renamable $q0 = COALESCER_BARRIER_FPR128 killed renamable $q0
+  ; CHECK-REGALLOC-NEXT:   STRQui killed renamable $q0, %stack.0, 0 :: (store (s128) into %stack.0)
+  ; CHECK-REGALLOC-NEXT:   MSRpstatesvcrImm1 1, 0, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit-def dead $q0, implicit $vg, implicit-def $vg
+  ; CHECK-REGALLOC-NEXT:   $q0 = LDRQui %stack.0, 0 :: (load (s128) from %stack.0)
+  ; CHECK-REGALLOC-NEXT:   RET_ReallyLR implicit $q0
+  %sa = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64> poison, <2 x i64> %a, i64 0)
+  call void @scalable_args(<vscale x 2 x i64> %sa)
+  ret <2 x i64> %a
+}
+
+declare void @scalable_args(<vscale x 2 x i64>) "aarch64_pstate_sm_enabled"
+declare <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64>, <2 x i64>, i64)
+
+declare <vscale x 2 x i64> @scalable_res() "aarch64_pstate_sm_enabled"
+declare <2 x i64> @llvm.vector.extract.v2i64.nxv2i64(<vscale x 2 x i64>, i64)
diff --git a/llvm/test/CodeGen/AArch64/sme-streaming-body-streaming-compatible-interface.ll b/llvm/test/CodeGen/AArch64/sme-streaming-body-streaming-compatible-interface.ll
index d67573384ca9..6e262cc0786e 100644
--- a/llvm/test/CodeGen/AArch64/sme-streaming-body-streaming-compatible-interface.ll
+++ b/llvm/test/CodeGen/AArch64/sme-streaming-body-streaming-compatible-interface.ll
@@ -8,27 +8,31 @@ declare void @streaming_compatible_callee() "aarch64_pstate_sm_compatible";
 define float @sm_body_sm_compatible_simple() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" nounwind {
 ; CHECK-LABEL: sm_body_sm_compatible_simple:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #96
+; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
 ; CHECK-NEXT:    and x8, x0, #0x1
 ; CHECK-NEXT:    tbnz w8, #0, .LBB0_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    fmov s0, wzr
+; CHECK-NEXT:    str s0, [sp, #12] // 4-byte Folded Spill
 ; CHECK-NEXT:    tbnz w8, #0, .LBB0_4
 ; CHECK-NEXT:  // %bb.3:
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:  .LBB0_4:
-; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    fmov s0, wzr
-; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
-; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr s0, [sp, #12] // 4-byte Folded Reload
+; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   ret float zeroinitializer
 }
diff --git a/llvm/test/CodeGen/AArch64/sme-streaming-body.ll b/llvm/test/CodeGen/AArch64/sme-streaming-body.ll
index 93875549cffc..08dec228d2f7 100644
--- a/llvm/test/CodeGen/AArch64/sme-streaming-body.ll
+++ b/llvm/test/CodeGen/AArch64/sme-streaming-body.ll
@@ -87,29 +87,27 @@ if.end:
 define <2 x i64> @locally_streaming_caller_no_callee(<2 x i64> %a) "aarch64_pstate_sm_body" nounwind {
 ; CHECK-LABEL: locally_streaming_caller_no_callee:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
-; CHECK-NEXT:    addsvl sp, sp, #-1
-; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT:    str z0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #80
+; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    index z0.d, #0, #1
-; CHECK-NEXT:    ldr z1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
 ; CHECK-NEXT:    add z0.d, z0.d, z1.d
 ; CHECK-NEXT:    add z0.d, z0.d, #41 // =0x29
-; CHECK-NEXT:    str z0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    smstop sm
-; CHECK-NEXT:    ldr z0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
-; CHECK-NEXT:    addsvl sp, sp, #1
-; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
-; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    smstop sm
+; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #80
 ; CHECK-NEXT:    ret
 
   %add = add <2 x i64> %a, <i64 41, i64 42>;
-- 
GitLab


From bfb12ef33a90bd20c3a76094ef4393a8defbb712 Mon Sep 17 00:00:00 2001
From: Jacek Caban <jacek@codeweavers.com>
Date: Tue, 26 Mar 2024 13:07:13 +0100
Subject: [PATCH 266/404] [LLD][COFF] Allow additional attributes in forwarding
 exports. (#86535)

Testing with MSVC link.exe showed that it respects such options, while
LLD currently discards them.
---
 lld/COFF/DriverUtils.cpp  | 13 ++++++-------
 lld/test/COFF/export.test | 33 ++++++++++++++++++++++++++++++---
 2 files changed, 36 insertions(+), 10 deletions(-)

diff --git a/lld/COFF/DriverUtils.cpp b/lld/COFF/DriverUtils.cpp
index fc8eb327be49..0fa4769bab19 100644
--- a/lld/COFF/DriverUtils.cpp
+++ b/lld/COFF/DriverUtils.cpp
@@ -577,16 +577,15 @@ Export LinkerDriver::parseExport(StringRef arg) {
     if (y.contains(".")) {
       e.name = x;
       e.forwardTo = y;
-      return e;
+    } else {
+      e.extName = x;
+      e.name = y;
+      if (e.name.empty())
+        goto err;
     }
-
-    e.extName = x;
-    e.name = y;
-    if (e.name.empty())
-      goto err;
   }
 
-  // If "<name>=<internalname>[,@ordinal[,NONAME]][,DATA][,PRIVATE]"
+  // Optional parameters "[,@ordinal[,NONAME]][,DATA][,PRIVATE]"
   while (!rest.empty()) {
     StringRef tok;
     std::tie(tok, rest) = rest.split(",");
diff --git a/lld/test/COFF/export.test b/lld/test/COFF/export.test
index d340e0174b56..b91fdcfeebb0 100644
--- a/lld/test/COFF/export.test
+++ b/lld/test/COFF/export.test
@@ -76,18 +76,45 @@ SYMTAB: exportfn3 in export.test.tmp.DLL
 
 # RUN: lld-link /out:%t.dll /dll %t.obj /export:foo=kernel32.foobar
 # RUN: llvm-objdump -p %t.dll | FileCheck --check-prefix=FORWARDER %s
+# RUN: llvm-nm -M %t.lib | FileCheck --check-prefix=SYMTAB-FWD %s
 
 # RUN: echo "EXPORTS foo=kernel32.foobar" > %t.def
-# RUN: lld-link /out:%t.dll /dll %t.obj /def:%t.def
-# RUN: llvm-objdump -p %t.dll | FileCheck --check-prefix=FORWARDER %s
+# RUN: lld-link /out:%t-def.dll /dll %t.obj /def:%t.def
+# RUN: llvm-objdump -p %t-def.dll | FileCheck --check-prefix=FORWARDER %s
+# RUN: llvm-nm -M %t-def.lib | FileCheck --check-prefix=SYMTAB-FWD %s
 
 FORWARDER: Export Table:
-FORWARDER:  DLL name: export.test.tmp.dll
+FORWARDER:  DLL name: export.test.tmp
 FORWARDER:  Ordinal base: 1
 FORWARDER:  Ordinal      RVA  Name
 FORWARDER:        1   0x1010  exportfn
 FORWARDER:        2           foo (forwarded to kernel32.foobar)
 
+SYMTAB-FWD:      __imp_exportfn3 in export.test.tmp
+SYMTAB-FWD-NEXT: __imp_foo in export.test.tmp
+SYMTAB-FWD-NEXT: exportfn3 in export.test.tmp
+SYMTAB-FWD-NEXT: foo in export.test.tmp
+
+# RUN: lld-link /out:%t-fwd-priv.dll /dll %t.obj /export:foo=kernel32.foobar,DATA,PRIVATE
+# RUN: llvm-objdump -p %t-fwd-priv.dll | FileCheck --check-prefix=FORWARDER %s
+# RUN: llvm-nm -M %t-fwd-priv.lib | FileCheck --check-prefix=SYMTAB-FWD-PRIV %s
+
+SYMTAB-FWD-PRIV:      __imp_exportfn3 in export.test.tmp-fwd-priv
+SYMTAB-FWD-PRIV-NOT:  __imp_foo
+SYMTAB-FWD-PRIV-NEXT: exportfn3 in export.test.tmp-fwd-priv
+SYMTAB-FWD-PRIV-NOT:  foo
+
+# RUN: lld-link /out:%t-fwd-ord.dll /dll %t.obj /export:foo=kernel32.foobar,@3,NONAME
+# RUN: llvm-objdump -p %t-fwd-ord.dll | FileCheck --check-prefix=FORWARDER-ORD %s
+# RUN: llvm-nm -M %t-fwd-ord.lib | FileCheck --check-prefix=SYMTAB-FWD %s
+
+FORWARDER-ORD:      Export Table:
+FORWARDER-ORD-NEXT:  DLL name: export.test.tmp-fwd-ord
+FORWARDER-ORD-NEXT:  Ordinal base: 3
+FORWARDER-ORD-NEXT:  Ordinal      RVA  Name
+FORWARDER-ORD-NEXT:        3           (forwarded to kernel32.foobar)
+FORWARDER-ORD-NEXT:        4   0x1010  exportfn3
+
 # RUN: lld-link /out:%t.dll /dll %t.obj /merge:.rdata=.text /export:exportfn1 /export:exportfn2
 # RUN: llvm-objdump -p %t.dll | FileCheck --check-prefix=MERGE --match-full-lines %s
 
-- 
GitLab


From 4dc322524871df02578a05b260dacf28ff130c02 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Tue, 26 Mar 2024 07:13:59 -0500
Subject: [PATCH 267/404] [Libomptarget] Replace global `PluginTy::get`
 interface with references (#86595)

Summary:
We have a plugin singleton that implements the Plugin interface. This
then spawns separate device and kernels. Previously when these needed to
reach into the global singleton they would use the `PluginTy::get`
routine to get access to it. In the future we will move away from this
as the lifetime of the plugin will be handled by `libomptarget`
directly. This patch removes uses of this inside of the plugin
implementaion themselves by simply keeping a reference to the plugin
inside of the device.

The external `__tgt_rtl` functions still use the global method, but will
be removed later.
---
 .../plugins-nextgen/amdgpu/src/rtl.cpp        | 43 +++++++++++--------
 .../common/include/PluginInterface.h          |  8 +++-
 .../common/src/PluginInterface.cpp            |  9 ++--
 .../plugins-nextgen/cuda/src/rtl.cpp          | 14 +++---
 .../plugins-nextgen/host/src/rtl.cpp          | 18 ++++----
 5 files changed, 53 insertions(+), 39 deletions(-)

diff --git a/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp b/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp
index 14461f14dd67..2dd08dd5d0b4 100644
--- a/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -371,7 +371,8 @@ private:
 struct AMDGPUMemoryManagerTy : public DeviceAllocatorTy {
 
   /// Create an empty memory manager.
-  AMDGPUMemoryManagerTy() : MemoryPool(nullptr), MemoryManager(nullptr) {}
+  AMDGPUMemoryManagerTy(AMDGPUPluginTy &Plugin)
+      : Plugin(Plugin), MemoryPool(nullptr), MemoryManager(nullptr) {}
 
   /// Initialize the memory manager from a memory pool.
   Error init(AMDGPUMemoryPoolTy &MemoryPool) {
@@ -429,6 +430,9 @@ private:
     return OFFLOAD_SUCCESS;
   }
 
+  /// The underlying plugin that owns this memory manager.
+  AMDGPUPluginTy &Plugin;
+
   /// The memory pool used to allocate memory.
   AMDGPUMemoryPoolTy *MemoryPool;
 
@@ -1744,9 +1748,10 @@ protected:
 /// HSA host agent. We aggregate all its resources into the same instance.
 struct AMDHostDeviceTy : public AMDGenericDeviceTy {
   /// Create a host device from an array of host agents.
-  AMDHostDeviceTy(const llvm::SmallVector<hsa_agent_t> &HostAgents)
-      : AMDGenericDeviceTy(), Agents(HostAgents), ArgsMemoryManager(),
-        PinnedMemoryManager() {
+  AMDHostDeviceTy(AMDGPUPluginTy &Plugin,
+                  const llvm::SmallVector<hsa_agent_t> &HostAgents)
+      : AMDGenericDeviceTy(), Agents(HostAgents), ArgsMemoryManager(Plugin),
+        PinnedMemoryManager(Plugin) {
     assert(HostAgents.size() && "No host agent found");
   }
 
@@ -1840,9 +1845,10 @@ private:
 /// generic device class.
 struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   // Create an AMDGPU device with a device id and default AMDGPU grid values.
-  AMDGPUDeviceTy(int32_t DeviceId, int32_t NumDevices,
+  AMDGPUDeviceTy(GenericPluginTy &Plugin, int32_t DeviceId, int32_t NumDevices,
                  AMDHostDeviceTy &HostDevice, hsa_agent_t Agent)
-      : GenericDeviceTy(DeviceId, NumDevices, {0}), AMDGenericDeviceTy(),
+      : GenericDeviceTy(Plugin, DeviceId, NumDevices, {0}),
+        AMDGenericDeviceTy(),
         OMPX_NumQueues("LIBOMPTARGET_AMDGPU_NUM_HSA_QUEUES", 4),
         OMPX_QueueSize("LIBOMPTARGET_AMDGPU_HSA_QUEUE_SIZE", 512),
         OMPX_DefaultTeamsPerCU("LIBOMPTARGET_AMDGPU_TEAMS_PER_CU", 4),
@@ -2088,7 +2094,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   /// Allocate and construct an AMDGPU kernel.
   Expected<GenericKernelTy &> constructKernel(const char *Name) override {
     // Allocate and construct the AMDGPU kernel.
-    AMDGPUKernelTy *AMDGPUKernel = PluginTy::get().allocate<AMDGPUKernelTy>();
+    AMDGPUKernelTy *AMDGPUKernel = Plugin.allocate<AMDGPUKernelTy>();
     if (!AMDGPUKernel)
       return Plugin::error("Failed to allocate memory for AMDGPU kernel");
 
@@ -2138,8 +2144,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   Expected<DeviceImageTy *> loadBinaryImpl(const __tgt_device_image *TgtImage,
                                            int32_t ImageId) override {
     // Allocate and initialize the image object.
-    AMDGPUDeviceImageTy *AMDImage =
-        PluginTy::get().allocate<AMDGPUDeviceImageTy>();
+    AMDGPUDeviceImageTy *AMDImage = Plugin.allocate<AMDGPUDeviceImageTy>();
     new (AMDImage) AMDGPUDeviceImageTy(ImageId, *this, TgtImage);
 
     // Load the HSA executable.
@@ -2697,7 +2702,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   }
   Error setDeviceHeapSize(uint64_t Value) override {
     for (DeviceImageTy *Image : LoadedImages)
-      if (auto Err = setupDeviceMemoryPool(PluginTy::get(), *Image, Value))
+      if (auto Err = setupDeviceMemoryPool(Plugin, *Image, Value))
         return Err;
     DeviceMemoryPoolSize = Value;
     return Plugin::success();
@@ -2737,7 +2742,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     return utils::iterateAgentMemoryPools(
         Agent, [&](hsa_amd_memory_pool_t HSAMemoryPool) {
           AMDGPUMemoryPoolTy *MemoryPool =
-              PluginTy::get().allocate<AMDGPUMemoryPoolTy>();
+              Plugin.allocate<AMDGPUMemoryPoolTy>();
           new (MemoryPool) AMDGPUMemoryPoolTy(HSAMemoryPool);
           AllMemoryPools.push_back(MemoryPool);
           return HSA_STATUS_SUCCESS;
@@ -3090,7 +3095,7 @@ struct AMDGPUPluginTy final : public GenericPluginTy {
 
     // Initialize the host device using host agents.
     HostDevice = allocate<AMDHostDeviceTy>();
-    new (HostDevice) AMDHostDeviceTy(HostAgents);
+    new (HostDevice) AMDHostDeviceTy(*this, HostAgents);
 
     // Setup the memory pools of available for the host.
     if (auto Err = HostDevice->init())
@@ -3116,8 +3121,9 @@ struct AMDGPUPluginTy final : public GenericPluginTy {
   }
 
   /// Creates an AMDGPU device.
-  GenericDeviceTy *createDevice(int32_t DeviceId, int32_t NumDevices) override {
-    return new AMDGPUDeviceTy(DeviceId, NumDevices, getHostDevice(),
+  GenericDeviceTy *createDevice(GenericPluginTy &Plugin, int32_t DeviceId,
+                                int32_t NumDevices) override {
+    return new AMDGPUDeviceTy(Plugin, DeviceId, NumDevices, getHostDevice(),
                               getKernelAgent(DeviceId));
   }
 
@@ -3248,7 +3254,9 @@ Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
   // 56 bytes per allocation.
   uint32_t AllArgsSize = KernelArgsSize + ImplicitArgsSize;
 
-  AMDHostDeviceTy &HostDevice = PluginTy::get<AMDGPUPluginTy>().getHostDevice();
+  AMDGPUPluginTy &AMDGPUPlugin =
+      static_cast<AMDGPUPluginTy &>(GenericDevice.Plugin);
+  AMDHostDeviceTy &HostDevice = AMDGPUPlugin.getHostDevice();
   AMDGPUMemoryManagerTy &ArgsMemoryManager = HostDevice.getArgsMemoryManager();
 
   void *AllArgs = nullptr;
@@ -3385,7 +3393,7 @@ void *AMDGPUMemoryManagerTy::allocate(size_t Size, void *HstPtr,
   }
   assert(Ptr && "Invalid pointer");
 
-  auto &KernelAgents = PluginTy::get<AMDGPUPluginTy>().getKernelAgents();
+  auto &KernelAgents = Plugin.getKernelAgents();
 
   // Allow all kernel agents to access the allocation.
   if (auto Err = MemoryPool->enableAccess(Ptr, Size, KernelAgents)) {
@@ -3428,7 +3436,8 @@ void *AMDGPUDeviceTy::allocate(size_t Size, void *, TargetAllocTy Kind) {
   }
 
   if (Alloc) {
-    auto &KernelAgents = PluginTy::get<AMDGPUPluginTy>().getKernelAgents();
+    auto &KernelAgents =
+        static_cast<AMDGPUPluginTy &>(Plugin).getKernelAgents();
     // Inherently necessary for host or shared allocations
     // Also enabled for device memory to allow device to device memcpy
 
diff --git a/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h b/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h
index 1440f8f678d8..b16b081e5eff 100644
--- a/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h
+++ b/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h
@@ -610,7 +610,7 @@ public:
 struct GenericDeviceTy : public DeviceAllocatorTy {
   /// Construct a device with its device id within the plugin, the number of
   /// devices in the plugin and the grid values for that kind of device.
-  GenericDeviceTy(int32_t DeviceId, int32_t NumDevices,
+  GenericDeviceTy(GenericPluginTy &Plugin, int32_t DeviceId, int32_t NumDevices,
                   const llvm::omp::GV &GridValues);
 
   /// Get the device identifier within the corresponding plugin. Notice that
@@ -860,6 +860,9 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   /// Allocate and construct a kernel object.
   virtual Expected<GenericKernelTy &> constructKernel(const char *Name) = 0;
 
+  /// Reference to the underlying plugin that created this device.
+  GenericPluginTy &Plugin;
+
 private:
   /// Get and set the stack size and heap size for the device. If not used, the
   /// plugin can implement the setters as no-op and setting the output
@@ -977,7 +980,8 @@ struct GenericPluginTy {
   virtual Error deinitImpl() = 0;
 
   /// Create a new device for the underlying plugin.
-  virtual GenericDeviceTy *createDevice(int32_t DeviceID,
+  virtual GenericDeviceTy *createDevice(GenericPluginTy &Plugin,
+                                        int32_t DeviceID,
                                         int32_t NumDevices) = 0;
 
   /// Create a new global handler for the underlying plugin.
diff --git a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp
index 4db787a2ea91..c32bd089d8a9 100644
--- a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp
@@ -438,7 +438,7 @@ Error GenericKernelTy::init(GenericDeviceTy &GenericDevice,
   // Retrieve kernel environment object for the kernel.
   GlobalTy KernelEnv(std::string(Name) + "_kernel_environment",
                      sizeof(KernelEnvironment), &KernelEnvironment);
-  GenericGlobalHandlerTy &GHandler = PluginTy::get().getGlobalHandler();
+  GenericGlobalHandlerTy &GHandler = GenericDevice.Plugin.getGlobalHandler();
   if (auto Err =
           GHandler.readGlobalFromImage(GenericDevice, *ImagePtr, KernelEnv)) {
     [[maybe_unused]] std::string ErrStr = toString(std::move(Err));
@@ -710,9 +710,10 @@ uint64_t GenericKernelTy::getNumBlocks(GenericDeviceTy &GenericDevice,
   return std::min(PreferredNumBlocks, GenericDevice.getBlockLimit());
 }
 
-GenericDeviceTy::GenericDeviceTy(int32_t DeviceId, int32_t NumDevices,
+GenericDeviceTy::GenericDeviceTy(GenericPluginTy &Plugin, int32_t DeviceId,
+                                 int32_t NumDevices,
                                  const llvm::omp::GV &OMPGridValues)
-    : MemoryManager(nullptr), OMP_TeamLimit("OMP_TEAM_LIMIT"),
+    : Plugin(Plugin), MemoryManager(nullptr), OMP_TeamLimit("OMP_TEAM_LIMIT"),
       OMP_NumTeams("OMP_NUM_TEAMS"),
       OMP_TeamsThreadLimit("OMP_TEAMS_THREAD_LIMIT"),
       OMPX_DebugKind("LIBOMPTARGET_DEVICE_RTL_DEBUG"),
@@ -1522,7 +1523,7 @@ Error GenericPluginTy::initDevice(int32_t DeviceId) {
   assert(!Devices[DeviceId] && "Device already initialized");
 
   // Create the device and save the reference.
-  GenericDeviceTy *Device = createDevice(DeviceId, NumDevices);
+  GenericDeviceTy *Device = createDevice(*this, DeviceId, NumDevices);
   assert(Device && "Invalid device");
 
   // Save the device reference into the list.
diff --git a/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp b/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp
index d43c723c3502..fc74c6aa23fd 100644
--- a/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp
+++ b/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp
@@ -255,8 +255,8 @@ private:
 /// generic device class.
 struct CUDADeviceTy : public GenericDeviceTy {
   // Create a CUDA device with a device id and the default CUDA grid values.
-  CUDADeviceTy(int32_t DeviceId, int32_t NumDevices)
-      : GenericDeviceTy(DeviceId, NumDevices, NVPTXGridValues),
+  CUDADeviceTy(GenericPluginTy &Plugin, int32_t DeviceId, int32_t NumDevices)
+      : GenericDeviceTy(Plugin, DeviceId, NumDevices, NVPTXGridValues),
         CUDAStreamManager(*this), CUDAEventManager(*this) {}
 
   ~CUDADeviceTy() {}
@@ -471,7 +471,7 @@ struct CUDADeviceTy : public GenericDeviceTy {
   /// Allocate and construct a CUDA kernel.
   Expected<GenericKernelTy &> constructKernel(const char *Name) override {
     // Allocate and construct the CUDA kernel.
-    CUDAKernelTy *CUDAKernel = PluginTy::get().allocate<CUDAKernelTy>();
+    CUDAKernelTy *CUDAKernel = Plugin.allocate<CUDAKernelTy>();
     if (!CUDAKernel)
       return Plugin::error("Failed to allocate memory for CUDA kernel");
 
@@ -529,8 +529,7 @@ struct CUDADeviceTy : public GenericDeviceTy {
       return std::move(Err);
 
     // Allocate and initialize the image object.
-    CUDADeviceImageTy *CUDAImage =
-        PluginTy::get().allocate<CUDADeviceImageTy>();
+    CUDADeviceImageTy *CUDAImage = Plugin.allocate<CUDADeviceImageTy>();
     new (CUDAImage) CUDADeviceImageTy(ImageId, *this, TgtImage);
 
     // Load the CUDA module.
@@ -1373,8 +1372,9 @@ struct CUDAPluginTy final : public GenericPluginTy {
   Error deinitImpl() override { return Plugin::success(); }
 
   /// Creates a CUDA device to use for offloading.
-  GenericDeviceTy *createDevice(int32_t DeviceId, int32_t NumDevices) override {
-    return new CUDADeviceTy(DeviceId, NumDevices);
+  GenericDeviceTy *createDevice(GenericPluginTy &Plugin, int32_t DeviceId,
+                                int32_t NumDevices) override {
+    return new CUDADeviceTy(Plugin, DeviceId, NumDevices);
   }
 
   /// Creates a CUDA global handler.
diff --git a/openmp/libomptarget/plugins-nextgen/host/src/rtl.cpp b/openmp/libomptarget/plugins-nextgen/host/src/rtl.cpp
index ea8ed8d6a856..f0ce24249301 100644
--- a/openmp/libomptarget/plugins-nextgen/host/src/rtl.cpp
+++ b/openmp/libomptarget/plugins-nextgen/host/src/rtl.cpp
@@ -66,7 +66,7 @@ struct GenELF64KernelTy : public GenericKernelTy {
     GlobalTy Global(getName(), 0);
 
     // Get the metadata (address) of the kernel function.
-    GenericGlobalHandlerTy &GHandler = PluginTy::get().getGlobalHandler();
+    GenericGlobalHandlerTy &GHandler = Device.Plugin.getGlobalHandler();
     if (auto Err = GHandler.getGlobalMetadataFromDevice(Device, Image, Global))
       return Err;
 
@@ -132,8 +132,9 @@ private:
 /// Class implementing the device functionalities for GenELF64.
 struct GenELF64DeviceTy : public GenericDeviceTy {
   /// Create the device with a specific id.
-  GenELF64DeviceTy(int32_t DeviceId, int32_t NumDevices)
-      : GenericDeviceTy(DeviceId, NumDevices, GenELF64GridValues) {}
+  GenELF64DeviceTy(GenericPluginTy &Plugin, int32_t DeviceId,
+                   int32_t NumDevices)
+      : GenericDeviceTy(Plugin, DeviceId, NumDevices, GenELF64GridValues) {}
 
   ~GenELF64DeviceTy() {}
 
@@ -149,8 +150,7 @@ struct GenELF64DeviceTy : public GenericDeviceTy {
   /// Construct the kernel for a specific image on the device.
   Expected<GenericKernelTy &> constructKernel(const char *Name) override {
     // Allocate and construct the kernel.
-    GenELF64KernelTy *GenELF64Kernel =
-        PluginTy::get().allocate<GenELF64KernelTy>();
+    GenELF64KernelTy *GenELF64Kernel = Plugin.allocate<GenELF64KernelTy>();
     if (!GenELF64Kernel)
       return Plugin::error("Failed to allocate memory for GenELF64 kernel");
 
@@ -166,8 +166,7 @@ struct GenELF64DeviceTy : public GenericDeviceTy {
   Expected<DeviceImageTy *> loadBinaryImpl(const __tgt_device_image *TgtImage,
                                            int32_t ImageId) override {
     // Allocate and initialize the image object.
-    GenELF64DeviceImageTy *Image =
-        PluginTy::get().allocate<GenELF64DeviceImageTy>();
+    GenELF64DeviceImageTy *Image = Plugin.allocate<GenELF64DeviceImageTy>();
     new (Image) GenELF64DeviceImageTy(ImageId, *this, TgtImage);
 
     // Create a temporary file.
@@ -400,8 +399,9 @@ struct GenELF64PluginTy final : public GenericPluginTy {
   Error deinitImpl() override { return Plugin::success(); }
 
   /// Creates a generic ELF device.
-  GenericDeviceTy *createDevice(int32_t DeviceId, int32_t NumDevices) override {
-    return new GenELF64DeviceTy(DeviceId, NumDevices);
+  GenericDeviceTy *createDevice(GenericPluginTy &Plugin, int32_t DeviceId,
+                                int32_t NumDevices) override {
+    return new GenELF64DeviceTy(Plugin, DeviceId, NumDevices);
   }
 
   /// Creates a generic global handler.
-- 
GitLab


From 603db7425ffa96915854f425b027cc8403ab333d Mon Sep 17 00:00:00 2001
From: Jacek Caban <jacek@codeweavers.com>
Date: Tue, 26 Mar 2024 13:18:09 +0100
Subject: [PATCH 268/404] [LLD][COFF] Preserve all attributes from forwarding
 exports from parsed .def files. (#86564)

It's similar to #86535, but for export specified in .def files.
---
 lld/COFF/Driver.cpp       | 15 +++++++--------
 lld/test/COFF/export.test | 10 ++++++++++
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index 1b075389325a..181492913c0d 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -1032,19 +1032,18 @@ void LinkerDriver::parseModuleDefs(StringRef path) {
 
   for (COFFShortExport e1 : m.Exports) {
     Export e2;
-    // In simple cases, only Name is set. Renamed exports are parsed
-    // and set as "ExtName = Name". If Name has the form "OtherDll.Func",
-    // it shouldn't be a normal exported function but a forward to another
-    // DLL instead. This is supported by both MS and GNU linkers.
+    // Renamed exports are parsed and set as "ExtName = Name". If Name has
+    // the form "OtherDll.Func", it shouldn't be a normal exported
+    // function but a forward to another DLL instead. This is supported
+    // by both MS and GNU linkers.
     if (!e1.ExtName.empty() && e1.ExtName != e1.Name &&
         StringRef(e1.Name).contains('.')) {
       e2.name = saver().save(e1.ExtName);
       e2.forwardTo = saver().save(e1.Name);
-      ctx.config.exports.push_back(e2);
-      continue;
+    } else {
+      e2.name = saver().save(e1.Name);
+      e2.extName = saver().save(e1.ExtName);
     }
-    e2.name = saver().save(e1.Name);
-    e2.extName = saver().save(e1.ExtName);
     e2.aliasTarget = saver().save(e1.AliasTarget);
     e2.ordinal = e1.Ordinal;
     e2.noname = e1.Noname;
diff --git a/lld/test/COFF/export.test b/lld/test/COFF/export.test
index b91fdcfeebb0..041b328dec9c 100644
--- a/lld/test/COFF/export.test
+++ b/lld/test/COFF/export.test
@@ -104,6 +104,11 @@ SYMTAB-FWD-PRIV-NOT:  __imp_foo
 SYMTAB-FWD-PRIV-NEXT: exportfn3 in export.test.tmp-fwd-priv
 SYMTAB-FWD-PRIV-NOT:  foo
 
+# RUN: echo "EXPORTS foo=kernel32.foobar DATA PRIVATE" > %t-fwd-priv.def
+# RUN: lld-link /out:%t-fwd-priv-def.dll /dll %t.obj /def:%t-fwd-priv.def
+# RUN: llvm-objdump -p %t-fwd-priv-def.dll | FileCheck --check-prefix=FORWARDER %s
+# RUN: llvm-nm -M %t-fwd-priv-def.lib | FileCheck --check-prefix=SYMTAB-FWD-PRIV %s
+
 # RUN: lld-link /out:%t-fwd-ord.dll /dll %t.obj /export:foo=kernel32.foobar,@3,NONAME
 # RUN: llvm-objdump -p %t-fwd-ord.dll | FileCheck --check-prefix=FORWARDER-ORD %s
 # RUN: llvm-nm -M %t-fwd-ord.lib | FileCheck --check-prefix=SYMTAB-FWD %s
@@ -115,6 +120,11 @@ FORWARDER-ORD-NEXT:  Ordinal      RVA  Name
 FORWARDER-ORD-NEXT:        3           (forwarded to kernel32.foobar)
 FORWARDER-ORD-NEXT:        4   0x1010  exportfn3
 
+# RUN: echo "EXPORTS foo=kernel32.foobar @3 NONAME" > %t-fwd-ord.def
+# RUN: lld-link /out:%t-fwd-ord-def.dll /dll %t.obj /def:%t-fwd-ord.def
+# RUN: llvm-objdump -p %t-fwd-ord-def.dll | FileCheck --check-prefix=FORWARDER-ORD %s
+# RUN: llvm-nm -M %t-fwd-ord-def.lib | FileCheck --check-prefix=SYMTAB-FWD %s
+
 # RUN: lld-link /out:%t.dll /dll %t.obj /merge:.rdata=.text /export:exportfn1 /export:exportfn2
 # RUN: llvm-objdump -p %t.dll | FileCheck --check-prefix=MERGE --match-full-lines %s
 
-- 
GitLab


From 19ca79e8671439338b429982c457304c70eca701 Mon Sep 17 00:00:00 2001
From: Shourya Goel <shouryagoel10000@gmail.com>
Date: Tue, 26 Mar 2024 17:58:22 +0530
Subject: [PATCH 269/404] [libc][math][c23] Implement canonicalize functions
 (#85940)

Fixes: #85286
---
 libc/config/linux/x86_64/entrypoints.txt      |   4 +
 libc/docs/math/index.rst                      | 543 +++++++++---------
 libc/spec/stdc.td                             |   5 +
 libc/src/__support/FPUtil/BasicOperations.h   |  63 ++
 libc/src/__support/FPUtil/CMakeLists.txt      |   3 +
 libc/src/math/CMakeLists.txt                  |   5 +
 libc/src/math/canonicalize.h                  |  18 +
 libc/src/math/canonicalizef.h                 |  18 +
 libc/src/math/canonicalizef128.h              |  20 +
 libc/src/math/canonicalizel.h                 |  18 +
 libc/src/math/generic/CMakeLists.txt          |  48 ++
 libc/src/math/generic/canonicalize.cpp        |  19 +
 libc/src/math/generic/canonicalizef.cpp       |  19 +
 libc/src/math/generic/canonicalizef128.cpp    |  19 +
 libc/src/math/generic/canonicalizel.cpp       |  20 +
 libc/test/src/math/smoke/CMakeLists.txt       |  60 ++
 libc/test/src/math/smoke/CanonicalizeTest.h   | 219 +++++++
 .../test/src/math/smoke/canonicalize_test.cpp |  13 +
 .../src/math/smoke/canonicalizef128_test.cpp  |  13 +
 .../src/math/smoke/canonicalizef_test.cpp     |  13 +
 .../src/math/smoke/canonicalizel_test.cpp     |  19 +
 .../llvm-project-overlay/libc/BUILD.bazel     |   3 +
 22 files changed, 895 insertions(+), 267 deletions(-)
 create mode 100644 libc/src/math/canonicalize.h
 create mode 100644 libc/src/math/canonicalizef.h
 create mode 100644 libc/src/math/canonicalizef128.h
 create mode 100644 libc/src/math/canonicalizel.h
 create mode 100644 libc/src/math/generic/canonicalize.cpp
 create mode 100644 libc/src/math/generic/canonicalizef.cpp
 create mode 100644 libc/src/math/generic/canonicalizef128.cpp
 create mode 100644 libc/src/math/generic/canonicalizel.cpp
 create mode 100644 libc/test/src/math/smoke/CanonicalizeTest.h
 create mode 100644 libc/test/src/math/smoke/canonicalize_test.cpp
 create mode 100644 libc/test/src/math/smoke/canonicalizef128_test.cpp
 create mode 100644 libc/test/src/math/smoke/canonicalizef_test.cpp
 create mode 100644 libc/test/src/math/smoke/canonicalizel_test.cpp

diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 29080df54922..5b428e51aee6 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -350,6 +350,9 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.asinhf
     libc.src.math.atanf
     libc.src.math.atanhf
+    libc.src.math.canonicalize
+    libc.src.math.canonicalizef
+    libc.src.math.canonicalizel
     libc.src.math.copysign
     libc.src.math.copysignf
     libc.src.math.copysignl
@@ -516,6 +519,7 @@ set(TARGET_LIBM_ENTRYPOINTS
 if(LIBC_TYPES_HAS_FLOAT128)
   list(APPEND TARGET_LIBM_ENTRYPOINTS
     # math.h C23 _Float128 entrypoints
+    libc.src.math.canonicalizef128
     libc.src.math.ceilf128
     libc.src.math.copysignf128
     libc.src.math.fabsf128
diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst
index fa604faaec2f..080b6a4427f5 100644
--- a/libc/docs/math/index.rst
+++ b/libc/docs/math/index.rst
@@ -104,273 +104,282 @@ Implementation Status
 Basic Operations
 ----------------
 
-+--------------+---------------------------------------+-------------------+-------------------+-------------------+-------------------+
-| <Func>       |  Linux                                | Windows           | MacOS             | Embedded          | GPU               |
-|              +---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-|              | x86_64  | aarch64 | aarch32 | riscv64 | x86_64  | aarch64 | x86_64  | aarch64 | aarch32 | riscv32 | AMD     | nVidia  |
-+==============+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+
-| ceil         | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ceilf        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ceill        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ceilf128     | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| copysign     | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| copysignf    | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| copysignl    | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| copysignf128 | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fabs         | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fabsf        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fabsl        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fabsf128     | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fdim         | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fdimf        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fdiml        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fdimf128     | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| floor        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| floorf       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| floorl       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| floorf128    | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fmax         | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fmaxf        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fmaxf128     | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fmaxl        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fmin         | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fminf        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fminf128     | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fminl        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fmod         | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fmodf        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fmodl        | |check| | |check| |         | |check| | |check| |         |         | |check| |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fmodf128     | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| frexp        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| frexpf       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| frexpl       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| frexpf128    | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fromfp       | |check| |         |         |         |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fromfpf      | |check| |         |         |         |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fromfpl      | |check| |         |         |         |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fromfpf128   | |check| |         |         |         |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fromfpx      | |check| |         |         |         |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fromfpxf     | |check| |         |         |         |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fromfpxl     | |check| |         |         |         |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| fromfpxf128  | |check| |         |         |         |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ilogb        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ilogbf       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ilogbl       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ilogf128     | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ldexp        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ldexpf       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ldexpl       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ldexpf128    | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| llogb        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| llogbf       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| llogbl       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| llogf128     | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| llrint       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| llrintf      | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| llrintl      | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| llrintf128   | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| llround      | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| llroundf     | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| llroundl     | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| llroundf128  | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| logb         | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| logbf        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| logbl        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| logf128      | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| lrint        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| lrintf       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| lrintl       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| lrintf128    | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| lround       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| lroundf      | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| lroundl      | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| lroundf128   | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| modf         | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| modff        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| modfl        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| modff128     | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nan          | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nanf         | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nanl         | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nanf128      | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nearbyint    | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nearbyintf   | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nearbyintl   | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nextafter    | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nextafterf   | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nextafterl   | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nextafterf128| |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nextdown     | |check| | |check| | |check| | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nextdownf    | |check| | |check| | |check| | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nextdownl    | |check| | |check| | |check| | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nextdownf128 | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nexttoward   | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nexttowardf  | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nexttowardl  | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nextup       | |check| | |check| | |check| | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nextupf      | |check| | |check| | |check| | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nextupl      | |check| | |check| | |check| | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| nextupf128   | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| remainder    | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| remainderf   | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| remainderl   | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| remquo       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| remquof      | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| remquol      | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| rint         | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| rintf        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| rintl        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| rintf128     | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| round        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| roundf       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| roundl       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| roundf128    | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| scalbn       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| scalbnf      | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| scalbnl      | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| trunc        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| truncf       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| truncl       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| truncf128    | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ufromfp      | |check| |         |         |         |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ufromfpf     | |check| |         |         |         |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ufromfpl     | |check| |         |         |         |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ufromfpf128  | |check| |         |         |         |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ufromfpx     | |check| |         |         |         |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ufromfpxf    | |check| |         |         |         |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ufromfpxl    | |check| |         |         |         |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
-| ufromfpxf128 | |check| |         |         |         |         |         |         |         |         |         |         |         |
-+--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+
++------------------+---------------------------------------+-------------------+-------------------+-------------------+-------------------+
+| <Func>           |  Linux                                | Windows           | MacOS             | Embedded          | GPU               |
+|                  +---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+|                  | x86_64  | aarch64 | aarch32 | riscv64 | x86_64  | aarch64 | x86_64  | aarch64 | aarch32 | riscv32 | AMD     | nVidia  |
++==================+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+
+| ceil             | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ceilf            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ceill            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ceilf128         | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+|  canoninicalize  | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| canoninicalizef  | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| canoninicalizel  | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+|canoninicalizef128| |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| copysign         | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| copysignf        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| copysignl        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| copysignf128     | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fabs             | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fabsf            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fabsl            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fabsf128         | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fdim             | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fdimf            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fdiml            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fdimf128         | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| floor            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| floorf           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| floorl           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| floorf128        | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fmax             | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fmaxf            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fmaxf128         | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fmaxl            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fmin             | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fminf            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fminf128         | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fminl            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fmod             | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fmodf            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fmodl            | |check| | |check| |         | |check| | |check| |         |         | |check| |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fmodf128         | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| frexp            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| frexpf           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| frexpl           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| frexpf128        | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fromfp           | |check| |         |         |         |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fromfpf          | |check| |         |         |         |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fromfpl          | |check| |         |         |         |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fromfpf128       | |check| |         |         |         |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fromfpx          | |check| |         |         |         |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fromfpxf         | |check| |         |         |         |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fromfpxl         | |check| |         |         |         |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| fromfpxf128      | |check| |         |         |         |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ilogb            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ilogbf           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ilogbl           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ilogf128         | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ldexp            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ldexpf           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ldexpl           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ldexpf128        | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| llogb            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| llogbf           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| llogbl           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| llogf128         | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| llrint           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| llrintf          | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| llrintl          | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| llrintf128       | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| llround          | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| llroundf         | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| llroundl         | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| llroundf128      | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| logb             | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| logbf            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| logbl            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| logf128          | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| lrint            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| lrintf           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| lrintl           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| lrintf128        | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| lround           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| lroundf          | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| lroundl          | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| lroundf128       | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| modf             | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| modff            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| modfl            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| modff128         | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nan              | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nanf             | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nanl             | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nanf128          | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nearbyint        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nearbyintf       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nearbyintl       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nextafter        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nextafterf       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nextafterl       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nextafterf128    | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nextdown         | |check| | |check| | |check| | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nextdownf        | |check| | |check| | |check| | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nextdownl        | |check| | |check| | |check| | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nextdownf128     | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nexttoward       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nexttowardf      | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nexttowardl      | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nextup           | |check| | |check| | |check| | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nextupf          | |check| | |check| | |check| | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nextupl          | |check| | |check| | |check| | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| nextupf128       | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| remainder        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| remainderf       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| remainderl       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| remquo           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| remquof          | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| remquol          | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| rint             | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| rintf            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| rintl            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| rintf128         | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| round            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| roundf           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| roundl           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| roundf128        | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| scalbn           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| scalbnf          | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| scalbnl          | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| trunc            | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| truncf           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| truncl           | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| truncf128        | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ufromfp          | |check| |         |         |         |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ufromfpf         | |check| |         |         |         |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ufromfpl         | |check| |         |         |         |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ufromfpf128      | |check| |         |         |         |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ufromfpx         | |check| |         |         |         |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ufromfpxf        | |check| |         |         |         |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ufromfpxl        | |check| |         |         |         |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| ufromfpxf128     | |check| |         |         |         |         |         |         |         |         |         |         |         |
++------------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 
 
 Higher Math Functions
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index 89b43f06778d..ac6e1d1801ba 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -636,6 +636,11 @@ def StdC : StandardSpec<"stdc"> {
           FunctionSpec<"nan", RetValSpec<DoubleType>, [ArgSpec<ConstCharPtr>]>,
           FunctionSpec<"nanl", RetValSpec<LongDoubleType>, [ArgSpec<ConstCharPtr>]>,
           GuardedFunctionSpec<"nanf128", RetValSpec<Float128Type>, [ArgSpec<ConstCharPtr>], "LIBC_TYPES_HAS_FLOAT128">,
+
+          FunctionSpec<"canonicalize", RetValSpec<IntType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
+          FunctionSpec<"canonicalizef", RetValSpec<IntType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
+          FunctionSpec<"canonicalizel", RetValSpec<IntType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
+          GuardedFunctionSpec<"canonicalizef128", RetValSpec<IntType>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT128">,
       ]
   >;
 
diff --git a/libc/src/__support/FPUtil/BasicOperations.h b/libc/src/__support/FPUtil/BasicOperations.h
index b8cff982d3a6..405755f8b57d 100644
--- a/libc/src/__support/FPUtil/BasicOperations.h
+++ b/libc/src/__support/FPUtil/BasicOperations.h
@@ -9,11 +9,14 @@
 #ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_BASICOPERATIONS_H
 #define LLVM_LIBC_SRC___SUPPORT_FPUTIL_BASICOPERATIONS_H
 
+#include "FEnvImpl.h"
 #include "FPBits.h"
 
 #include "FEnvImpl.h"
 #include "src/__support/CPP/type_traits.h"
+#include "src/__support/UInt128.h"
 #include "src/__support/common.h"
+#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
 
 namespace LIBC_NAMESPACE {
 namespace fputil {
@@ -178,6 +181,66 @@ LIBC_INLINE T fdim(T x, T y) {
   return (x > y ? x - y : 0);
 }
 
+template <typename T, cpp::enable_if_t<cpp::is_floating_point_v<T>, int> = 0>
+LIBC_INLINE int canonicalize(T &cx, const T &x) {
+  FPBits<T> sx(x);
+  if constexpr (get_fp_type<T>() == FPType::X86_Binary80) {
+    // All the pseudo and unnormal numbers are not canonical.
+    // More precisely :
+    // Exponent   |       Significand      | Meaning
+    //            | Bits 63-62 | Bits 61-0 |
+    // All Ones   |     00     |    Zero   | Pseudo Infinity, Value = Infinty
+    // All Ones   |     00     |  Non-Zero | Pseudo NaN, Value = SNaN
+    // All Ones   |     01     | Anything  | Pseudo NaN, Value = SNaN
+    //            |   Bit 63   | Bits 62-0 |
+    // All zeroes |   One      | Anything  | Pseudo Denormal, Value =
+    //            |            |           | (−1)**s × m × 2**−16382
+    // All Other  |   Zero     | Anything  | Unnormal, Value =
+    //  Values    |            |           | (−1)**s × m × 2**−16382
+    bool bit63 = sx.get_implicit_bit();
+    UInt128 mantissa = sx.get_explicit_mantissa();
+    bool bit62 = ((mantissa & (1ULL << 62)) >> 62);
+    int exponent = sx.get_biased_exponent();
+    if (exponent == 0x7FFF) {
+      if (!bit63 && !bit62) {
+        if (mantissa == 0)
+          cx = FPBits<T>::inf(sx.sign()).get_val();
+        else {
+          cx = FPBits<T>::quiet_nan(sx.sign(), mantissa).get_val();
+          raise_except_if_required(FE_INVALID);
+          return 1;
+        }
+      } else if (!bit63 && bit62) {
+        cx = FPBits<T>::quiet_nan(sx.sign(), mantissa).get_val();
+        raise_except_if_required(FE_INVALID);
+        return 1;
+      } else if (LIBC_UNLIKELY(sx.is_signaling_nan())) {
+        cx = FPBits<T>::quiet_nan(sx.sign(), sx.get_explicit_mantissa())
+                 .get_val();
+        raise_except_if_required(FE_INVALID);
+        return 1;
+      } else
+        cx = x;
+    } else if (exponent == 0 && bit63)
+      cx = FPBits<T>::make_value(mantissa, 0).get_val();
+    else if (exponent != 0 && !bit63)
+      cx = FPBits<T>::make_value(mantissa, 0).get_val();
+    else if (LIBC_UNLIKELY(sx.is_signaling_nan())) {
+      cx =
+          FPBits<T>::quiet_nan(sx.sign(), sx.get_explicit_mantissa()).get_val();
+      raise_except_if_required(FE_INVALID);
+      return 1;
+    } else
+      cx = x;
+  } else if (LIBC_UNLIKELY(sx.is_signaling_nan())) {
+    cx = FPBits<T>::quiet_nan(sx.sign(), sx.get_explicit_mantissa()).get_val();
+    raise_except_if_required(FE_INVALID);
+    return 1;
+  } else
+    cx = x;
+  return 0;
+}
+
 } // namespace fputil
 } // namespace LIBC_NAMESPACE
 
diff --git a/libc/src/__support/FPUtil/CMakeLists.txt b/libc/src/__support/FPUtil/CMakeLists.txt
index 4ded70a675ea..0f4350234197 100644
--- a/libc/src/__support/FPUtil/CMakeLists.txt
+++ b/libc/src/__support/FPUtil/CMakeLists.txt
@@ -82,8 +82,11 @@ add_header_library(
     BasicOperations.h
   DEPENDS
     .fp_bits
+    .fenv_impl
     libc.src.__support.CPP.type_traits
+    libc.src.__support.uint128
     libc.src.__support.common
+    libc.src.__support.macros.optimization
 )
 
 add_header_library(
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index 61baa498321c..b67ee3a8e0df 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -59,6 +59,11 @@ add_math_entrypoint_object(atan2f)
 add_math_entrypoint_object(atanh)
 add_math_entrypoint_object(atanhf)
 
+add_math_entrypoint_object(canonicalize)
+add_math_entrypoint_object(canonicalizef)
+add_math_entrypoint_object(canonicalizef128)
+add_math_entrypoint_object(canonicalizel)
+
 add_math_entrypoint_object(ceil)
 add_math_entrypoint_object(ceilf)
 add_math_entrypoint_object(ceill)
diff --git a/libc/src/math/canonicalize.h b/libc/src/math/canonicalize.h
new file mode 100644
index 000000000000..b7b5959fb667
--- /dev/null
+++ b/libc/src/math/canonicalize.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for canonicalize -------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_CANONICALIZE_H
+#define LLVM_LIBC_SRC_MATH_CANONICALIZE_H
+
+namespace LIBC_NAMESPACE {
+
+int canonicalize(double *cx, const double *x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_CANONICALIZE_H
diff --git a/libc/src/math/canonicalizef.h b/libc/src/math/canonicalizef.h
new file mode 100644
index 000000000000..556607f13349
--- /dev/null
+++ b/libc/src/math/canonicalizef.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for canonicalizef ------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_CANONICALIZEF_H
+#define LLVM_LIBC_SRC_MATH_CANONICALIZEF_H
+
+namespace LIBC_NAMESPACE {
+
+int canonicalizef(float *cx, const float *x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_CANONICALIZEF_H
diff --git a/libc/src/math/canonicalizef128.h b/libc/src/math/canonicalizef128.h
new file mode 100644
index 000000000000..6db800947537
--- /dev/null
+++ b/libc/src/math/canonicalizef128.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for canonicalizef128 ---------------*-C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_CANONICALIZEF128_H
+#define LLVM_LIBC_SRC_MATH_CANONICALIZEF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+int canonicalizef128(float128 *cx, const float128 *x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_CANONICALIZEF128_H
diff --git a/libc/src/math/canonicalizel.h b/libc/src/math/canonicalizel.h
new file mode 100644
index 000000000000..1cab29e8e8b1
--- /dev/null
+++ b/libc/src/math/canonicalizel.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for canonicalizel ------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_CANONICALIZEL_H
+#define LLVM_LIBC_SRC_MATH_CANONICALIZEL_H
+
+namespace LIBC_NAMESPACE {
+
+int canonicalizel(long double *cx, const long double *x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_CANONICALIZEL_H
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index 60d329f9adc6..4d9b91150d02 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -1,3 +1,51 @@
+add_entrypoint_object(
+  canonicalize
+  SRCS
+    canonicalize.cpp
+  HDRS
+    ../canonicalize.h
+  COMPILE_OPTIONS
+    -O3
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+)
+
+add_entrypoint_object(
+  canonicalizef
+  SRCS
+    canonicalizef.cpp
+  HDRS
+    ../canonicalizef.h
+  COMPILE_OPTIONS
+    -O3
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+)
+
+add_entrypoint_object(
+  canonicalizef128
+  SRCS
+    canonicalizef128.cpp
+  HDRS
+    ../canonicalizef128.h
+  COMPILE_OPTIONS
+    -O3
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+)
+
+add_entrypoint_object(
+  canonicalizel
+  SRCS
+    canonicalizel.cpp
+  HDRS
+    ../canonicalizel.h
+  COMPILE_OPTIONS
+    -O3
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+)
+
 add_entrypoint_object(
   ceil
   SRCS
diff --git a/libc/src/math/generic/canonicalize.cpp b/libc/src/math/generic/canonicalize.cpp
new file mode 100644
index 000000000000..f38ca01e157f
--- /dev/null
+++ b/libc/src/math/generic/canonicalize.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of canonicalize function----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/canonicalize.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(int, canonicalize, (double *cx, const double *x)) {
+  return fputil::canonicalize(*cx, *x);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/canonicalizef.cpp b/libc/src/math/generic/canonicalizef.cpp
new file mode 100644
index 000000000000..dce601de1491
--- /dev/null
+++ b/libc/src/math/generic/canonicalizef.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of canonicalizef function---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/canonicalizef.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(int, canonicalizef, (float *cx, const float *x)) {
+  return fputil::canonicalize(*cx, *x);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/canonicalizef128.cpp b/libc/src/math/generic/canonicalizef128.cpp
new file mode 100644
index 000000000000..0078b478238c
--- /dev/null
+++ b/libc/src/math/generic/canonicalizef128.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of canonicalizef128 function------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/canonicalizef128.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(int, canonicalizef128, (float128 * cx, const float128 *x)) {
+  return fputil::canonicalize(*cx, *x);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/canonicalizel.cpp b/libc/src/math/generic/canonicalizel.cpp
new file mode 100644
index 000000000000..5310a316acdd
--- /dev/null
+++ b/libc/src/math/generic/canonicalizel.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of canonicalizel function---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/canonicalizel.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(int, canonicalizel,
+                   (long double *cx, const long double *x)) {
+  return fputil::canonicalize(*cx, *x);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index 1345dca08a6a..3b756127fe21 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -165,6 +165,66 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  canonicalize_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    canonicalize_test.cpp
+  HDRS
+    CanonicalizeTest.h
+  DEPENDS
+    libc.include.math
+    libc.src.math.canonicalize
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.fenv_impl
+)
+
+add_fp_unittest(
+  canonicalizef_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    canonicalizef_test.cpp
+  HDRS
+    CanonicalizeTest.h
+  DEPENDS
+    libc.include.math
+    libc.src.math.canonicalizef
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.fenv_impl
+)
+
+add_fp_unittest(
+  canonicalizef128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    canonicalizef128_test.cpp
+  HDRS
+    CanonicalizeTest.h
+  DEPENDS
+    libc.include.math
+    libc.src.math.canonicalizef128
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.fenv_impl
+)
+
+add_fp_unittest(
+  canonicalizel_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    canonicalizel_test.cpp
+  HDRS
+    CanonicalizeTest.h
+  DEPENDS
+    libc.include.math
+    libc.src.math.canonicalizel
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.fenv_impl
+)
+
 add_fp_unittest(
   ceil_test
   SUITE
diff --git a/libc/test/src/math/smoke/CanonicalizeTest.h b/libc/test/src/math/smoke/CanonicalizeTest.h
new file mode 100644
index 000000000000..ce855d516856
--- /dev/null
+++ b/libc/test/src/math/smoke/CanonicalizeTest.h
@@ -0,0 +1,219 @@
+//===-- Utility class to test canonicalize[f|l] -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TEST_SRC_MATH_SMOKE_CANONICALIZETEST_H
+#define LLVM_LIBC_TEST_SRC_MATH_SMOKE_CANONICALIZETEST_H
+
+#include "src/__support/FPUtil/FPBits.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+#include "include/llvm-libc-macros/math-macros.h"
+
+#define TEST_SPECIAL(x, y, expected, expected_exception)                       \
+  EXPECT_EQ(expected, f(&x, &y));                                              \
+  EXPECT_FP_EXCEPTION(expected_exception);                                     \
+  LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT)
+
+#define TEST_REGULAR(x, y, expected) TEST_SPECIAL(x, y, expected, 0)
+
+#define LIBC_NAMESPACE __llvm_libc_19_0_0_git
+
+template <typename T>
+class CanonicalizeTest : public LIBC_NAMESPACE::testing::Test {
+
+  DECLARE_SPECIAL_CONSTANTS(T)
+
+public:
+  typedef int (*CanonicalizeFunc)(T *, const T *);
+
+  void testSpecialNumbers(CanonicalizeFunc f) {
+    T cx;
+
+    TEST_SPECIAL(cx, zero, 0, 0);
+    EXPECT_FP_EQ(cx, zero);
+
+    TEST_SPECIAL(cx, neg_zero, 0, 0);
+    EXPECT_FP_EQ(cx, neg_zero);
+
+    TEST_SPECIAL(cx, inf, 0, 0);
+    EXPECT_FP_EQ(cx, inf);
+
+    TEST_SPECIAL(cx, neg_inf, 0, 0);
+    EXPECT_FP_EQ(cx, neg_inf);
+
+    TEST_SPECIAL(cx, sNaN, 1, FE_INVALID);
+    EXPECT_FP_EQ(cx, aNaN);
+  }
+
+  void testX64_80SpecialNumbers(CanonicalizeFunc f) {
+    if constexpr (LIBC_NAMESPACE::fputil::get_fp_type<T>() ==
+                  LIBC_NAMESPACE::fputil::FPType::X86_Binary80) {
+      T cx;
+      // Exponent   |       Significand      | Meaning
+      //            | Bits 63-62 | Bits 61-0 |
+      // All Ones   |     00     |    Zero   | Pseudo Infinity, Value = Infinty
+
+      FPBits test1((UInt128(0x7FFF) << 64) + UInt128(0x0000000000000000));
+      const T test1_val = test1.get_val();
+      TEST_SPECIAL(cx, test1_val, 0, 0);
+      EXPECT_FP_EQ(cx, inf);
+
+      // Exponent   |       Significand      | Meaning
+      //            | Bits 63-62 | Bits 61-0 |
+      // All Ones   |     00     |  Non-Zero | Pseudo NaN, Value = SNaN
+
+      FPBits test2_1((UInt128(0x7FFF) << 64) + UInt128(0x0000000000000001));
+      const T test2_1_val = test2_1.get_val();
+      TEST_SPECIAL(cx, test2_1_val, 1, FE_INVALID);
+      EXPECT_FP_EQ(cx, aNaN);
+
+      FPBits test2_2((UInt128(0x7FFF) << 64) + UInt128(0x0000004270000001));
+      const T test2_2_val = test2_2.get_val();
+      TEST_SPECIAL(cx, test2_2_val, 1, FE_INVALID);
+      EXPECT_FP_EQ(cx, aNaN);
+
+      FPBits test2_3((UInt128(0x7FFF) << 64) + UInt128(0x0000000008261001));
+      const T test2_3_val = test2_3.get_val();
+      TEST_SPECIAL(cx, test2_3_val, 1, FE_INVALID);
+      EXPECT_FP_EQ(cx, aNaN);
+
+      FPBits test2_4((UInt128(0x7FFF) << 64) + UInt128(0x0000780008261001));
+      const T test2_4_val = test2_4.get_val();
+      TEST_SPECIAL(cx, test2_4_val, 1, FE_INVALID);
+      EXPECT_FP_EQ(cx, aNaN);
+
+      // Exponent   |       Significand      | Meaning
+      //            | Bits 63-62 | Bits 61-0 |
+      // All Ones   |     01     | Anything  | Pseudo NaN, Value = SNaN
+
+      FPBits test3_1((UInt128(0x7FFF) << 64) + UInt128(0x4000000000000000));
+      const T test3_1_val = test3_1.get_val();
+      TEST_SPECIAL(cx, test3_1_val, 1, FE_INVALID);
+      EXPECT_FP_EQ(cx, aNaN);
+
+      FPBits test3_2((UInt128(0x7FFF) << 64) + UInt128(0x4000004270000001));
+      const T test3_2_val = test3_2.get_val();
+      TEST_SPECIAL(cx, test3_2_val, 1, FE_INVALID);
+      EXPECT_FP_EQ(cx, aNaN);
+
+      FPBits test3_3((UInt128(0x7FFF) << 64) + UInt128(0x4000000008261001));
+      const T test3_3_val = test3_3.get_val();
+      TEST_SPECIAL(cx, test3_3_val, 1, FE_INVALID);
+      EXPECT_FP_EQ(cx, aNaN);
+
+      FPBits test3_4((UInt128(0x7FFF) << 64) + UInt128(0x4007800008261001));
+      const T test3_4_val = test3_4.get_val();
+      TEST_SPECIAL(cx, test3_4_val, 1, FE_INVALID);
+      EXPECT_FP_EQ(cx, aNaN);
+
+      // Exponent   |       Significand      | Meaning
+      //            |   Bit 63   | Bits 62-0 |
+      // All zeroes |   One      | Anything  | Pseudo Denormal, Value =
+      //            |            |           | (−1)**s × m × 2**−16382
+
+      FPBits test4_1((UInt128(0x0000) << 64) + UInt128(0x8000000000000000));
+      const T test4_1_val = test4_1.get_val();
+      TEST_SPECIAL(cx, test4_1_val, 0, 0);
+      EXPECT_FP_EQ(
+          cx, FPBits::make_value(test4_1.get_explicit_mantissa(), 0).get_val());
+
+      FPBits test4_2((UInt128(0x0000) << 64) + UInt128(0x8000004270000001));
+      const T test4_2_val = test4_2.get_val();
+      TEST_SPECIAL(cx, test4_2_val, 0, 0);
+      EXPECT_FP_EQ(
+          cx, FPBits::make_value(test4_2.get_explicit_mantissa(), 0).get_val());
+
+      FPBits test4_3((UInt128(0x0000) << 64) + UInt128(0x8000000008261001));
+      const T test4_3_val = test4_3.get_val();
+      TEST_SPECIAL(cx, test4_3_val, 0, 0);
+      EXPECT_FP_EQ(
+          cx, FPBits::make_value(test4_3.get_explicit_mantissa(), 0).get_val());
+
+      // Exponent   |       Significand      | Meaning
+      //            |   Bit 63   | Bits 62-0 |
+      // All Other  |   Zero     | Anything  | Unnormal, Value =
+      //  Values    |            |           | (−1)**s × m × 2**−16382
+
+      FPBits test5_1(UInt128(0x0000000000000001));
+      const T test5_1_val = test5_1.get_val();
+      TEST_SPECIAL(cx, test5_1_val, 0, 0);
+      EXPECT_FP_EQ(
+          cx, FPBits::make_value(test5_1.get_explicit_mantissa(), 0).get_val());
+
+      FPBits test5_2(UInt128(0x0000004270000001));
+      const T test5_2_val = test5_2.get_val();
+      TEST_SPECIAL(cx, test5_2_val, 0, 0);
+      EXPECT_FP_EQ(
+          cx, FPBits::make_value(test5_2.get_explicit_mantissa(), 0).get_val());
+
+      FPBits test5_3(UInt128(0x0000000008261001));
+      const T test5_3_val = test5_3.get_val();
+      TEST_SPECIAL(cx, test5_3_val, 0, 0);
+      EXPECT_FP_EQ(
+          cx, FPBits::make_value(test5_3.get_explicit_mantissa(), 0).get_val());
+
+      FPBits test5_4(UInt128(0x0000002816000000));
+      const T test5_4_val = test5_4.get_val();
+      TEST_SPECIAL(cx, test5_4_val, 0, 0);
+      EXPECT_FP_EQ(
+          cx, FPBits::make_value(test5_4.get_explicit_mantissa(), 0).get_val());
+
+      FPBits test5_5(UInt128(0x0000004270000001));
+      const T test5_5_val = test5_5.get_val();
+      TEST_SPECIAL(cx, test5_5_val, 0, 0);
+      EXPECT_FP_EQ(
+          cx, FPBits::make_value(test5_5.get_explicit_mantissa(), 0).get_val());
+
+      FPBits test5_6(UInt128(0x0000000008261001));
+      const T test5_6_val = test5_6.get_val();
+      TEST_SPECIAL(cx, test5_6_val, 0, 0);
+      EXPECT_FP_EQ(
+          cx, FPBits::make_value(test5_6.get_explicit_mantissa(), 0).get_val());
+    }
+  }
+
+  void testRegularNumbers(CanonicalizeFunc f) {
+    T cx;
+    const T test_var_1 = T(1.0);
+    TEST_REGULAR(cx, test_var_1, 0);
+    EXPECT_FP_EQ(cx, test_var_1);
+    const T test_var_2 = T(-1.0);
+    TEST_REGULAR(cx, test_var_2, 0);
+    EXPECT_FP_EQ(cx, test_var_2);
+    const T test_var_3 = T(10.0);
+    TEST_REGULAR(cx, test_var_3, 0);
+    EXPECT_FP_EQ(cx, test_var_3);
+    const T test_var_4 = T(-10.0);
+    TEST_REGULAR(cx, test_var_4, 0);
+    EXPECT_FP_EQ(cx, test_var_4);
+    const T test_var_5 = T(1234.0);
+    TEST_REGULAR(cx, test_var_5, 0);
+    EXPECT_FP_EQ(cx, test_var_5);
+    const T test_var_6 = T(-1234.0);
+    TEST_REGULAR(cx, test_var_6, 0);
+    EXPECT_FP_EQ(cx, test_var_6);
+  }
+};
+
+#define LIST_CANONICALIZE_TESTS(T, func)                                       \
+  using LlvmLibcCanonicalizeTest = CanonicalizeTest<T>;                        \
+  TEST_F(LlvmLibcCanonicalizeTest, SpecialNumbers) {                           \
+    testSpecialNumbers(&func);                                                 \
+  }                                                                            \
+  TEST_F(LlvmLibcCanonicalizeTest, RegularNubmers) {                           \
+    testRegularNumbers(&func);                                                 \
+  }
+
+#define X86_80_SPECIAL_CANONICALIZE_TEST(T, func)                              \
+  using LlvmLibcCanonicalizeTest = CanonicalizeTest<T>;                        \
+  TEST_F(LlvmLibcCanonicalizeTest, X64_80SpecialNumbers) {                     \
+    testX64_80SpecialNumbers(&func);                                           \
+  }
+
+#endif // LLVM_LIBC_TEST_SRC_MATH_SMOKE_CANONICALIZETEST_H
diff --git a/libc/test/src/math/smoke/canonicalize_test.cpp b/libc/test/src/math/smoke/canonicalize_test.cpp
new file mode 100644
index 000000000000..54a1ddd49ca4
--- /dev/null
+++ b/libc/test/src/math/smoke/canonicalize_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for canonicalize ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "CanonicalizeTest.h"
+
+#include "src/math/canonicalize.h"
+
+LIST_CANONICALIZE_TESTS(double, LIBC_NAMESPACE::canonicalize)
diff --git a/libc/test/src/math/smoke/canonicalizef128_test.cpp b/libc/test/src/math/smoke/canonicalizef128_test.cpp
new file mode 100644
index 000000000000..242e2331a29e
--- /dev/null
+++ b/libc/test/src/math/smoke/canonicalizef128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for canonicalizef128 ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "CanonicalizeTest.h"
+
+#include "src/math/canonicalizef128.h"
+
+LIST_CANONICALIZE_TESTS(float128, LIBC_NAMESPACE::canonicalizef128)
diff --git a/libc/test/src/math/smoke/canonicalizef_test.cpp b/libc/test/src/math/smoke/canonicalizef_test.cpp
new file mode 100644
index 000000000000..17cf3c3639a4
--- /dev/null
+++ b/libc/test/src/math/smoke/canonicalizef_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for canonicalizef ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "CanonicalizeTest.h"
+
+#include "src/math/canonicalizef.h"
+
+LIST_CANONICALIZE_TESTS(float, LIBC_NAMESPACE::canonicalizef)
diff --git a/libc/test/src/math/smoke/canonicalizel_test.cpp b/libc/test/src/math/smoke/canonicalizel_test.cpp
new file mode 100644
index 000000000000..23cb21dce84e
--- /dev/null
+++ b/libc/test/src/math/smoke/canonicalizel_test.cpp
@@ -0,0 +1,19 @@
+//===-- Unittests for canonicalizel ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "CanonicalizeTest.h"
+
+#include "src/math/canonicalizel.h"
+
+LIST_CANONICALIZE_TESTS(long double, LIBC_NAMESPACE::canonicalizel)
+
+#ifdef LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80
+
+X86_80_SPECIAL_CANONICALIZE_TEST(long double, LIBC_NAMESPACE::canonicalizel)
+
+#endif // LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 0d531a3dc12a..0557c3aec0d8 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -622,6 +622,9 @@ libc_support_library(
     name = "__support_fputil_basic_operations",
     hdrs = ["src/__support/FPUtil/BasicOperations.h"],
     deps = [
+        ":__support_fputil_fenv_impl",
+        ":__support_macros_optimization",
+        ":__support_uint128",
         ":__support_common",
         ":__support_cpp_type_traits",
         ":__support_fputil_fp_bits",
-- 
GitLab


From 59e052568e7d46cc0489269e3c76f53bb21941f5 Mon Sep 17 00:00:00 2001
From: ycdtosa <ycdtosa@users.noreply.github.com>
Date: Tue, 26 Mar 2024 13:38:31 +0100
Subject: [PATCH 270/404] [[clang::always_destroy]] attribute silences
 warn-exit-time-destructor (#86486)

This attribute tells the compiler that the variable must have its exit-time
destructor run, so it makes sense that it would silence the warning telling
users that an exit-time destructor is required.

Fixes https://github.com/llvm/llvm-project/issues/68686
---
 clang/docs/ReleaseNotes.rst                       |  4 ++++
 clang/include/clang/Basic/AttrDocs.td             |  3 +++
 clang/lib/Sema/SemaDeclCXX.cpp                    |  3 ++-
 clang/test/SemaCXX/warn-exit-time-destructors.cpp | 10 ++++++++++
 4 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 7fbe2fec6ca0..af2295c43d7e 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -295,6 +295,10 @@ Improvements to Clang's diagnostics
 - Clang now correctly diagnoses no arguments to a variadic macro parameter as a C23/C++20 extension.
   Fixes #GH84495.
 
+- Clang no longer emits a ``-Wexit-time destructors`` warning on static variables explicitly
+  annotated with the ``clang::always_destroy`` attribute.
+  Fixes #GH68686, #GH86486
+
 Improvements to Clang's time-trace
 ----------------------------------
 
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 9de14f608fd1..384aebbdf2e3 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -6069,6 +6069,9 @@ def AlwaysDestroyDocs : Documentation {
 The ``always_destroy`` attribute specifies that a variable with static or thread
 storage duration should have its exit-time destructor run. This attribute is the
 default unless clang was invoked with -fno-c++-static-destructors.
+
+If a variable is explicitly declared with this attribute, Clang will silence
+otherwise applicable ``-Wexit-time-destructors`` warnings.
   }];
 }
 
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index ee732679417e..7070ea00cff2 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -16202,7 +16202,8 @@ void Sema::FinalizeVarWithDestructor(VarDecl *VD, const RecordType *Record) {
 
   // Emit warning for non-trivial dtor in global scope (a real global,
   // class-static, function-static).
-  Diag(VD->getLocation(), diag::warn_exit_time_destructor);
+  if (!VD->hasAttr<AlwaysDestroyAttr>())
+    Diag(VD->getLocation(), diag::warn_exit_time_destructor);
 
   // TODO: this should be re-enabled for static locals by !CXAAtExit
   if (!VD->isStaticLocal())
diff --git a/clang/test/SemaCXX/warn-exit-time-destructors.cpp b/clang/test/SemaCXX/warn-exit-time-destructors.cpp
index 2f14243cb48c..55ae37d7368f 100644
--- a/clang/test/SemaCXX/warn-exit-time-destructors.cpp
+++ b/clang/test/SemaCXX/warn-exit-time-destructors.cpp
@@ -51,6 +51,15 @@ struct A { ~A(); };
 }
 
 namespace test5 {
+  struct A { ~A(); };
+  [[clang::always_destroy]] A a; // no warning
+
+  void func() {
+    [[clang::always_destroy]] static A a; // no warning
+  }
+}
+
+namespace test6 {
 #if __cplusplus >= 202002L
 #define CPP20_CONSTEXPR constexpr
 #else
@@ -68,3 +77,4 @@ namespace test5 {
   T t; // expected-warning {{exit-time destructor}}
 #undef CPP20_CONSTEXPR
 }
+
-- 
GitLab


From feebcd65fb7e0534f5219e05432a05e45aa8cd2a Mon Sep 17 00:00:00 2001
From: Jie Fu <jiefu@tencent.com>
Date: Tue, 26 Mar 2024 20:41:46 +0800
Subject: [PATCH 271/404] [clang] Fix -Wunused-variable in SemaDecl.cpp (NFC)

llvm-project/clang/lib/Sema/SemaDecl.cpp:11653:20:
error: unused variable 'OldMVKind' [-Werror,-Wunused-variable]
  MultiVersionKind OldMVKind = OldFD->getMultiVersionKind();
                   ^
1 error generated.
---
 clang/lib/Sema/SemaDecl.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 9a286e0b26a4..b907fee01d5f 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -11650,7 +11650,7 @@ static bool CheckMultiVersionAdditionalDecl(
   const auto *NewTA = NewFD->getAttr<TargetAttr>();
   const auto *NewTVA = NewFD->getAttr<TargetVersionAttr>();
   MultiVersionKind NewMVKind = NewFD->getMultiVersionKind();
-  MultiVersionKind OldMVKind = OldFD->getMultiVersionKind();
+  [[maybe_unused]] MultiVersionKind OldMVKind = OldFD->getMultiVersionKind();
 
   ParsedTargetAttr NewParsed;
   if (NewTA) {
-- 
GitLab


From caa225825075237cd6b15757476dd1999bec290b Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Tue, 26 Mar 2024 20:56:16 +0800
Subject: [PATCH 272/404] [LLVM] Remove nuw neg (#86295)

This patch removes APIs that creating NUW neg. It is a trivial case
because `sub nuw 0, X` always gets simplified into zero.
I believe there is no optimization opportunities in the real-world
applications that we can take advantage of the nuw flag.

Motivated by
https://github.com/llvm/llvm-project/pull/84792#discussion_r1524891134.

Compile-time improvement:
https://llvm-compile-time-tracker.com/compare.php?from=d1f182c895728d89c5c3d198b133e212a5d9d4a3&to=da7b7478b7cbb32c09d760f6b8d0e67901e0d533&stat=instructions:u
---
 llvm/docs/ReleaseNotes.rst                          |  2 ++
 llvm/include/llvm-c/Core.h                          | 10 +++++++---
 llvm/include/llvm/IR/Constants.h                    |  6 ++----
 llvm/include/llvm/IR/IRBuilder.h                    | 13 ++++---------
 llvm/include/llvm/IR/InstrTypes.h                   |  6 ------
 llvm/lib/IR/Constants.cpp                           |  4 ++--
 llvm/lib/IR/Core.cpp                                |  7 +++++--
 llvm/lib/IR/Instructions.cpp                        | 12 ------------
 .../Transforms/InstCombine/InstCombineAddSub.cpp    |  7 ++++---
 .../Transforms/InstCombine/InstCombineAndOrXor.cpp  |  7 ++++---
 .../lib/Transforms/InstCombine/InstCombineCalls.cpp |  2 +-
 .../Transforms/InstCombine/InstCombineMulDivRem.cpp |  9 ++++-----
 .../Transforms/InstCombine/InstCombineNegator.cpp   |  2 +-
 .../Scalar/CorrelatedValuePropagation.cpp           |  2 +-
 14 files changed, 37 insertions(+), 52 deletions(-)

diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index 01ecbdba5060..08b496a909a7 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -146,6 +146,8 @@ Changes to the C API
   * ``LLVMGetPrologueData``
   * ``LLVMSetPrologueData``
 
+* Deprecated ``LLVMConstNUWNeg`` and ``LLVMBuildNUWNeg``.
+
 Changes to the CodeGen infrastructure
 -------------------------------------
 
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h
index f56a6c961aad..254c298abe4b 100644
--- a/llvm/include/llvm-c/Core.h
+++ b/llvm/include/llvm-c/Core.h
@@ -2316,7 +2316,9 @@ LLVMValueRef LLVMAlignOf(LLVMTypeRef Ty);
 LLVMValueRef LLVMSizeOf(LLVMTypeRef Ty);
 LLVMValueRef LLVMConstNeg(LLVMValueRef ConstantVal);
 LLVMValueRef LLVMConstNSWNeg(LLVMValueRef ConstantVal);
-LLVMValueRef LLVMConstNUWNeg(LLVMValueRef ConstantVal);
+LLVM_ATTRIBUTE_C_DEPRECATED(
+    LLVMValueRef LLVMConstNUWNeg(LLVMValueRef ConstantVal),
+    "Use LLVMConstNull instead.");
 LLVMValueRef LLVMConstNot(LLVMValueRef ConstantVal);
 LLVMValueRef LLVMConstAdd(LLVMValueRef LHSConstant, LLVMValueRef RHSConstant);
 LLVMValueRef LLVMConstNSWAdd(LLVMValueRef LHSConstant, LLVMValueRef RHSConstant);
@@ -4152,8 +4154,10 @@ LLVMValueRef LLVMBuildBinOp(LLVMBuilderRef B, LLVMOpcode Op,
 LLVMValueRef LLVMBuildNeg(LLVMBuilderRef, LLVMValueRef V, const char *Name);
 LLVMValueRef LLVMBuildNSWNeg(LLVMBuilderRef B, LLVMValueRef V,
                              const char *Name);
-LLVMValueRef LLVMBuildNUWNeg(LLVMBuilderRef B, LLVMValueRef V,
-                             const char *Name);
+LLVM_ATTRIBUTE_C_DEPRECATED(LLVMValueRef LLVMBuildNUWNeg(LLVMBuilderRef B,
+                                                         LLVMValueRef V,
+                                                         const char *Name),
+                            "Use LLVMBuildNeg + LLVMSetNUW instead.");
 LLVMValueRef LLVMBuildFNeg(LLVMBuilderRef, LLVMValueRef V, const char *Name);
 LLVMValueRef LLVMBuildNot(LLVMBuilderRef, LLVMValueRef V, const char *Name);
 
diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h
index e50cd1f1c73e..4290ef4486c6 100644
--- a/llvm/include/llvm/IR/Constants.h
+++ b/llvm/include/llvm/IR/Constants.h
@@ -1046,8 +1046,7 @@ public:
   ///
   static Constant *getSizeOf(Type *Ty);
 
-  static Constant *getNeg(Constant *C, bool HasNUW = false,
-                          bool HasNSW = false);
+  static Constant *getNeg(Constant *C, bool HasNSW = false);
   static Constant *getNot(Constant *C);
   static Constant *getAdd(Constant *C1, Constant *C2, bool HasNUW = false,
                           bool HasNSW = false);
@@ -1068,8 +1067,7 @@ public:
   static Constant *getAddrSpaceCast(Constant *C, Type *Ty,
                                     bool OnlyIfReduced = false);
 
-  static Constant *getNSWNeg(Constant *C) { return getNeg(C, false, true); }
-  static Constant *getNUWNeg(Constant *C) { return getNeg(C, true, false); }
+  static Constant *getNSWNeg(Constant *C) { return getNeg(C, /*HasNSW=*/true); }
 
   static Constant *getNSWAdd(Constant *C1, Constant *C2) {
     return getAdd(C1, C2, false, true);
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index bee1a1f207fc..7d98009ec435 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -1712,18 +1712,13 @@ public:
       const Twine &Name = "", MDNode *FPMathTag = nullptr,
       std::optional<fp::ExceptionBehavior> Except = std::nullopt);
 
-  Value *CreateNeg(Value *V, const Twine &Name = "", bool HasNUW = false,
-                   bool HasNSW = false) {
-    return CreateSub(Constant::getNullValue(V->getType()), V, Name, HasNUW,
-                     HasNSW);
+  Value *CreateNeg(Value *V, const Twine &Name = "", bool HasNSW = false) {
+    return CreateSub(Constant::getNullValue(V->getType()), V, Name,
+                     /*HasNUW=*/0, HasNSW);
   }
 
   Value *CreateNSWNeg(Value *V, const Twine &Name = "") {
-    return CreateNeg(V, Name, false, true);
-  }
-
-  Value *CreateNUWNeg(Value *V, const Twine &Name = "") {
-    return CreateNeg(V, Name, true, false);
+    return CreateNeg(V, Name, /*HasNSW=*/true);
   }
 
   Value *CreateFNeg(Value *V, const Twine &Name = "",
diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h
index d2b33fe9c651..3a30e935849b 100644
--- a/llvm/include/llvm/IR/InstrTypes.h
+++ b/llvm/include/llvm/IR/InstrTypes.h
@@ -476,12 +476,6 @@ public:
                                       Instruction *InsertBefore = nullptr);
   static BinaryOperator *CreateNSWNeg(Value *Op, const Twine &Name,
                                       BasicBlock *InsertAtEnd);
-  static BinaryOperator *CreateNUWNeg(Value *Op, const Twine &Name,
-                                      BasicBlock::iterator InsertBefore);
-  static BinaryOperator *CreateNUWNeg(Value *Op, const Twine &Name = "",
-                                      Instruction *InsertBefore = nullptr);
-  static BinaryOperator *CreateNUWNeg(Value *Op, const Twine &Name,
-                                      BasicBlock *InsertAtEnd);
   static BinaryOperator *CreateNot(Value *Op, const Twine &Name,
                                    BasicBlock::iterator InsertBefore);
   static BinaryOperator *CreateNot(Value *Op, const Twine &Name = "",
diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp
index c17419b529ac..a5fb497f54ed 100644
--- a/llvm/lib/IR/Constants.cpp
+++ b/llvm/lib/IR/Constants.cpp
@@ -2520,10 +2520,10 @@ Constant *ConstantExpr::getShuffleVector(Constant *V1, Constant *V2,
   return pImpl->ExprConstants.getOrCreate(ShufTy, Key);
 }
 
-Constant *ConstantExpr::getNeg(Constant *C, bool HasNUW, bool HasNSW) {
+Constant *ConstantExpr::getNeg(Constant *C, bool HasNSW) {
   assert(C->getType()->isIntOrIntVectorTy() &&
          "Cannot NEG a nonintegral value!");
-  return getSub(ConstantInt::get(C->getType(), 0), C, HasNUW, HasNSW);
+  return getSub(ConstantInt::get(C->getType(), 0), C, /*HasNUW=*/false, HasNSW);
 }
 
 Constant *ConstantExpr::getNot(Constant *C) {
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index 023cabc46911..3aee61957252 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -1648,7 +1648,7 @@ LLVMValueRef LLVMConstNSWNeg(LLVMValueRef ConstantVal) {
 }
 
 LLVMValueRef LLVMConstNUWNeg(LLVMValueRef ConstantVal) {
-  return wrap(ConstantExpr::getNUWNeg(unwrap<Constant>(ConstantVal)));
+  return wrap(ConstantExpr::getNeg(unwrap<Constant>(ConstantVal)));
 }
 
 
@@ -3557,7 +3557,10 @@ LLVMValueRef LLVMBuildNSWNeg(LLVMBuilderRef B, LLVMValueRef V,
 
 LLVMValueRef LLVMBuildNUWNeg(LLVMBuilderRef B, LLVMValueRef V,
                              const char *Name) {
-  return wrap(unwrap(B)->CreateNUWNeg(unwrap(V), Name));
+  Value *Neg = unwrap(B)->CreateNeg(unwrap(V), Name);
+  if (auto *I = dyn_cast<BinaryOperator>(Neg))
+    I->setHasNoUnsignedWrap();
+  return wrap(Neg);
 }
 
 LLVMValueRef LLVMBuildFNeg(LLVMBuilderRef B, LLVMValueRef V, const char *Name) {
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 8ca211e6e2e7..c2abe875e019 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -3372,18 +3372,6 @@ BinaryOperator *BinaryOperator::CreateNSWNeg(Value *Op, const Twine &Name,
   return BinaryOperator::CreateNSWSub(Zero, Op, Name, InsertAtEnd);
 }
 
-BinaryOperator *BinaryOperator::CreateNUWNeg(Value *Op, const Twine &Name,
-                                             Instruction *InsertBefore) {
-  Value *Zero = ConstantInt::get(Op->getType(), 0);
-  return BinaryOperator::CreateNUWSub(Zero, Op, Name, InsertBefore);
-}
-
-BinaryOperator *BinaryOperator::CreateNUWNeg(Value *Op, const Twine &Name,
-                                             BasicBlock *InsertAtEnd) {
-  Value *Zero = ConstantInt::get(Op->getType(), 0);
-  return BinaryOperator::CreateNUWSub(Zero, Op, Name, InsertAtEnd);
-}
-
 BinaryOperator *BinaryOperator::CreateNot(Value *Op, const Twine &Name,
                                           BasicBlock::iterator InsertBefore) {
   Constant *C = Constant::getAllOnesValue(Op->getType());
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index a978e9a643f5..fa1e2280991f 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -2524,9 +2524,10 @@ Instruction *InstCombinerImpl::visitSub(BinaryOperator &I) {
     // sub (xor A, B), B  ; flip bits if negative and subtract -1 (add 1)
     // --> (A < 0) ? -A : A
     Value *IsNeg = Builder.CreateIsNeg(A);
-    // Copy the nuw/nsw flags from the sub to the negate.
-    Value *NegA = Builder.CreateNeg(A, "", I.hasNoUnsignedWrap(),
-                                    I.hasNoSignedWrap());
+    // Copy the nsw flags from the sub to the negate.
+    Value *NegA = I.hasNoUnsignedWrap()
+                      ? Constant::getNullValue(A->getType())
+                      : Builder.CreateNeg(A, "", I.hasNoSignedWrap());
     return SelectInst::Create(IsNeg, NegA, A);
   }
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index bf8794bba885..c0cf1a7db726 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -4250,10 +4250,11 @@ static Instruction *canonicalizeAbs(BinaryOperator &Xor,
     // xor (add A, Op1), Op1  ; add -1 and flip bits if negative
     // --> (A < 0) ? -A : A
     Value *IsNeg = Builder.CreateIsNeg(A);
-    // Copy the nuw/nsw flags from the add to the negate.
+    // Copy the nsw flags from the add to the negate.
     auto *Add = cast<BinaryOperator>(Op0);
-    Value *NegA = Builder.CreateNeg(A, "", Add->hasNoUnsignedWrap(),
-                                   Add->hasNoSignedWrap());
+    Value *NegA = Add->hasNoUnsignedWrap()
+                      ? Constant::getNullValue(A->getType())
+                      : Builder.CreateNeg(A, "", Add->hasNoSignedWrap());
     return SelectInst::Create(IsNeg, NegA, A);
   }
   return nullptr;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 526fe5d08059..b1017f4d6bc1 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -1795,7 +1795,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
       // We don't have a "nabs" intrinsic, so negate if needed based on the
       // max/min operation.
       if (IID == Intrinsic::smin || IID == Intrinsic::umax)
-        Abs = Builder.CreateNeg(Abs, "nabs", /* NUW */ false, IntMinIsPoison);
+        Abs = Builder.CreateNeg(Abs, "nabs", IntMinIsPoison);
       return replaceInstUsesWith(CI, Abs);
     }
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index 10a4b1c2060c..af238a43b11a 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -105,7 +105,7 @@ static Value *foldMulSelectToNegate(BinaryOperator &I,
   if (match(&I, m_c_Mul(m_OneUse(m_Select(m_Value(Cond), m_One(), m_AllOnes())),
                         m_Value(OtherOp)))) {
     bool HasAnyNoWrap = I.hasNoSignedWrap() || I.hasNoUnsignedWrap();
-    Value *Neg = Builder.CreateNeg(OtherOp, "", false, HasAnyNoWrap);
+    Value *Neg = Builder.CreateNeg(OtherOp, "", HasAnyNoWrap);
     return Builder.CreateSelect(Cond, OtherOp, Neg);
   }
   // mul (select Cond, -1, 1), OtherOp --> select Cond, -OtherOp, OtherOp
@@ -113,7 +113,7 @@ static Value *foldMulSelectToNegate(BinaryOperator &I,
   if (match(&I, m_c_Mul(m_OneUse(m_Select(m_Value(Cond), m_AllOnes(), m_One())),
                         m_Value(OtherOp)))) {
     bool HasAnyNoWrap = I.hasNoSignedWrap() || I.hasNoUnsignedWrap();
-    Value *Neg = Builder.CreateNeg(OtherOp, "", false, HasAnyNoWrap);
+    Value *Neg = Builder.CreateNeg(OtherOp, "", HasAnyNoWrap);
     return Builder.CreateSelect(Cond, Neg, OtherOp);
   }
 
@@ -452,9 +452,8 @@ Instruction *InstCombinerImpl::visitMul(BinaryOperator &I) {
   // mul Y, (sext X) -> select X, -Y, 0
   if (match(&I, m_c_Mul(m_OneUse(m_SExt(m_Value(X))), m_Value(Y))) &&
       X->getType()->isIntOrIntVectorTy(1))
-    return SelectInst::Create(
-        X, Builder.CreateNeg(Y, "", /*HasNUW=*/false, I.hasNoSignedWrap()),
-        ConstantInt::getNullValue(Op0->getType()));
+    return SelectInst::Create(X, Builder.CreateNeg(Y, "", I.hasNoSignedWrap()),
+                              ConstantInt::getNullValue(Op0->getType()));
 
   Constant *ImmC;
   if (match(Op1, m_ImmConstant(ImmC))) {
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp b/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
index f73679f9461b..cd8e47e1b391 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
@@ -140,7 +140,7 @@ std::array<Value *, 2> Negator::getSortedOperandsOfBinOp(Instruction *I) {
 
   // Integral constants can be freely negated.
   if (match(V, m_AnyIntegralConstant()))
-    return ConstantExpr::getNeg(cast<Constant>(V), /*HasNUW=*/false,
+    return ConstantExpr::getNeg(cast<Constant>(V),
                                 /*HasNSW=*/false);
 
   // If we have a non-instruction, then give up.
diff --git a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
index de3bfb57b538..fb334e8292eb 100644
--- a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
+++ b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
@@ -497,7 +497,7 @@ static bool processAbsIntrinsic(IntrinsicInst *II, LazyValueInfo *LVI) {
   // Is X in [IntMin, 0]?  NOTE: INT_MIN is fine!
   if (Range.getSignedMax().isNonPositive()) {
     IRBuilder<> B(II);
-    Value *NegX = B.CreateNeg(X, II->getName(), /*HasNUW=*/false,
+    Value *NegX = B.CreateNeg(X, II->getName(),
                               /*HasNSW=*/IsIntMinPoison);
     ++NumAbs;
     II->replaceAllUsesWith(NegX);
-- 
GitLab


From 3a2c70b3713a856ea416d92abdddb7893fca308b Mon Sep 17 00:00:00 2001
From: Zahira Ammarguellat <zahira.ammarguellat@intel.com>
Date: Tue, 26 Mar 2024 06:06:46 -0700
Subject: [PATCH 273/404] Fix printing of templated records. (#86339)

Fixed the printing of templated argument list and added test case.
---
 clang/lib/AST/TypePrinter.cpp           |  5 --
 clang/unittests/AST/DeclPrinterTest.cpp | 69 +++++++++++++++++++++++++
 2 files changed, 69 insertions(+), 5 deletions(-)

diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index 7032ff2f1846..d9504f9dcb38 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -2303,11 +2303,6 @@ printTo(raw_ostream &OS, ArrayRef<TA> Args, const PrintingPolicy &Policy,
     } else {
       if (!FirstArg)
         OS << Comma;
-      if (!Policy.SuppressTagKeyword &&
-          Argument.getKind() == TemplateArgument::Type &&
-          isa<TagType>(Argument.getAsType()))
-        OS << Argument.getAsType().getAsString();
-      else
         // Tries to print the argument with location info if exists.
         printArgument(Arg, Policy, ArgOS,
                       TemplateParameterList::shouldIncludeTypeForArgument(
diff --git a/clang/unittests/AST/DeclPrinterTest.cpp b/clang/unittests/AST/DeclPrinterTest.cpp
index e024c41e03b4..07fa02bd96e2 100644
--- a/clang/unittests/AST/DeclPrinterTest.cpp
+++ b/clang/unittests/AST/DeclPrinterTest.cpp
@@ -1386,6 +1386,75 @@ TEST(DeclPrinter, TestTemplateArgumentList16) {
   ASSERT_TRUE(PrintedDeclCXX11Matches(Code, "NT2", "int NT2 = 5"));
 }
 
+TEST(DeclPrinter, TestCXXRecordDecl17) {
+  ASSERT_TRUE(PrintedDeclCXX98Matches("template<typename T> struct Z {};"
+                                      "struct X {};"
+                                      "Z<X> A;",
+                                      "A", "Z<X> A"));
+  [](PrintingPolicy &Policy) { Policy.SuppressTagKeyword = false; };
+}
+
+TEST(DeclPrinter, TestCXXRecordDecl18) {
+  ASSERT_TRUE(PrintedDeclCXX98Matches("template<typename T> struct Z {};"
+                                      "struct X {};"
+                                      "Z<X> A;"
+                                      "template <typename T1, int>"
+                                      "struct Y{};"
+                                      "Y<Z<X>, 2> B;",
+                                      "B", "Y<Z<X>, 2> B"));
+  [](PrintingPolicy &Policy) { Policy.SuppressTagKeyword = false; };
+}
+
+TEST(DeclPrinter, TestCXXRecordDecl19) {
+  ASSERT_TRUE(PrintedDeclCXX98Matches("template<typename T> struct Z {};"
+                                      "struct X {};"
+                                      "Z<X> A;"
+                                      "template <typename T1, int>"
+                                      "struct Y{};"
+                                      "Y<Z<X>, 2> B;",
+                                      "B", "Y<Z<X>, 2> B"));
+  [](PrintingPolicy &Policy) { Policy.SuppressTagKeyword = true; };
+}
+TEST(DeclPrinter, TestCXXRecordDecl20) {
+  ASSERT_TRUE(PrintedDeclCXX98Matches(
+      "template <typename T, int N> class Inner;"
+      "template <typename T, int N>"
+      "class Inner{Inner(T val){}};"
+      "template <class InnerClass, int N> class Outer {"
+      "public:"
+      "struct NestedStruct {"
+      "int nestedValue;"
+      "NestedStruct(int val) : nestedValue(val) {}"
+      "};"
+      "InnerClass innerInstance;"
+      "Outer(const InnerClass &inner) : innerInstance(inner) {}"
+      "};"
+      "Outer<Inner<int, 10>, 5>::NestedStruct nestedInstance(100);",
+      "nestedInstance",
+      "Outer<Inner<int, 10>, 5>::NestedStruct nestedInstance(100)"));
+  [](PrintingPolicy &Policy) { Policy.SuppressTagKeyword = false; };
+}
+
+TEST(DeclPrinter, TestCXXRecordDecl21) {
+  ASSERT_TRUE(PrintedDeclCXX98Matches(
+      "template <typename T, int N> class Inner;"
+      "template <typename T, int N>"
+      "class Inner{Inner(T val){}};"
+      "template <class InnerClass, int N> class Outer {"
+      "public:"
+      "struct NestedStruct {"
+      "int nestedValue;"
+      "NestedStruct(int val) : nestedValue(val) {}"
+      "};"
+      "InnerClass innerInstance;"
+      "Outer(const InnerClass &inner) : innerInstance(inner) {}"
+      "};"
+      "Outer<Inner<int, 10>, 5>::NestedStruct nestedInstance(100);",
+      "nestedInstance",
+      "Outer<Inner<int, 10>, 5>::NestedStruct nestedInstance(100)"));
+  [](PrintingPolicy &Policy) { Policy.SuppressTagKeyword = true; };
+}
+
 TEST(DeclPrinter, TestFunctionParamUglified) {
   llvm::StringLiteral Code = R"cpp(
     class __c;
-- 
GitLab


From 93d33a1355438638232e66991f3a90c376c61fea Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Tue, 26 Mar 2024 21:32:57 +0800
Subject: [PATCH 274/404] [PatternMatch] Refactor `m_SpecificInt` to avoid
 constructing APInt. NFC. (#86259)

This patch passes APInt by const reference in m_SpecificInt instead of
by value. Specifically, it refactors `m_SpecificInt(uint64_t V)` to
avoid APInt construction and dangling reference.

I believe it is safe to pass the APInt by const reference into
`m_SpecificInt` even if it is a temporary.
See also https://en.cppreference.com/w/cpp/language/lifetime
> All temporary objects are destroyed as the last step in evaluating the
[full-expression](https://en.cppreference.com/w/cpp/language/expressions#Full-expressions)
that (lexically) contains the point where they were created

Compile-time impact:
https://llvm-compile-time-tracker.com/compare.php?from=d1f182c895728d89c5c3d198b133e212a5d9d4a3&to=7edf459b95ab2be33b70ec67faf87b3b8cc84f09&stat=instructions:u
---
 llvm/include/llvm/IR/PatternMatch.h | 35 ++++++++++++++++++++---------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 04feb1fd17fd..49f9ed5f6e7b 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -884,9 +884,9 @@ struct bind_const_intval_ty {
 /// Match a specified integer value or vector of all elements of that
 /// value.
 template <bool AllowUndefs> struct specific_intval {
-  APInt Val;
+  const APInt &Val;
 
-  specific_intval(APInt V) : Val(std::move(V)) {}
+  specific_intval(const APInt &V) : Val(V) {}
 
   template <typename ITy> bool match(ITy *V) {
     const auto *CI = dyn_cast<ConstantInt>(V);
@@ -898,22 +898,37 @@ template <bool AllowUndefs> struct specific_intval {
   }
 };
 
+template <bool AllowUndefs> struct specific_intval64 {
+  uint64_t Val;
+
+  specific_intval64(uint64_t V) : Val(V) {}
+
+  template <typename ITy> bool match(ITy *V) {
+    const auto *CI = dyn_cast<ConstantInt>(V);
+    if (!CI && V->getType()->isVectorTy())
+      if (const auto *C = dyn_cast<Constant>(V))
+        CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue(AllowUndefs));
+
+    return CI && CI->getValue() == Val;
+  }
+};
+
 /// Match a specific integer value or vector with all elements equal to
 /// the value.
-inline specific_intval<false> m_SpecificInt(APInt V) {
-  return specific_intval<false>(std::move(V));
+inline specific_intval<false> m_SpecificInt(const APInt &V) {
+  return specific_intval<false>(V);
 }
 
-inline specific_intval<false> m_SpecificInt(uint64_t V) {
-  return m_SpecificInt(APInt(64, V));
+inline specific_intval64<false> m_SpecificInt(uint64_t V) {
+  return specific_intval64<false>(V);
 }
 
-inline specific_intval<true> m_SpecificIntAllowUndef(APInt V) {
-  return specific_intval<true>(std::move(V));
+inline specific_intval<true> m_SpecificIntAllowUndef(const APInt &V) {
+  return specific_intval<true>(V);
 }
 
-inline specific_intval<true> m_SpecificIntAllowUndef(uint64_t V) {
-  return m_SpecificIntAllowUndef(APInt(64, V));
+inline specific_intval64<true> m_SpecificIntAllowUndef(uint64_t V) {
+  return specific_intval64<true>(V);
 }
 
 /// Match a ConstantInt and bind to its value.  This does not match
-- 
GitLab


From 308ed0233a3da5b3a7d646808c6470ac30870603 Mon Sep 17 00:00:00 2001
From: Il-Capitano <52455591+Il-Capitano@users.noreply.github.com>
Date: Tue, 26 Mar 2024 14:38:52 +0100
Subject: [PATCH 275/404] [Intrinsics] Make `patchpoint.i64` generic on its
 return type (#85911)

Currently patchpoints can only have two result types, `void` and `i64`.
This limits the result to general purpose registers.
This patch makes `patchpoint.i64` an overloadable intrinsic, allowing
result values that can fit in a single register (e.g. integers,
pointers, floats).
---
 llvm/include/llvm/IR/Intrinsics.td            |  10 +-
 llvm/lib/CodeGen/SelectionDAG/FastISel.cpp    |  25 +-
 .../SelectionDAG/SelectionDAGBuilder.cpp      |  16 +-
 llvm/lib/IR/Verifier.cpp                      |   9 +-
 .../AArch64/AArch64TargetTransformInfo.cpp    |   2 +-
 .../Target/PowerPC/PPCTargetTransformInfo.cpp |   2 +-
 .../SystemZ/SystemZTargetTransformInfo.cpp    |   2 +-
 .../lib/Target/X86/X86TargetTransformInfo.cpp |   2 +-
 .../lib/Transforms/Scalar/PlaceSafepoints.cpp |   2 +-
 llvm/test/CodeGen/AArch64/arm64-anyregcc.ll   | 225 +++++++++++++++++-
 llvm/test/CodeGen/AArch64/arm64-patchpoint.ll | 139 +++++++++++
 11 files changed, 404 insertions(+), 30 deletions(-)

diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 091f9b381079..d0ef9c25f39e 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -1590,11 +1590,11 @@ def int_experimental_patchpoint_void : Intrinsic<[],
                                                   llvm_ptr_ty, llvm_i32_ty,
                                                   llvm_vararg_ty],
                                                   [Throws]>;
-def int_experimental_patchpoint_i64 : Intrinsic<[llvm_i64_ty],
-                                                [llvm_i64_ty, llvm_i32_ty,
-                                                 llvm_ptr_ty, llvm_i32_ty,
-                                                 llvm_vararg_ty],
-                                                 [Throws]>;
+def int_experimental_patchpoint : Intrinsic<[llvm_any_ty],
+                                            [llvm_i64_ty, llvm_i32_ty,
+                                             llvm_ptr_ty, llvm_i32_ty,
+                                             llvm_vararg_ty],
+                                            [Throws]>;
 
 
 //===------------------------ Garbage Collection Intrinsics ---------------===//
diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index 27b8472ddb73..51562400b77d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -752,17 +752,25 @@ FastISel::CallLoweringInfo &FastISel::CallLoweringInfo::setCallee(
 }
 
 bool FastISel::selectPatchpoint(const CallInst *I) {
-  // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
-  //                                                 i32 <numBytes>,
-  //                                                 i8* <target>,
-  //                                                 i32 <numArgs>,
-  //                                                 [Args...],
-  //                                                 [live variables...])
+  // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
+  //                                         i32 <numBytes>,
+  //                                         i8* <target>,
+  //                                         i32 <numArgs>,
+  //                                         [Args...],
+  //                                         [live variables...])
   CallingConv::ID CC = I->getCallingConv();
   bool IsAnyRegCC = CC == CallingConv::AnyReg;
   bool HasDef = !I->getType()->isVoidTy();
   Value *Callee = I->getOperand(PatchPointOpers::TargetPos)->stripPointerCasts();
 
+  // Check if we can lower the return type when using anyregcc.
+  MVT ValueType;
+  if (IsAnyRegCC && HasDef) {
+    ValueType = TLI.getSimpleValueType(DL, I->getType(), /*AllowUnknown=*/true);
+    if (ValueType == MVT::Other)
+      return false;
+  }
+
   // Get the real number of arguments participating in the call <numArgs>
   assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) &&
          "Expected a constant integer.");
@@ -790,7 +798,8 @@ bool FastISel::selectPatchpoint(const CallInst *I) {
   // Add an explicit result reg if we use the anyreg calling convention.
   if (IsAnyRegCC && HasDef) {
     assert(CLI.NumResultRegs == 0 && "Unexpected result register.");
-    CLI.ResultReg = createResultReg(TLI.getRegClassFor(MVT::i64));
+    assert(ValueType.isValid());
+    CLI.ResultReg = createResultReg(TLI.getRegClassFor(ValueType));
     CLI.NumResultRegs = 1;
     Ops.push_back(MachineOperand::CreateReg(CLI.ResultReg, /*isDef=*/true));
   }
@@ -1464,7 +1473,7 @@ bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) {
   case Intrinsic::experimental_stackmap:
     return selectStackmap(II);
   case Intrinsic::experimental_patchpoint_void:
-  case Intrinsic::experimental_patchpoint_i64:
+  case Intrinsic::experimental_patchpoint:
     return selectPatchpoint(II);
 
   case Intrinsic::xray_customevent:
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index ae6bd7e938e8..f99ab55a14ba 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3331,7 +3331,7 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
           EHPadMBB->setMachineBlockAddressTaken();
       break;
     case Intrinsic::experimental_patchpoint_void:
-    case Intrinsic::experimental_patchpoint_i64:
+    case Intrinsic::experimental_patchpoint:
       visitPatchpoint(I, EHPadBB);
       break;
     case Intrinsic::experimental_gc_statepoint:
@@ -7445,7 +7445,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     visitStackmap(I);
     return;
   case Intrinsic::experimental_patchpoint_void:
-  case Intrinsic::experimental_patchpoint_i64:
+  case Intrinsic::experimental_patchpoint:
     visitPatchpoint(I);
     return;
   case Intrinsic::experimental_gc_statepoint:
@@ -10257,12 +10257,12 @@ void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
 /// Lower llvm.experimental.patchpoint directly to its target opcode.
 void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
                                           const BasicBlock *EHPadBB) {
-  // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
-  //                                                 i32 <numBytes>,
-  //                                                 i8* <target>,
-  //                                                 i32 <numArgs>,
-  //                                                 [Args...],
-  //                                                 [live variables...])
+  // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
+  //                                         i32 <numBytes>,
+  //                                         i8* <target>,
+  //                                         i32 <numArgs>,
+  //                                         [Args...],
+  //                                         [live variables...])
 
   CallingConv::ID CC = CB.getCallingConv();
   bool IsAnyRegCC = CC == CallingConv::AnyReg;
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 819722566831..33f358440a31 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -5016,7 +5016,7 @@ void Verifier::visitInstruction(Instruction &I) {
                 F->getIntrinsicID() == Intrinsic::coro_await_suspend_handle ||
                 F->getIntrinsicID() ==
                     Intrinsic::experimental_patchpoint_void ||
-                F->getIntrinsicID() == Intrinsic::experimental_patchpoint_i64 ||
+                F->getIntrinsicID() == Intrinsic::experimental_patchpoint ||
                 F->getIntrinsicID() == Intrinsic::experimental_gc_statepoint ||
                 F->getIntrinsicID() == Intrinsic::wasm_rethrow ||
                 IsAttachedCallOperand(F, CBI, i),
@@ -5661,6 +5661,13 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
     }
     break;
   }
+  case Intrinsic::experimental_patchpoint: {
+    if (Call.getCallingConv() == CallingConv::AnyReg) {
+      Check(Call.getType()->isSingleValueType(),
+            "patchpoint: invalid return type used with anyregcc", Call);
+    }
+    break;
+  }
   case Intrinsic::eh_exceptioncode:
   case Intrinsic::eh_exceptionpointer: {
     Check(isa<CatchPadInst>(Call.getArgOperand(0)),
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 7a86c5c60881..ee7137b92445 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -479,7 +479,7 @@ AArch64TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
       return TTI::TCC_Free;
     break;
   case Intrinsic::experimental_patchpoint_void:
-  case Intrinsic::experimental_patchpoint_i64:
+  case Intrinsic::experimental_patchpoint:
     if ((Idx < 4) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue())))
       return TTI::TCC_Free;
     break;
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index 7adf1adcc647..57e1019adb74 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -219,7 +219,7 @@ InstructionCost PPCTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
       return TTI::TCC_Free;
     break;
   case Intrinsic::experimental_patchpoint_void:
-  case Intrinsic::experimental_patchpoint_i64:
+  case Intrinsic::experimental_patchpoint:
     if ((Idx < 4) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue())))
       return TTI::TCC_Free;
     break;
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index e4adb7be5649..5bdbaf47064d 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -265,7 +265,7 @@ SystemZTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
       return TTI::TCC_Free;
     break;
   case Intrinsic::experimental_patchpoint_void:
-  case Intrinsic::experimental_patchpoint_i64:
+  case Intrinsic::experimental_patchpoint:
     if ((Idx < 4) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue())))
       return TTI::TCC_Free;
     break;
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index e9bdc6ab5bea..2ec29463d2fa 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -5659,7 +5659,7 @@ InstructionCost X86TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
       return TTI::TCC_Free;
     break;
   case Intrinsic::experimental_patchpoint_void:
-  case Intrinsic::experimental_patchpoint_i64:
+  case Intrinsic::experimental_patchpoint:
     if ((Idx < 4) || (Imm.getBitWidth() <= 64 && Imm.isSignedIntN(64)))
       return TTI::TCC_Free;
     break;
diff --git a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp
index 436a85f62df6..f5c9aaa4f20b 100644
--- a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp
+++ b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp
@@ -517,7 +517,7 @@ static bool doesNotRequireEntrySafepointBefore(CallBase *Call) {
     switch (II->getIntrinsicID()) {
     case Intrinsic::experimental_gc_statepoint:
     case Intrinsic::experimental_patchpoint_void:
-    case Intrinsic::experimental_patchpoint_i64:
+    case Intrinsic::experimental_patchpoint:
       // The can wrap an actual call which may grow the stack by an unbounded
       // amount or run forever.
       return false;
diff --git a/llvm/test/CodeGen/AArch64/arm64-anyregcc.ll b/llvm/test/CodeGen/AArch64/arm64-anyregcc.ll
index 225d4c602f18..cb6586718450 100644
--- a/llvm/test/CodeGen/AArch64/arm64-anyregcc.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-anyregcc.ll
@@ -1,6 +1,6 @@
 ; RUN: llc < %s -debug-entry-values -mtriple=arm64-apple-darwin | FileCheck %s
 
-; Stackmap Header: no constants - 6 callsites
+; Stackmap Header: no constants - 18 callsites
 ; CHECK-LABEL: .section	__LLVM_STACKMAPS,__llvm_stackmaps
 ; CHECK-NEXT:  __LLVM_StackMaps:
 ; Header
@@ -8,11 +8,11 @@
 ; CHECK-NEXT:   .byte 0
 ; CHECK-NEXT:   .short 0
 ; Num Functions
-; CHECK-NEXT:   .long 8
+; CHECK-NEXT:   .long 18
 ; Num LargeConstants
 ; CHECK-NEXT:   .long 0
 ; Num Callsites
-; CHECK-NEXT:   .long 8
+; CHECK-NEXT:   .long 18
 
 ; Functions and stack size
 ; CHECK-NEXT:   .quad _test
@@ -39,6 +39,36 @@
 ; CHECK-NEXT:   .quad _patchpoint_spillargs
 ; CHECK-NEXT:   .quad 128
 ; CHECK-NEXT:   .quad 1
+; CHECK-NEXT:   .quad _generic_test_i32
+; CHECK-NEXT:   .quad 16
+; CHECK-NEXT:   .quad 1
+; CHECK-NEXT:   .quad _generic_test_i64
+; CHECK-NEXT:   .quad 16
+; CHECK-NEXT:   .quad 1
+; CHECK-NEXT:   .quad _generic_test_p0
+; CHECK-NEXT:   .quad 16
+; CHECK-NEXT:   .quad 1
+; CHECK-NEXT:   .quad _generic_test_f16
+; CHECK-NEXT:   .quad 16
+; CHECK-NEXT:   .quad 1
+; CHECK-NEXT:   .quad _generic_test_f32
+; CHECK-NEXT:   .quad 16
+; CHECK-NEXT:   .quad 1
+; CHECK-NEXT:   .quad _generic_test_f64
+; CHECK-NEXT:   .quad 16
+; CHECK-NEXT:   .quad 1
+; CHECK-NEXT:   .quad _generic_test_v16i8
+; CHECK-NEXT:   .quad 16
+; CHECK-NEXT:   .quad 1
+; CHECK-NEXT:   .quad _generic_test_v4i32
+; CHECK-NEXT:   .quad 16
+; CHECK-NEXT:   .quad 1
+; CHECK-NEXT:   .quad _generic_test_v4f32
+; CHECK-NEXT:   .quad 16
+; CHECK-NEXT:   .quad 1
+; CHECK-NEXT:   .quad _generic_test_v2f64
+; CHECK-NEXT:   .quad 16
+; CHECK-NEXT:   .quad 1
 
 
 ; test
@@ -457,5 +487,194 @@ entry:
   ret i64 %result
 }
 
+; generic_test_i32
+; CHECK-LABEL:  .long   L{{.*}}-_generic_test_i32
+; CHECK-NEXT:   .short  0
+; 1 location
+; CHECK-NEXT:   .short  1
+; Loc 0: Register <-- this is the return register
+; CHECK-NEXT:   .byte 1
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .short 4
+; CHECK-NEXT:   .short {{[0-9]+}}
+; CHECK-NEXT:   .short 0
+; CHECK-NEXT:   .long 0
+define i32 @generic_test_i32() nounwind ssp uwtable {
+entry:
+  %ret = call anyregcc i32 (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.i32(i64 14, i32 20, ptr null, i32 0)
+  ret i32 %ret
+}
+
+; generic_test_i64
+; CHECK-LABEL:  .long   L{{.*}}-_generic_test_i64
+; CHECK-NEXT:   .short  0
+; 1 location
+; CHECK-NEXT:   .short  1
+; Loc 0: Register <-- this is the return register
+; CHECK-NEXT:   .byte 1
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .short 8
+; CHECK-NEXT:   .short {{[0-9]+}}
+; CHECK-NEXT:   .short 0
+; CHECK-NEXT:   .long 0
+define i64 @generic_test_i64() nounwind ssp uwtable {
+entry:
+  %ret = call anyregcc i64 (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.i64(i64 14, i32 20, ptr null, i32 0)
+  ret i64 %ret
+}
+
+; generic_test_p0
+; CHECK-LABEL:  .long   L{{.*}}-_generic_test_p0
+; CHECK-NEXT:   .short  0
+; 1 location
+; CHECK-NEXT:   .short  1
+; Loc 0: Register <-- this is the return register
+; CHECK-NEXT:   .byte 1
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .short 8
+; CHECK-NEXT:   .short {{[0-9]+}}
+; CHECK-NEXT:   .short 0
+; CHECK-NEXT:   .long 0
+define ptr @generic_test_p0() nounwind ssp uwtable {
+entry:
+  %ret = call anyregcc ptr (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.p0(i64 14, i32 20, ptr null, i32 0)
+  ret ptr %ret
+}
+
+; generic_test_f16
+; CHECK-LABEL:  .long   L{{.*}}-_generic_test_f16
+; CHECK-NEXT:   .short  0
+; 1 location
+; CHECK-NEXT:   .short  1
+; Loc 0: Register <-- this is the return register
+; CHECK-NEXT:   .byte 1
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .short 2
+; CHECK-NEXT:   .short {{[0-9]+}}
+; CHECK-NEXT:   .short 0
+; CHECK-NEXT:   .long 0
+define half @generic_test_f16() nounwind ssp uwtable {
+entry:
+  %ret = call anyregcc half (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.f16(i64 14, i32 20, ptr null, i32 0)
+  ret half %ret
+}
+
+; generic_test_f32
+; CHECK-LABEL:  .long   L{{.*}}-_generic_test_f32
+; CHECK-NEXT:   .short  0
+; 1 location
+; CHECK-NEXT:   .short  1
+; Loc 0: Register <-- this is the return register
+; CHECK-NEXT:   .byte 1
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .short 4
+; CHECK-NEXT:   .short {{[0-9]+}}
+; CHECK-NEXT:   .short 0
+; CHECK-NEXT:   .long 0
+define float @generic_test_f32() nounwind ssp uwtable {
+entry:
+  %ret = call anyregcc float (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.f32(i64 14, i32 20, ptr null, i32 0)
+  ret float %ret
+}
+
+; generic_test_f64
+; CHECK-LABEL:  .long   L{{.*}}-_generic_test_f64
+; CHECK-NEXT:   .short  0
+; 1 location
+; CHECK-NEXT:   .short  1
+; Loc 0: Register <-- this is the return register
+; CHECK-NEXT:   .byte 1
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .short 8
+; CHECK-NEXT:   .short {{[0-9]+}}
+; CHECK-NEXT:   .short 0
+; CHECK-NEXT:   .long 0
+define double @generic_test_f64() nounwind ssp uwtable {
+entry:
+  %ret = call anyregcc double (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.f64(i64 14, i32 20, ptr null, i32 0)
+  ret double %ret
+}
+
+; generic_test_v16i8
+; CHECK-LABEL:  .long   L{{.*}}-_generic_test_v16i8
+; CHECK-NEXT:   .short  0
+; 1 location
+; CHECK-NEXT:   .short  1
+; Loc 0: Register <-- this is the return register
+; CHECK-NEXT:   .byte 1
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .short 16
+; CHECK-NEXT:   .short {{[0-9]+}}
+; CHECK-NEXT:   .short 0
+; CHECK-NEXT:   .long 0
+define <16 x i8> @generic_test_v16i8() nounwind ssp uwtable {
+entry:
+  %ret = call anyregcc <16 x i8> (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.v16i8(i64 14, i32 20, ptr null, i32 0)
+  ret <16 x i8> %ret
+}
+
+; generic_test_v4i32
+; CHECK-LABEL:  .long   L{{.*}}-_generic_test_v4i32
+; CHECK-NEXT:   .short  0
+; 1 location
+; CHECK-NEXT:   .short  1
+; Loc 0: Register <-- this is the return register
+; CHECK-NEXT:   .byte 1
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .short 16
+; CHECK-NEXT:   .short {{[0-9]+}}
+; CHECK-NEXT:   .short 0
+; CHECK-NEXT:   .long 0
+define <4 x i32> @generic_test_v4i32() nounwind ssp uwtable {
+entry:
+  %ret = call anyregcc <4 x i32> (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.v4i32(i64 14, i32 20, ptr null, i32 0)
+  ret <4 x i32> %ret
+}
+
+; generic_test_v4f32
+; CHECK-LABEL:  .long   L{{.*}}-_generic_test_v4f32
+; CHECK-NEXT:   .short  0
+; 1 location
+; CHECK-NEXT:   .short  1
+; Loc 0: Register <-- this is the return register
+; CHECK-NEXT:   .byte 1
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .short 16
+; CHECK-NEXT:   .short {{[0-9]+}}
+; CHECK-NEXT:   .short 0
+; CHECK-NEXT:   .long 0
+define <4 x float> @generic_test_v4f32() nounwind ssp uwtable {
+entry:
+  %ret = call anyregcc <4 x float> (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.v4f32(i64 14, i32 20, ptr null, i32 0)
+  ret <4 x float> %ret
+}
+
+; generic_test_v2f64
+; CHECK-LABEL:  .long   L{{.*}}-_generic_test_v2f64
+; CHECK-NEXT:   .short  0
+; 1 location
+; CHECK-NEXT:   .short  1
+; Loc 0: Register <-- this is the return register
+; CHECK-NEXT:   .byte 1
+; CHECK-NEXT:   .byte 0
+; CHECK-NEXT:   .short 16
+; CHECK-NEXT:   .short {{[0-9]+}}
+; CHECK-NEXT:   .short 0
+; CHECK-NEXT:   .long 0
+define <2 x double> @generic_test_v2f64() nounwind ssp uwtable {
+entry:
+  %ret = call anyregcc <2 x double> (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.v2f64(i64 14, i32 20, ptr null, i32 0)
+  ret <2 x double> %ret
+}
+
 declare void @llvm.experimental.patchpoint.void(i64, i32, ptr, i32, ...)
+declare i32 @llvm.experimental.patchpoint.i32(i64, i32, ptr, i32, ...)
 declare i64 @llvm.experimental.patchpoint.i64(i64, i32, ptr, i32, ...)
+declare ptr @llvm.experimental.patchpoint.p0(i64, i32, ptr, i32, ...)
+declare half @llvm.experimental.patchpoint.f16(i64, i32, ptr, i32, ...)
+declare float @llvm.experimental.patchpoint.f32(i64, i32, ptr, i32, ...)
+declare double @llvm.experimental.patchpoint.f64(i64, i32, ptr, i32, ...)
+declare <16 x i8> @llvm.experimental.patchpoint.v16i8(i64, i32, ptr, i32, ...)
+declare <4 x i32> @llvm.experimental.patchpoint.v4i32(i64, i32, ptr, i32, ...)
+declare <4 x float> @llvm.experimental.patchpoint.v4f32(i64, i32, ptr, i32, ...)
+declare <2 x double> @llvm.experimental.patchpoint.v2f64(i64, i32, ptr, i32, ...)
diff --git a/llvm/test/CodeGen/AArch64/arm64-patchpoint.ll b/llvm/test/CodeGen/AArch64/arm64-patchpoint.ll
index c58f4b102909..f948d78723e9 100644
--- a/llvm/test/CodeGen/AArch64/arm64-patchpoint.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-patchpoint.ll
@@ -79,6 +79,145 @@ entry:
   ret void
 }
 
+; Test register allocation for an i32 result value of patchpoint.
+define i32 @generic_patchpoint_i32() {
+entry:
+; CHECK-LABEL: generic_patchpoint_i32:
+; CHECK:      Ltmp
+; CHECK-NEXT: nop
+; The return value is already in w0.
+; CHECK-NEXT: ldp
+; CHECK-NEXT: ret
+  %result = tail call i32 (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.i32(i64 5, i32 4, ptr null, i32 0)
+  ret i32 %result
+}
+
+; Test register allocation for an i64 result value of patchpoint.
+define i64 @generic_patchpoint_i64() {
+entry:
+; CHECK-LABEL: generic_patchpoint_i64:
+; CHECK:      Ltmp
+; CHECK-NEXT: nop
+; The return value is already in x0.
+; CHECK-NEXT: ldp
+; CHECK-NEXT: ret
+  %result = tail call i64 (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.i64(i64 5, i32 4, ptr null, i32 0)
+  ret i64 %result
+}
+
+; Test register allocation for a ptr result value of patchpoint.
+define ptr @generic_patchpoint_p0() {
+entry:
+; CHECK-LABEL: generic_patchpoint_p0:
+; CHECK:      Ltmp
+; CHECK-NEXT: nop
+; The return value is already in x0.
+; CHECK-NEXT: ldp
+; CHECK-NEXT: ret
+  %result = tail call ptr (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.p0(i64 5, i32 4, ptr null, i32 0)
+  ret ptr %result
+}
+
+; Test register allocation for a half result value of patchpoint.
+define half @generic_patchpoint_f16() {
+entry:
+; CHECK-LABEL: generic_patchpoint_f16:
+; CHECK:      Ltmp
+; CHECK-NEXT: nop
+; The return value is already in h0.
+; CHECK-NEXT: ldp
+; CHECK-NEXT: ret
+  %result = tail call half (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.f16(i64 5, i32 4, ptr null, i32 0)
+  ret half %result
+}
+
+; Test register allocation for a float result value of patchpoint.
+define float @generic_patchpoint_f32() {
+entry:
+; CHECK-LABEL: generic_patchpoint_f32:
+; CHECK:      Ltmp
+; CHECK-NEXT: nop
+; The return value is already in s0.
+; CHECK-NEXT: ldp
+; CHECK-NEXT: ret
+  %result = tail call float (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.f32(i64 5, i32 4, ptr null, i32 0)
+  ret float %result
+}
+
+; Test register allocation for a double result value of patchpoint.
+define double @generic_patchpoint_f64() {
+entry:
+; CHECK-LABEL: generic_patchpoint_f64:
+; CHECK:      Ltmp
+; CHECK-NEXT: nop
+; The return value is already in d0.
+; CHECK-NEXT: ldp
+; CHECK-NEXT: ret
+  %result = tail call double (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.f64(i64 5, i32 4, ptr null, i32 0)
+  ret double %result
+}
+
+; Test register allocation for a <16 x i8> result value of patchpoint.
+define <16 x i8> @generic_patchpoint_v16i8() {
+entry:
+; CHECK-LABEL: generic_patchpoint_v16i8:
+; CHECK:      Ltmp
+; CHECK-NEXT: nop
+; The return value is already in v0.16b.
+; CHECK-NEXT: ldp
+; CHECK-NEXT: ret
+  %result = tail call <16 x i8> (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.v16i8(i64 5, i32 4, ptr null, i32 0)
+  ret <16 x i8> %result
+}
+
+; Test register allocation for a <4 x i32> result value of patchpoint.
+define <4 x i32> @generic_patchpoint_v4i32() {
+entry:
+; CHECK-LABEL: generic_patchpoint_v4i32:
+; CHECK:      Ltmp
+; CHECK-NEXT: nop
+; The return value is already in v0.4s.
+; CHECK-NEXT: ldp
+; CHECK-NEXT: ret
+  %result = tail call <4 x i32> (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.v4i32(i64 5, i32 4, ptr null, i32 0)
+  ret <4 x i32> %result
+}
+
+; Test register allocation for a <4 x float> result value of patchpoint.
+define <4 x float> @generic_patchpoint_v4f32() {
+entry:
+; CHECK-LABEL: generic_patchpoint_v4f32:
+; CHECK:      Ltmp
+; CHECK-NEXT: nop
+; The return value is already in v0.4s.
+; CHECK-NEXT: ldp
+; CHECK-NEXT: ret
+  %result = tail call <4 x float> (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.v4f32(i64 5, i32 4, ptr null, i32 0)
+  ret <4 x float> %result
+}
+
+; Test register allocation for a <2 x double> result value of patchpoint.
+define <2 x double> @generic_patchpoint_v2f64() {
+entry:
+; CHECK-LABEL: generic_patchpoint_v2f64:
+; CHECK:      Ltmp
+; CHECK-NEXT: nop
+; The return value is already in v0.2d.
+; CHECK-NEXT: ldp
+; CHECK-NEXT: ret
+  %result = tail call <2 x double> (i64, i32, ptr, i32, ...) @llvm.experimental.patchpoint.v2f64(i64 5, i32 4, ptr null, i32 0)
+  ret <2 x double> %result
+}
+
 declare void @llvm.experimental.stackmap(i64, i32, ...)
 declare void @llvm.experimental.patchpoint.void(i64, i32, ptr, i32, ...)
+declare i32 @llvm.experimental.patchpoint.i32(i64, i32, ptr, i32, ...)
 declare i64 @llvm.experimental.patchpoint.i64(i64, i32, ptr, i32, ...)
+declare ptr @llvm.experimental.patchpoint.p0(i64, i32, ptr, i32, ...)
+declare half @llvm.experimental.patchpoint.f16(i64, i32, ptr, i32, ...)
+declare float @llvm.experimental.patchpoint.f32(i64, i32, ptr, i32, ...)
+declare double @llvm.experimental.patchpoint.f64(i64, i32, ptr, i32, ...)
+declare <16 x i8> @llvm.experimental.patchpoint.v16i8(i64, i32, ptr, i32, ...)
+declare <4 x i32> @llvm.experimental.patchpoint.v4i32(i64, i32, ptr, i32, ...)
+declare <4 x float> @llvm.experimental.patchpoint.v4f32(i64, i32, ptr, i32, ...)
+declare <2 x double> @llvm.experimental.patchpoint.v2f64(i64, i32, ptr, i32, ...)
-- 
GitLab


From dfde6e89ecc10b1f1eebdb0e409ef1a084030a6c Mon Sep 17 00:00:00 2001
From: Hui <hui.xie1990@gmail.com>
Date: Tue, 26 Mar 2024 13:39:20 +0000
Subject: [PATCH 276/404] [libc++][test] Remove tests that testing
 std::variant<T&> (#84222)

Fixes #83600
---
 .../variant/variant.get/get_if_index.pass.cpp |  55 --------
 .../variant/variant.get/get_if_type.pass.cpp  |  55 --------
 .../variant/variant.get/get_index.pass.cpp    | 121 -----------------
 .../variant/variant.get/get_type.pass.cpp     | 125 ------------------
 .../variant_alternative.pass.cpp              |  10 --
 .../variant.variant/variant.assign/T.pass.cpp |  29 ----
 .../variant.variant/variant.ctor/T.pass.cpp   |  70 +++-------
 .../variant.ctor/default.pass.cpp             |   6 -
 .../variant.mod/emplace_index_args.pass.cpp   |  58 --------
 .../variant.mod/emplace_type_args.pass.cpp    |  59 ---------
 .../variant.visit.member/visit.pass.cpp       |  33 -----
 .../visit_return_type.pass.cpp                |  32 -----
 .../variant/variant.visit/visit.pass.cpp      |  30 -----
 .../variant.visit/visit_return_type.pass.cpp  |  30 -----
 libcxx/test/support/variant_test_helpers.h    |   3 -
 15 files changed, 21 insertions(+), 695 deletions(-)

diff --git a/libcxx/test/std/utilities/variant/variant.get/get_if_index.pass.cpp b/libcxx/test/std/utilities/variant/variant.get/get_if_index.pass.cpp
index 3333d2a993ec..09e2e85abe66 100644
--- a/libcxx/test/std/utilities/variant/variant.get/get_if_index.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.get/get_if_index.pass.cpp
@@ -44,30 +44,6 @@ void test_const_get_if() {
     static_assert(*std::get_if<1>(&v) == 42, "");
     static_assert(std::get_if<0>(&v) == nullptr, "");
   }
-// FIXME: Remove these once reference support is reinstated
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int &>;
-    int x = 42;
-    const V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get_if<0>(&v)), int *);
-    assert(std::get_if<0>(&v) == &x);
-  }
-  {
-    using V = std::variant<int &&>;
-    int x = 42;
-    const V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get_if<0>(&v)), int *);
-    assert(std::get_if<0>(&v) == &x);
-  }
-  {
-    using V = std::variant<const int &&>;
-    int x = 42;
-    const V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get_if<0>(&v)), const int *);
-    assert(std::get_if<0>(&v) == &x);
-  }
-#endif
 }
 
 void test_get_if() {
@@ -91,37 +67,6 @@ void test_get_if() {
     assert(*std::get_if<1>(&v) == 42);
     assert(std::get_if<0>(&v) == nullptr);
   }
-// FIXME: Remove these once reference support is reinstated
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int &>;
-    int x = 42;
-    V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get_if<0>(&v)), int *);
-    assert(std::get_if<0>(&v) == &x);
-  }
-  {
-    using V = std::variant<const int &>;
-    int x = 42;
-    V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get_if<0>(&v)), const int *);
-    assert(std::get_if<0>(&v) == &x);
-  }
-  {
-    using V = std::variant<int &&>;
-    int x = 42;
-    V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get_if<0>(&v)), int *);
-    assert(std::get_if<0>(&v) == &x);
-  }
-  {
-    using V = std::variant<const int &&>;
-    int x = 42;
-    V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get_if<0>(&v)), const int *);
-    assert(std::get_if<0>(&v) == &x);
-  }
-#endif
 }
 
 int main(int, char**) {
diff --git a/libcxx/test/std/utilities/variant/variant.get/get_if_type.pass.cpp b/libcxx/test/std/utilities/variant/variant.get/get_if_type.pass.cpp
index b81e000f5bf6..c4fefc74e62a 100644
--- a/libcxx/test/std/utilities/variant/variant.get/get_if_type.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.get/get_if_type.pass.cpp
@@ -42,30 +42,6 @@ void test_const_get_if() {
     static_assert(*std::get_if<const long>(&v) == 42, "");
     static_assert(std::get_if<int>(&v) == nullptr, "");
   }
-// FIXME: Remove these once reference support is reinstated
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int &>;
-    int x = 42;
-    const V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get_if<int &>(&v)), int *);
-    assert(std::get_if<int &>(&v) == &x);
-  }
-  {
-    using V = std::variant<int &&>;
-    int x = 42;
-    const V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get_if<int &&>(&v)), int *);
-    assert(std::get_if<int &&>(&v) == &x);
-  }
-  {
-    using V = std::variant<const int &&>;
-    int x = 42;
-    const V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get_if<const int &&>(&v)), const int *);
-    assert(std::get_if<const int &&>(&v) == &x);
-  }
-#endif
 }
 
 void test_get_if() {
@@ -89,37 +65,6 @@ void test_get_if() {
     assert(*std::get_if<const long>(&v) == 42);
     assert(std::get_if<int>(&v) == nullptr);
   }
-// FIXME: Remove these once reference support is reinstated
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int &>;
-    int x = 42;
-    V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get_if<int &>(&v)), int *);
-    assert(std::get_if<int &>(&v) == &x);
-  }
-  {
-    using V = std::variant<const int &>;
-    int x = 42;
-    V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get_if<const int &>(&v)), const int *);
-    assert(std::get_if<const int &>(&v) == &x);
-  }
-  {
-    using V = std::variant<int &&>;
-    int x = 42;
-    V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get_if<int &&>(&v)), int *);
-    assert(std::get_if<int &&>(&v) == &x);
-  }
-  {
-    using V = std::variant<const int &&>;
-    int x = 42;
-    V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get_if<const int &&>(&v)), const int *);
-    assert(std::get_if<const int &&>(&v) == &x);
-  }
-#endif
 }
 
 int main(int, char**) {
diff --git a/libcxx/test/std/utilities/variant/variant.get/get_index.pass.cpp b/libcxx/test/std/utilities/variant/variant.get/get_index.pass.cpp
index 97c7ff0ed095..7ec9d8827f6b 100644
--- a/libcxx/test/std/utilities/variant/variant.get/get_index.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.get/get_index.pass.cpp
@@ -60,30 +60,6 @@ void test_const_lvalue_get() {
     ASSERT_SAME_TYPE(decltype(std::get<1>(v)), const long &);
     assert(std::get<1>(v) == 42);
   }
-// FIXME: Remove these once reference support is reinstated
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int &>;
-    int x = 42;
-    const V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get<0>(v)), int &);
-    assert(&std::get<0>(v) == &x);
-  }
-  {
-    using V = std::variant<int &&>;
-    int x = 42;
-    const V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get<0>(v)), int &);
-    assert(&std::get<0>(v) == &x);
-  }
-  {
-    using V = std::variant<const int &&>;
-    int x = 42;
-    const V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get<0>(v)), const int &);
-    assert(&std::get<0>(v) == &x);
-  }
-#endif
 }
 
 void test_lvalue_get() {
@@ -100,37 +76,6 @@ void test_lvalue_get() {
     ASSERT_SAME_TYPE(decltype(std::get<1>(v)), const long &);
     assert(std::get<1>(v) == 42);
   }
-// FIXME: Remove these once reference support is reinstated
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int &>;
-    int x = 42;
-    V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get<0>(v)), int &);
-    assert(&std::get<0>(v) == &x);
-  }
-  {
-    using V = std::variant<const int &>;
-    int x = 42;
-    V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get<0>(v)), const int &);
-    assert(&std::get<0>(v) == &x);
-  }
-  {
-    using V = std::variant<int &&>;
-    int x = 42;
-    V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get<0>(v)), int &);
-    assert(&std::get<0>(v) == &x);
-  }
-  {
-    using V = std::variant<const int &&>;
-    int x = 42;
-    V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get<0>(v)), const int &);
-    assert(&std::get<0>(v) == &x);
-  }
-#endif
 }
 
 void test_rvalue_get() {
@@ -147,39 +92,6 @@ void test_rvalue_get() {
     ASSERT_SAME_TYPE(decltype(std::get<1>(std::move(v))), const long &&);
     assert(std::get<1>(std::move(v)) == 42);
   }
-// FIXME: Remove these once reference support is reinstated
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int &>;
-    int x = 42;
-    V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get<0>(std::move(v))), int &);
-    assert(&std::get<0>(std::move(v)) == &x);
-  }
-  {
-    using V = std::variant<const int &>;
-    int x = 42;
-    V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get<0>(std::move(v))), const int &);
-    assert(&std::get<0>(std::move(v)) == &x);
-  }
-  {
-    using V = std::variant<int &&>;
-    int x = 42;
-    V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get<0>(std::move(v))), int &&);
-    int &&xref = std::get<0>(std::move(v));
-    assert(&xref == &x);
-  }
-  {
-    using V = std::variant<const int &&>;
-    int x = 42;
-    V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get<0>(std::move(v))), const int &&);
-    const int &&xref = std::get<0>(std::move(v));
-    assert(&xref == &x);
-  }
-#endif
 }
 
 void test_const_rvalue_get() {
@@ -196,39 +108,6 @@ void test_const_rvalue_get() {
     ASSERT_SAME_TYPE(decltype(std::get<1>(std::move(v))), const long &&);
     assert(std::get<1>(std::move(v)) == 42);
   }
-// FIXME: Remove these once reference support is reinstated
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int &>;
-    int x = 42;
-    const V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get<0>(std::move(v))), int &);
-    assert(&std::get<0>(std::move(v)) == &x);
-  }
-  {
-    using V = std::variant<const int &>;
-    int x = 42;
-    const V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get<0>(std::move(v))), const int &);
-    assert(&std::get<0>(std::move(v)) == &x);
-  }
-  {
-    using V = std::variant<int &&>;
-    int x = 42;
-    const V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get<0>(std::move(v))), int &&);
-    int &&xref = std::get<0>(std::move(v));
-    assert(&xref == &x);
-  }
-  {
-    using V = std::variant<const int &&>;
-    int x = 42;
-    const V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get<0>(std::move(v))), const int &&);
-    const int &&xref = std::get<0>(std::move(v));
-    assert(&xref == &x);
-  }
-#endif
 }
 
 template <std::size_t I> using Idx = std::integral_constant<std::size_t, I>;
diff --git a/libcxx/test/std/utilities/variant/variant.get/get_type.pass.cpp b/libcxx/test/std/utilities/variant/variant.get/get_type.pass.cpp
index d5e54d41e2f7..3485122c98f0 100644
--- a/libcxx/test/std/utilities/variant/variant.get/get_type.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.get/get_type.pass.cpp
@@ -54,30 +54,6 @@ void test_const_lvalue_get() {
     ASSERT_SAME_TYPE(decltype(std::get<const long>(v)), const long &);
     assert(std::get<const long>(v) == 42);
   }
-// FIXME: Remove these once reference support is reinstated
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int &>;
-    int x = 42;
-    const V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get<int &>(v)), int &);
-    assert(&std::get<int &>(v) == &x);
-  }
-  {
-    using V = std::variant<int &&>;
-    int x = 42;
-    const V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get<int &&>(v)), int &);
-    assert(&std::get<int &&>(v) == &x);
-  }
-  {
-    using V = std::variant<const int &&>;
-    int x = 42;
-    const V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get<const int &&>(v)), const int &);
-    assert(&std::get<const int &&>(v) == &x);
-  }
-#endif
 }
 
 void test_lvalue_get() {
@@ -94,37 +70,6 @@ void test_lvalue_get() {
     ASSERT_SAME_TYPE(decltype(std::get<const long>(v)), const long &);
     assert(std::get<const long>(v) == 42);
   }
-// FIXME: Remove these once reference support is reinstated
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int &>;
-    int x = 42;
-    V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get<int &>(v)), int &);
-    assert(&std::get<int &>(v) == &x);
-  }
-  {
-    using V = std::variant<const int &>;
-    int x = 42;
-    V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get<const int &>(v)), const int &);
-    assert(&std::get<const int &>(v) == &x);
-  }
-  {
-    using V = std::variant<int &&>;
-    int x = 42;
-    V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get<int &&>(v)), int &);
-    assert(&std::get<int &&>(v) == &x);
-  }
-  {
-    using V = std::variant<const int &&>;
-    int x = 42;
-    V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get<const int &&>(v)), const int &);
-    assert(&std::get<const int &&>(v) == &x);
-  }
-#endif
 }
 
 void test_rvalue_get() {
@@ -142,41 +87,6 @@ void test_rvalue_get() {
                      const long &&);
     assert(std::get<const long>(std::move(v)) == 42);
   }
-// FIXME: Remove these once reference support is reinstated
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int &>;
-    int x = 42;
-    V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get<int &>(std::move(v))), int &);
-    assert(&std::get<int &>(std::move(v)) == &x);
-  }
-  {
-    using V = std::variant<const int &>;
-    int x = 42;
-    V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get<const int &>(std::move(v))),
-                     const int &);
-    assert(&std::get<const int &>(std::move(v)) == &x);
-  }
-  {
-    using V = std::variant<int &&>;
-    int x = 42;
-    V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get<int &&>(std::move(v))), int &&);
-    int &&xref = std::get<int &&>(std::move(v));
-    assert(&xref == &x);
-  }
-  {
-    using V = std::variant<const int &&>;
-    int x = 42;
-    V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get<const int &&>(std::move(v))),
-                     const int &&);
-    const int &&xref = std::get<const int &&>(std::move(v));
-    assert(&xref == &x);
-  }
-#endif
 }
 
 void test_const_rvalue_get() {
@@ -194,41 +104,6 @@ void test_const_rvalue_get() {
                      const long &&);
     assert(std::get<const long>(std::move(v)) == 42);
   }
-// FIXME: Remove these once reference support is reinstated
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int &>;
-    int x = 42;
-    const V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get<int &>(std::move(v))), int &);
-    assert(&std::get<int &>(std::move(v)) == &x);
-  }
-  {
-    using V = std::variant<const int &>;
-    int x = 42;
-    const V v(x);
-    ASSERT_SAME_TYPE(decltype(std::get<const int &>(std::move(v))),
-                     const int &);
-    assert(&std::get<const int &>(std::move(v)) == &x);
-  }
-  {
-    using V = std::variant<int &&>;
-    int x = 42;
-    const V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get<int &&>(std::move(v))), int &&);
-    int &&xref = std::get<int &&>(std::move(v));
-    assert(&xref == &x);
-  }
-  {
-    using V = std::variant<const int &&>;
-    int x = 42;
-    const V v(std::move(x));
-    ASSERT_SAME_TYPE(decltype(std::get<const int &&>(std::move(v))),
-                     const int &&);
-    const int &&xref = std::get<const int &&>(std::move(v));
-    assert(&xref == &x);
-  }
-#endif
 }
 
 template <class Tp> struct identity { using type = Tp; };
diff --git a/libcxx/test/std/utilities/variant/variant.helpers/variant_alternative.pass.cpp b/libcxx/test/std/utilities/variant/variant.helpers/variant_alternative.pass.cpp
index be1a0c960d1c..31b9b76213c4 100644
--- a/libcxx/test/std/utilities/variant/variant.helpers/variant_alternative.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.helpers/variant_alternative.pass.cpp
@@ -62,16 +62,6 @@ int main(int, char**) {
     test<V, 2, const void *>();
     test<V, 3, long double>();
   }
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int, int &, const int &, int &&, long double>;
-    test<V, 0, int>();
-    test<V, 1, int &>();
-    test<V, 2, const int &>();
-    test<V, 3, int &&>();
-    test<V, 4, long double>();
-  }
-#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.assign/T.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.assign/T.pass.cpp
index b38b10d89dfd..4b9eaba2d2ba 100644
--- a/libcxx/test/std/utilities/variant/variant.variant/variant.assign/T.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.variant/variant.assign/T.pass.cpp
@@ -155,16 +155,6 @@ void test_T_assignment_sfinae() {
     static_assert(std::is_assignable<V, Y>::value,
                   "regression on user-defined conversions in operator=");
   }
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int, int &&>;
-    static_assert(!std::is_assignable<V, int>::value, "ambiguous");
-  }
-  {
-    using V = std::variant<int, const int &>;
-    static_assert(!std::is_assignable<V, int>::value, "ambiguous");
-  }
-#endif // TEST_VARIANT_HAS_NO_REFERENCES
 }
 
 void test_T_assignment_basic() {
@@ -204,25 +194,6 @@ void test_T_assignment_basic() {
     assert(v.index() == 1);
     assert(std::get<1>(v) == nullptr);
   }
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int &, int &&, long>;
-    int x = 42;
-    V v(43l);
-    v = x;
-    assert(v.index() == 0);
-    assert(&std::get<0>(v) == &x);
-    v = std::move(x);
-    assert(v.index() == 1);
-    assert(&std::get<1>(v) == &x);
-    // 'long' is selected by FUN(const int &) since 'const int &' cannot bind
-    // to 'int&'.
-    const int &cx = x;
-    v = cx;
-    assert(v.index() == 2);
-    assert(std::get<2>(v) == 42);
-  }
-#endif // TEST_VARIANT_HAS_NO_REFERENCES
 }
 
 void test_T_assignment_performs_construction() {
diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/T.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/T.pass.cpp
index 6b7de8888849..142da1d820d9 100644
--- a/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/T.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/T.pass.cpp
@@ -36,10 +36,17 @@ struct NoThrowT {
   NoThrowT(int) noexcept(true) {}
 };
 
-struct AnyConstructible { template <typename T> AnyConstructible(T&&) {} };
-struct NoConstructible { NoConstructible() = delete; };
+struct AnyConstructible {
+  template <typename T>
+  AnyConstructible(T&&) {}
+};
+struct NoConstructible {
+  NoConstructible() = delete;
+};
 template <class T>
-struct RValueConvertibleFrom { RValueConvertibleFrom(T&&) {} };
+struct RValueConvertibleFrom {
+  RValueConvertibleFrom(T&&) {}
+};
 
 void test_T_ctor_noexcept() {
   {
@@ -59,12 +66,11 @@ void test_T_ctor_sfinae() {
   }
   {
     using V = std::variant<std::string, std::string>;
-    static_assert(!std::is_constructible<V, const char *>::value, "ambiguous");
+    static_assert(!std::is_constructible<V, const char*>::value, "ambiguous");
   }
   {
-    using V = std::variant<std::string, void *>;
-    static_assert(!std::is_constructible<V, int>::value,
-                  "no matching constructor");
+    using V = std::variant<std::string, void*>;
+    static_assert(!std::is_constructible<V, int>::value, "no matching constructor");
   }
   {
     using V = std::variant<std::string, float>;
@@ -72,8 +78,7 @@ void test_T_ctor_sfinae() {
   }
   {
     using V = std::variant<std::unique_ptr<int>, bool>;
-    static_assert(!std::is_constructible<V, std::unique_ptr<char>>::value,
-                  "no explicit bool in constructor");
+    static_assert(!std::is_constructible<V, std::unique_ptr<char>>::value, "no explicit bool in constructor");
     struct X {
       operator void*();
     };
@@ -86,30 +91,13 @@ void test_T_ctor_sfinae() {
       operator X();
     };
     using V = std::variant<X>;
-    static_assert(std::is_constructible<V, Y>::value,
-                  "regression on user-defined conversions in constructor");
+    static_assert(std::is_constructible<V, Y>::value, "regression on user-defined conversions in constructor");
   }
   {
     using V = std::variant<AnyConstructible, NoConstructible>;
-    static_assert(
-        !std::is_constructible<V, std::in_place_type_t<NoConstructible>>::value,
-        "no matching constructor");
-    static_assert(!std::is_constructible<V, std::in_place_index_t<1>>::value,
-                  "no matching constructor");
-  }
-
-
-
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int, int &&>;
-    static_assert(!std::is_constructible<V, int>::value, "ambiguous");
+    static_assert(!std::is_constructible<V, std::in_place_type_t<NoConstructible>>::value, "no matching constructor");
+    static_assert(!std::is_constructible<V, std::in_place_index_t<1>>::value, "no matching constructor");
   }
-  {
-    using V = std::variant<int, const int &>;
-    static_assert(!std::is_constructible<V, int>::value, "ambiguous");
-  }
-#endif
 }
 
 void test_T_ctor_basic() {
@@ -147,33 +135,17 @@ void test_T_ctor_basic() {
     std::variant<RValueConvertibleFrom<int>> v1 = 42;
     assert(v1.index() == 0);
 
-    int x = 42;
+    int x                                                         = 42;
     std::variant<RValueConvertibleFrom<int>, AnyConstructible> v2 = x;
     assert(v2.index() == 1);
   }
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<const int &, int &&, long>;
-    static_assert(std::is_convertible<int &, V>::value, "must be implicit");
-    int x = 42;
-    V v(x);
-    assert(v.index() == 0);
-    assert(&std::get<0>(v) == &x);
-  }
-  {
-    using V = std::variant<const int &, int &&, long>;
-    static_assert(std::is_convertible<int, V>::value, "must be implicit");
-    int x = 42;
-    V v(std::move(x));
-    assert(v.index() == 1);
-    assert(&std::get<1>(v) == &x);
-  }
-#endif
 }
 
 struct BoomOnAnything {
   template <class T>
-  constexpr BoomOnAnything(T) { static_assert(!std::is_same<T, T>::value, ""); }
+  constexpr BoomOnAnything(T) {
+    static_assert(!std::is_same<T, T>::value, "");
+  }
 };
 
 void test_no_narrowing_check_for_class_types() {
diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/default.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/default.pass.cpp
index cc1a3fe8ff78..40db038a0033 100644
--- a/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/default.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/default.pass.cpp
@@ -44,12 +44,6 @@ void test_default_ctor_sfinae() {
     using V = std::variant<NonDefaultConstructible, int>;
     static_assert(!std::is_default_constructible<V>::value, "");
   }
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int &, int>;
-    static_assert(!std::is_default_constructible<V>::value, "");
-  }
-#endif
 }
 
 void test_default_ctor_noexcept() {
diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_index_args.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_index_args.pass.cpp
index 96fcd7e7bee4..2fe9033dd816 100644
--- a/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_index_args.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_index_args.pass.cpp
@@ -55,29 +55,6 @@ void test_emplace_sfinae() {
     static_assert(emplace_exists<V, 2, int *>(), "");
     static_assert(!emplace_exists<V, 3>(), "cannot construct");
   }
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int, int &, const int &, int &&, TestTypes::NoCtors>;
-    static_assert(emplace_exists<V, 0>(), "");
-    static_assert(emplace_exists<V, 0, int>(), "");
-    static_assert(emplace_exists<V, 0, long long>(), "");
-    static_assert(!emplace_exists<V, 0, int, int>(), "too many args");
-    static_assert(emplace_exists<V, 1, int &>(), "");
-    static_assert(!emplace_exists<V, 1>(), "cannot default construct ref");
-    static_assert(!emplace_exists<V, 1, const int &>(), "cannot bind ref");
-    static_assert(!emplace_exists<V, 1, int &&>(), "cannot bind ref");
-    static_assert(emplace_exists<V, 2, int &>(), "");
-    static_assert(emplace_exists<V, 2, const int &>(), "");
-    static_assert(emplace_exists<V, 2, int &&>(), "");
-    static_assert(!emplace_exists<V, 2, void *>(),
-                  "not constructible from void*");
-    static_assert(emplace_exists<V, 3, int>(), "");
-    static_assert(!emplace_exists<V, 3, int &>(), "cannot bind ref");
-    static_assert(!emplace_exists<V, 3, const int &>(), "cannot bind ref");
-    static_assert(!emplace_exists<V, 3, const int &&>(), "cannot bind ref");
-    static_assert(!emplace_exists<V, 4>(), "no ctors");
-  }
-#endif
 }
 
 void test_basic() {
@@ -113,41 +90,6 @@ void test_basic() {
     assert(std::get<4>(v) == "aaa");
     assert(&ref3 == &std::get<4>(v));
   }
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int, long, const int &, int &&, TestTypes::NoCtors,
-                           std::string>;
-    const int x = 100;
-    int y = 42;
-    int z = 43;
-    V v(std::in_place_index<0>, -1);
-    // default emplace a value
-    auto& ref1 = v.emplace<1>();
-    static_assert(std::is_same_v<long&, decltype(ref1)>, "");
-    assert(std::get<1>(v) == 0);
-    assert(&ref1 == &std::get<1>(v));
-    // emplace a reference
-    auto& ref2 = v.emplace<2>(x);
-    static_assert(std::is_same_v<&, decltype(ref)>, "");
-    assert(&std::get<2>(v) == &x);
-    assert(&ref2 == &std::get<2>(v));
-    // emplace an rvalue reference
-    auto& ref3 = v.emplace<3>(std::move(y));
-    static_assert(std::is_same_v<&, decltype(ref)>, "");
-    assert(&std::get<3>(v) == &y);
-    assert(&ref3 == &std::get<3>(v));
-    // re-emplace a new reference over the active member
-    auto& ref4 = v.emplace<3>(std::move(z));
-    static_assert(std::is_same_v<&, decltype(ref)>, "");
-    assert(&std::get<3>(v) == &z);
-    assert(&ref4 == &std::get<3>(v));
-    // emplace with multiple args
-    auto& ref5 = v.emplace<5>(3u, 'a');
-    static_assert(std::is_same_v<std::string&, decltype(ref5)>, "");
-    assert(std::get<5>(v) == "aaa");
-    assert(&ref5 == &std::get<5>(v));
-  }
-#endif
 }
 
 int main(int, char**) {
diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_type_args.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_type_args.pass.cpp
index 24305aa0a35d..4e9f67775d10 100644
--- a/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_type_args.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.variant/variant.mod/emplace_type_args.pass.cpp
@@ -54,30 +54,6 @@ void test_emplace_sfinae() {
     static_assert(emplace_exists<V, const void *, int *>(), "");
     static_assert(!emplace_exists<V, TestTypes::NoCtors>(), "cannot construct");
   }
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  using V = std::variant<int, int &, const int &, int &&, long, long,
-                         TestTypes::NoCtors>;
-  static_assert(emplace_exists<V, int>(), "");
-  static_assert(emplace_exists<V, int, int>(), "");
-  static_assert(emplace_exists<V, int, long long>(), "");
-  static_assert(!emplace_exists<V, int, int, int>(), "too many args");
-  static_assert(emplace_exists<V, int &, int &>(), "");
-  static_assert(!emplace_exists<V, int &>(), "cannot default construct ref");
-  static_assert(!emplace_exists<V, int &, const int &>(), "cannot bind ref");
-  static_assert(!emplace_exists<V, int &, int &&>(), "cannot bind ref");
-  static_assert(emplace_exists<V, const int &, int &>(), "");
-  static_assert(emplace_exists<V, const int &, const int &>(), "");
-  static_assert(emplace_exists<V, const int &, int &&>(), "");
-  static_assert(!emplace_exists<V, const int &, void *>(),
-                "not constructible from void*");
-  static_assert(emplace_exists<V, int &&, int>(), "");
-  static_assert(!emplace_exists<V, int &&, int &>(), "cannot bind ref");
-  static_assert(!emplace_exists<V, int &&, const int &>(), "cannot bind ref");
-  static_assert(!emplace_exists<V, int &&, const int &&>(), "cannot bind ref");
-  static_assert(!emplace_exists<V, long, long>(), "ambiguous");
-  static_assert(!emplace_exists<V, TestTypes::NoCtors>(),
-                "cannot construct void");
-#endif
 }
 
 void test_basic() {
@@ -113,41 +89,6 @@ void test_basic() {
     assert(std::get<4>(v) == "aaa");
     assert(&ref3 == &std::get<4>(v));
   }
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  {
-    using V = std::variant<int, long, const int &, int &&, TestTypes::NoCtors,
-                           std::string>;
-    const int x = 100;
-    int y = 42;
-    int z = 43;
-    V v(std::in_place_index<0>, -1);
-    // default emplace a value
-    auto& ref1 = v.emplace<long>();
-    static_assert(std::is_same_v<long&, decltype(ref1)>, "");
-    assert(std::get<long>(v) == 0);
-    assert(&ref1 == &std::get<long>(v));
-    // emplace a reference
-    auto& ref2 = v.emplace<const int &>(x);
-    static_assert(std::is_same_v<const int&, decltype(ref2)>, "");
-    assert(&std::get<const int &>(v) == &x);
-    assert(&ref2 == &std::get<const int &>(v));
-    // emplace an rvalue reference
-    auto& ref3 = v.emplace<int &&>(std::move(y));
-    static_assert(std::is_same_v<int &&, decltype(ref3)>, "");
-    assert(&std::get<int &&>(v) == &y);
-    assert(&ref3 == &std::get<int &&>(v));
-    // re-emplace a new reference over the active member
-    auto& ref4 = v.emplace<int &&>(std::move(z));
-    static_assert(std::is_same_v<int &, decltype(ref4)>, "");
-    assert(&std::get<int &&>(v) == &z);
-    assert(&ref4 == &std::get<int &&>(v));
-    // emplace with multiple args
-    auto& ref5 = v.emplace<std::string>(3u, 'a');
-    static_assert(std::is_same_v<std::string&, decltype(ref5)>, "");
-    assert(std::get<std::string>(v) == "aaa");
-    assert(&ref5 == &std::get<std::string>(v));
-  }
-#endif
 }
 
 int main(int, char**) {
diff --git a/libcxx/test/std/utilities/variant/variant.visit.member/visit.pass.cpp b/libcxx/test/std/utilities/variant/variant.visit.member/visit.pass.cpp
index 50e7fc81387a..d0c909985bbb 100644
--- a/libcxx/test/std/utilities/variant/variant.visit.member/visit.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.visit.member/visit.pass.cpp
@@ -82,39 +82,6 @@ void test_argument_forwarding() {
     std::move(cv).visit(obj);
     assert(Fn::check_call<const int&&>(val));
   }
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  { // single argument - lvalue reference
-    using V = std::variant<int&>;
-    int x   = 42;
-    V v(x);
-    const V& cv = v;
-
-    v.visit(obj);
-    assert(Fn::check_call<int&>(val));
-    cv.visit(obj);
-    assert(Fn::check_call<int&>(val));
-    std::move(v).visit(obj);
-    assert(Fn::check_call<int&>(val));
-    std::move(cv).visit(obj);
-    assert(Fn::check_call<int&>(val));
-    assert(false);
-  }
-  { // single argument - rvalue reference
-    using V = std::variant<int&&>;
-    int x   = 42;
-    V v(std::move(x));
-    const V& cv = v;
-
-    v.visit(obj);
-    assert(Fn::check_call<int&>(val));
-    cvstd::visit(obj);
-    assert(Fn::check_call<int&>(val));
-    std::move(v).visit(obj);
-    assert(Fn::check_call<int&&>(val));
-    std::move(cv).visit(obj);
-    assert(Fn::check_call<int&&>(val));
-  }
-#endif
 }
 
 void test_return_type() {
diff --git a/libcxx/test/std/utilities/variant/variant.visit.member/visit_return_type.pass.cpp b/libcxx/test/std/utilities/variant/variant.visit.member/visit_return_type.pass.cpp
index b005f303bc4b..3312197d8df9 100644
--- a/libcxx/test/std/utilities/variant/variant.visit.member/visit_return_type.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.visit.member/visit_return_type.pass.cpp
@@ -110,38 +110,6 @@ void test_argument_forwarding() {
     std::move(cv).visit<ReturnType>(obj);
     assert(Fn::check_call<const int&&>(val));
   }
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  { // single argument - lvalue reference
-    using V = std::variant<int&>;
-    int x   = 42;
-    V v(x);
-    const V& cv = v;
-
-    v.visit<ReturnType>(obj);
-    assert(Fn::check_call<int&>(val));
-    cv.visit<ReturnType>(obj);
-    assert(Fn::check_call<int&>(val));
-    std::move(v).visit<ReturnType>(obj);
-    assert(Fn::check_call<int&>(val));
-    std::move(cv).visit<ReturnType>(obj);
-    assert(Fn::check_call<int&>(val));
-  }
-  { // single argument - rvalue reference
-    using V = std::variant<int&&>;
-    int x   = 42;
-    V v(std::move(x));
-    const V& cv = v;
-
-    v.visit<ReturnType>(obj);
-    assert(Fn::check_call<int&>(val));
-    cv.visit<ReturnType>(obj);
-    assert(Fn::check_call<int&>(val));
-    std::move(v).visit<ReturnType>(obj);
-    assert(Fn::check_call<int&&>(val));
-    std::move(cv).visit<ReturnType>(obj);
-    assert(Fn::check_call<int&&>(val));
-  }
-#endif
 }
 
 template <typename ReturnType>
diff --git a/libcxx/test/std/utilities/variant/variant.visit/visit.pass.cpp b/libcxx/test/std/utilities/variant/variant.visit/visit.pass.cpp
index 798ce7ded72a..0caecbe875d5 100644
--- a/libcxx/test/std/utilities/variant/variant.visit/visit.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.visit/visit.pass.cpp
@@ -118,36 +118,6 @@ void test_argument_forwarding() {
     std::visit(obj, std::move(cv));
     assert(Fn::check_call<const int &&>(Val));
   }
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  { // single argument - lvalue reference
-    using V = std::variant<int &>;
-    int x = 42;
-    V v(x);
-    const V &cv = v;
-    std::visit(obj, v);
-    assert(Fn::check_call<int &>(Val));
-    std::visit(obj, cv);
-    assert(Fn::check_call<int &>(Val));
-    std::visit(obj, std::move(v));
-    assert(Fn::check_call<int &>(Val));
-    std::visit(obj, std::move(cv));
-    assert(Fn::check_call<int &>(Val));
-  }
-  { // single argument - rvalue reference
-    using V = std::variant<int &&>;
-    int x = 42;
-    V v(std::move(x));
-    const V &cv = v;
-    std::visit(obj, v);
-    assert(Fn::check_call<int &>(Val));
-    std::visit(obj, cv);
-    assert(Fn::check_call<int &>(Val));
-    std::visit(obj, std::move(v));
-    assert(Fn::check_call<int &&>(Val));
-    std::visit(obj, std::move(cv));
-    assert(Fn::check_call<int &&>(Val));
-  }
-#endif
   { // multi argument - multi variant
     using V = std::variant<int, std::string, long>;
     V v1(42), v2("hello"), v3(43l);
diff --git a/libcxx/test/std/utilities/variant/variant.visit/visit_return_type.pass.cpp b/libcxx/test/std/utilities/variant/variant.visit/visit_return_type.pass.cpp
index b1189dff656d..d26c785c0374 100644
--- a/libcxx/test/std/utilities/variant/variant.visit/visit_return_type.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.visit/visit_return_type.pass.cpp
@@ -120,36 +120,6 @@ void test_argument_forwarding() {
     std::visit<ReturnType>(obj, std::move(cv));
     assert(Fn::check_call<const int &&>(Val));
   }
-#if !defined(TEST_VARIANT_HAS_NO_REFERENCES)
-  { // single argument - lvalue reference
-    using V = std::variant<int &>;
-    int x = 42;
-    V v(x);
-    const V &cv = v;
-    std::visit<ReturnType>(obj, v);
-    assert(Fn::check_call<int &>(Val));
-    std::visit<ReturnType>(obj, cv);
-    assert(Fn::check_call<int &>(Val));
-    std::visit<ReturnType>(obj, std::move(v));
-    assert(Fn::check_call<int &>(Val));
-    std::visit<ReturnType>(obj, std::move(cv));
-    assert(Fn::check_call<int &>(Val));
-  }
-  { // single argument - rvalue reference
-    using V = std::variant<int &&>;
-    int x = 42;
-    V v(std::move(x));
-    const V &cv = v;
-    std::visit<ReturnType>(obj, v);
-    assert(Fn::check_call<int &>(Val));
-    std::visit<ReturnType>(obj, cv);
-    assert(Fn::check_call<int &>(Val));
-    std::visit<ReturnType>(obj, std::move(v));
-    assert(Fn::check_call<int &&>(Val));
-    std::visit<ReturnType>(obj, std::move(cv));
-    assert(Fn::check_call<int &&>(Val));
-  }
-#endif
   { // multi argument - multi variant
     using V = std::variant<int, std::string, long>;
     V v1(42), v2("hello"), v3(43l);
diff --git a/libcxx/test/support/variant_test_helpers.h b/libcxx/test/support/variant_test_helpers.h
index 345e32170e58..d1bc36dea671 100644
--- a/libcxx/test/support/variant_test_helpers.h
+++ b/libcxx/test/support/variant_test_helpers.h
@@ -21,9 +21,6 @@
 #error This file requires C++17
 #endif
 
-// FIXME: Currently the variant<T&> tests are disabled using this macro.
-#define TEST_VARIANT_HAS_NO_REFERENCES
-
 #ifndef TEST_HAS_NO_EXCEPTIONS
 struct CopyThrows {
   CopyThrows() = default;
-- 
GitLab


From 54ca1e2c041a3780eca83549d7f7137581d32abb Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Tue, 26 Mar 2024 06:18:00 -0700
Subject: [PATCH 277/404] [SLP]Fix PR80027: include initial trunc nodes to the
 demoted values.

Need to include initial sext/zext/trunc nodes to the list of the demoted
root values to correctly calculate the cost and handle the
vectorization.
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 15 ++++++-
 .../RISCV/partial-vec-invalid-cost.ll         | 12 +-----
 .../SystemZ/minbitwidth-root-trunc.ll         | 41 +++++++++++++++++++
 3 files changed, 56 insertions(+), 12 deletions(-)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index f98d15c285a6..f7cb06ad3e96 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14251,6 +14251,7 @@ void BoUpSLP::computeMinimumValueSizes() {
   // resize to the final type.
   bool IsTruncRoot = false;
   bool IsProfitableToDemoteRoot = !IsStoreOrInsertElt;
+  SmallVector<unsigned> RootDemotes;
   if (NodeIdx != 0 &&
       VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize &&
       (VectorizableTree[NodeIdx]->getOpcode() == Instruction::ZExt ||
@@ -14258,6 +14259,7 @@ void BoUpSLP::computeMinimumValueSizes() {
        VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc)) {
     assert(IsStoreOrInsertElt && "Expected store/insertelement seeded graph.");
     IsTruncRoot = VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc;
+    RootDemotes.push_back(NodeIdx);
     IsProfitableToDemoteRoot = true;
     ++NodeIdx;
   }
@@ -14394,6 +14396,7 @@ void BoUpSLP::computeMinimumValueSizes() {
   while (NodeIdx < VectorizableTree.size() &&
          VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize &&
          VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc) {
+    RootDemotes.push_back(NodeIdx);
     ++NodeIdx;
     IsTruncRoot = true;
   }
@@ -14409,14 +14412,22 @@ void BoUpSLP::computeMinimumValueSizes() {
     unsigned MaxBitWidth = ComputeMaxBitWidth(
         TreeRoot, VectorizableTree[NodeIdx]->getVectorFactor(), IsTopRoot,
         IsProfitableToDemoteRoot, Opcode, Limit, IsTruncRoot);
+    for (unsigned Idx : RootDemotes)
+      ToDemote.append(VectorizableTree[Idx]->Scalars.begin(),
+                      VectorizableTree[Idx]->Scalars.end());
+    RootDemotes.clear();
     IsTopRoot = false;
     IsProfitableToDemoteRoot = true;
 
     if (TruncNodes.empty()) {
       NodeIdx = VectorizableTree.size();
     } else {
-      NodeIdx = *TruncNodes.begin() + 1;
-      TruncNodes.erase(TruncNodes.begin());
+      unsigned NewIdx = 0;
+      do {
+        NewIdx = *TruncNodes.begin() + 1;
+        TruncNodes.erase(TruncNodes.begin());
+      } while (NewIdx <= NodeIdx && !TruncNodes.empty());
+      NodeIdx = NewIdx;
       IsTruncRoot = true;
     }
 
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll
index 31f16801b7a6..6388cc2dedc7 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll
@@ -7,17 +7,9 @@ define void @partial_vec_invalid_cost() #0 {
 ; CHECK-LABEL: define void @partial_vec_invalid_cost(
 ; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LSHR_1:%.*]] = lshr i96 0, 0
-; CHECK-NEXT:    [[LSHR_2:%.*]] = lshr i96 0, 0
-; CHECK-NEXT:    [[TRUNC_I96_1:%.*]] = trunc i96 [[LSHR_1]] to i32
-; CHECK-NEXT:    [[TRUNC_I96_2:%.*]] = trunc i96 [[LSHR_2]] to i32
-; CHECK-NEXT:    [[TRUNC_I96_3:%.*]] = trunc i96 0 to i32
-; CHECK-NEXT:    [[TRUNC_I96_4:%.*]] = trunc i96 0 to i32
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> zeroinitializer)
-; CHECK-NEXT:    [[OP_RDX:%.*]] = or i32 [[TMP0]], [[TRUNC_I96_2]]
-; CHECK-NEXT:    [[OP_RDX1:%.*]] = or i32 [[TRUNC_I96_1]], [[TRUNC_I96_3]]
-; CHECK-NEXT:    [[OP_RDX2:%.*]] = or i32 [[OP_RDX]], [[OP_RDX1]]
-; CHECK-NEXT:    [[OP_RDX3:%.*]] = or i32 [[OP_RDX2]], [[TRUNC_I96_4]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[OP_RDX3:%.*]] = or i32 [[TMP0]], [[TMP1]]
 ; CHECK-NEXT:    [[STORE_THIS:%.*]] = zext i32 [[OP_RDX3]] to i96
 ; CHECK-NEXT:    store i96 [[STORE_THIS]], ptr null, align 16
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll
new file mode 100644
index 000000000000..cfe3ca9f8f9e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/minbitwidth-root-trunc.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -S --passes=slp-vectorizer -mtriple=systemz -mcpu=z15 %s | FileCheck %s
+
+define void @test(ptr %a, i8 %0, i16 %b.promoted.i) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[TMP0:%.*]], i16 [[B_PROMOTED_I:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP0]] to i128
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i16> poison, i16 [[B_PROMOTED_I]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[TMP3]], <4 x i16> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i128> poison, i128 [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i128> [[TMP5]], <4 x i128> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = trunc <4 x i128> [[TMP6]] to <4 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i16> [[TMP4]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc <4 x i16> [[TMP8]] to <4 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP9]])
+; CHECK-NEXT:    [[TMP11:%.*]] = zext i1 [[TMP10]] to i64
+; CHECK-NEXT:    [[OP_RDX:%.*]] = and i64 [[TMP11]], 1
+; CHECK-NEXT:    store i64 [[OP_RDX]], ptr [[A]], align 8
+; CHECK-NEXT:    ret void
+;
+  %2 = zext i8 %0 to i128
+  %3 = zext i16 %b.promoted.i to i128
+  %4 = or i128 %3, %2
+  %5 = trunc i128 %4 to i64
+  %6 = and i64 %5, 1
+  %7 = zext i16 %b.promoted.i to i128
+  %8 = or i128 %7, %2
+  %9 = trunc i128 %8 to i64
+  %10 = and i64 %6, %9
+  %11 = zext i16 %b.promoted.i to i128
+  %12 = or i128 %11, %2
+  %13 = trunc i128 %12 to i64
+  %14 = and i64 %10, %13
+  %15 = zext i16 %b.promoted.i to i128
+  %16 = or i128 %15, %2
+  %17 = trunc i128 %16 to i64
+  %18 = and i64 %14, %17
+  store i64 %18, ptr %a, align 8
+  ret void
+}
+
-- 
GitLab


From 47f4a07a2fe8db0ccf63dc1bf6f991124d8eff81 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Tue, 26 Mar 2024 13:42:08 +0000
Subject: [PATCH 278/404] [GlobalISel] Add Knownbits for
 G_LOAD/ZEXTLOAD/SEXTLOAD with range metadata (#86431)

Similar to #80829 for GlobalISel.
---
 .../lib/CodeGen/GlobalISel/GISelKnownBits.cpp | 19 ++--
 .../CodeGen/GlobalISel/KnownBitsTest.cpp      | 94 ++++++++++++++++---
 .../GlobalISel/KnownBitsVectorTest.cpp        |  2 +-
 3 files changed, 93 insertions(+), 22 deletions(-)

diff --git a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
index 914022ab7301..51ab7b6262c6 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
@@ -405,18 +405,23 @@ void GISelKnownBits::computeKnownBitsImpl(Register R, KnownBits &Known,
   }
   case TargetOpcode::G_LOAD: {
     const MachineMemOperand *MMO = *MI.memoperands_begin();
-    if (const MDNode *Ranges = MMO->getRanges()) {
-      computeKnownBitsFromRangeMetadata(*Ranges, Known);
-    }
-
+    KnownBits KnownRange(MMO->getMemoryType().getScalarSizeInBits());
+    if (const MDNode *Ranges = MMO->getRanges())
+      computeKnownBitsFromRangeMetadata(*Ranges, KnownRange);
+    Known = KnownRange.anyext(Known.getBitWidth());
     break;
   }
+  case TargetOpcode::G_SEXTLOAD:
   case TargetOpcode::G_ZEXTLOAD: {
     if (DstTy.isVector())
       break;
-    // Everything above the retrieved bits is zero
-    Known.Zero.setBitsFrom(
-        (*MI.memoperands_begin())->getSizeInBits().getValue());
+    const MachineMemOperand *MMO = *MI.memoperands_begin();
+    KnownBits KnownRange(MMO->getMemoryType().getScalarSizeInBits());
+    if (const MDNode *Ranges = MMO->getRanges())
+      computeKnownBitsFromRangeMetadata(*Ranges, KnownRange);
+    Known = Opcode == TargetOpcode::G_SEXTLOAD
+                ? KnownRange.sext(Known.getBitWidth())
+                : KnownRange.zext(Known.getBitWidth());
     break;
   }
   case TargetOpcode::G_ASHR: {
diff --git a/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp b/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
index f3659bc69563..ef80eed8d180 100644
--- a/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
@@ -1100,6 +1100,25 @@ TEST_F(AMDGPUGISelMITest, TestIsKnownToBeAPowerOfTwo) {
   EXPECT_TRUE(isKnownToBeAPowerOfTwo(CopyOrPow2, *MRI, &KB));
 }
 
+static void AddRangeMetadata(LLVMContext &Context, MachineInstr *Load) {
+  IntegerType *Int8Ty = Type::getInt8Ty(Context);
+
+  // Value must be in [0, 2)
+  Metadata *LowAndHigh[] = {
+      ConstantAsMetadata::get(ConstantInt::get(Int8Ty, 0)),
+      ConstantAsMetadata::get(ConstantInt::get(Int8Ty, 2))};
+  auto NewMDNode = MDNode::get(Context, LowAndHigh);
+  const MachineMemOperand *OldMMO = *Load->memoperands_begin();
+  MachineMemOperand *NewMMO =
+      Load->getParent()->getParent()->getMachineMemOperand(
+          OldMMO->getPointerInfo(), OldMMO->getFlags(), OldMMO->getMemoryType(),
+          OldMMO->getAlign(), OldMMO->getAAInfo(), NewMDNode);
+  MachineIRBuilder MIB(*Load);
+  MIB.buildLoadInstr(Load->getOpcode(), Load->getOperand(0),
+                     Load->getOperand(1), *NewMMO);
+  Load->eraseFromParent();
+}
+
 TEST_F(AArch64GISelMITest, TestMetadata) {
   StringRef MIRString = "  %imp:_(p0) = G_IMPLICIT_DEF\n"
                         "  %load:_(s8) = G_LOAD %imp(p0) :: (load (s8))\n"
@@ -1120,20 +1139,7 @@ TEST_F(AArch64GISelMITest, TestMetadata) {
   MachineInstr *And = MRI->getVRegDef(SrcReg);
   MachineInstr *Ext = MRI->getVRegDef(And->getOperand(1).getReg());
   MachineInstr *Load = MRI->getVRegDef(Ext->getOperand(1).getReg());
-  IntegerType *Int8Ty = Type::getInt8Ty(Context);
-
-  // Value must be in [0, 2)
-  Metadata *LowAndHigh[] = {
-      ConstantAsMetadata::get(ConstantInt::get(Int8Ty, 0)),
-      ConstantAsMetadata::get(ConstantInt::get(Int8Ty, 2))};
-  auto NewMDNode = MDNode::get(Context, LowAndHigh);
-  const MachineMemOperand *OldMMO = *Load->memoperands_begin();
-  MachineMemOperand NewMMO(OldMMO->getPointerInfo(), OldMMO->getFlags(),
-                           OldMMO->getSizeInBits(), OldMMO->getAlign(),
-                           OldMMO->getAAInfo(), NewMDNode);
-  MachineIRBuilder MIB(*Load);
-  MIB.buildLoad(Load->getOperand(0), Load->getOperand(1), NewMMO);
-  Load->eraseFromParent();
+  AddRangeMetadata(Context, Load);
 
   GISelKnownBits Info(*MF);
   KnownBits Res = Info.getKnownBits(And->getOperand(1).getReg());
@@ -1148,6 +1154,66 @@ TEST_F(AArch64GISelMITest, TestMetadata) {
   EXPECT_EQ(Mask.getZExtValue(), Res.Zero.getZExtValue());
 }
 
+TEST_F(AArch64GISelMITest, TestMetadataExt) {
+  StringRef MIRString = "  %imp:_(p0) = G_IMPLICIT_DEF\n"
+                        "  %load:_(s32) = G_LOAD %imp(p0) :: (load (s8))\n"
+                        "  %copy:_(s32) = COPY %load(s32)\n";
+  setUp(MIRString);
+  if (!TM)
+    GTEST_SKIP();
+
+  Register CopyReg = Copies[Copies.size() - 1];
+  MachineInstr *FinalCopy = MRI->getVRegDef(CopyReg);
+  Register SrcReg = FinalCopy->getOperand(1).getReg();
+  MachineInstr *Load = MRI->getVRegDef(SrcReg);
+  AddRangeMetadata(Context, Load);
+
+  GISelKnownBits Info(*MF);
+  KnownBits Res = Info.getKnownBits(SrcReg);
+  EXPECT_TRUE(Res.One.isZero());
+  EXPECT_EQ(Res.Zero.getZExtValue(), 0xfeu);
+}
+
+TEST_F(AArch64GISelMITest, TestMetadataZExt) {
+  StringRef MIRString = "  %imp:_(p0) = G_IMPLICIT_DEF\n"
+                        "  %load:_(s32) = G_ZEXTLOAD %imp(p0) :: (load (s8))\n"
+                        "  %copy:_(s32) = COPY %load(s32)\n";
+  setUp(MIRString);
+  if (!TM)
+    GTEST_SKIP();
+
+  Register CopyReg = Copies[Copies.size() - 1];
+  MachineInstr *FinalCopy = MRI->getVRegDef(CopyReg);
+  Register SrcReg = FinalCopy->getOperand(1).getReg();
+  MachineInstr *Load = MRI->getVRegDef(SrcReg);
+  AddRangeMetadata(Context, Load);
+
+  GISelKnownBits Info(*MF);
+  KnownBits Res = Info.getKnownBits(SrcReg);
+  EXPECT_TRUE(Res.One.isZero());
+  EXPECT_EQ(Res.Zero.getZExtValue(), 0xfffffffe);
+}
+
+TEST_F(AArch64GISelMITest, TestMetadataSExt) {
+  StringRef MIRString = "  %imp:_(p0) = G_IMPLICIT_DEF\n"
+                        "  %load:_(s32) = G_SEXTLOAD %imp(p0) :: (load (s8))\n"
+                        "  %copy:_(s32) = COPY %load(s32)\n";
+  setUp(MIRString);
+  if (!TM)
+    GTEST_SKIP();
+
+  Register CopyReg = Copies[Copies.size() - 1];
+  MachineInstr *FinalCopy = MRI->getVRegDef(CopyReg);
+  Register SrcReg = FinalCopy->getOperand(1).getReg();
+  MachineInstr *Load = MRI->getVRegDef(SrcReg);
+  AddRangeMetadata(Context, Load);
+
+  GISelKnownBits Info(*MF);
+  KnownBits Res = Info.getKnownBits(SrcReg);
+  EXPECT_TRUE(Res.One.isZero());
+  EXPECT_EQ(Res.Zero.getZExtValue(), 0xfffffffe);
+}
+
 TEST_F(AArch64GISelMITest, TestKnownBitsExt) {
   StringRef MIRString = "  %c1:_(s16) = G_CONSTANT i16 1\n"
                         "  %x:_(s16) = G_IMPLICIT_DEF\n"
diff --git a/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp b/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp
index ab180cd1cc27..dd6edd35a846 100644
--- a/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp
@@ -1008,7 +1008,7 @@ TEST_F(AArch64GISelMITest, TestVectorMetadata) {
   auto *NewMDNode = MDNode::get(Context, LowAndHigh);
   const MachineMemOperand *OldMMO = *Load->memoperands_begin();
   MachineMemOperand NewMMO(OldMMO->getPointerInfo(), OldMMO->getFlags(),
-                           OldMMO->getSizeInBits(), OldMMO->getAlign(),
+                           OldMMO->getMemoryType(), OldMMO->getAlign(),
                            OldMMO->getAAInfo(), NewMDNode);
   MachineIRBuilder MIB(*Load);
   MIB.buildLoad(Load->getOperand(0), Load->getOperand(1), NewMMO);
-- 
GitLab


From a51d13f5db08e36e0b734bc2aa9b5c4fea9cf116 Mon Sep 17 00:00:00 2001
From: Carlos Seo <carlos.seo@linaro.org>
Date: Tue, 26 Mar 2024 10:47:18 -0300
Subject: [PATCH 279/404] [Flang] Add new CHECK_MSG() function (#86576)

Added a new variant of the CHECK() function that takes a custom message
as a parameter. This is useful for more meaninful error messages when
the compiler is expected to crash.

Fixes #78931
---
 flang/include/flang/Common/idioms.h | 2 ++
 flang/lib/Evaluate/constant.cpp     | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/flang/include/flang/Common/idioms.h b/flang/include/flang/Common/idioms.h
index f6c9cbbc0f7c..231fbd8601a2 100644
--- a/flang/include/flang/Common/idioms.h
+++ b/flang/include/flang/Common/idioms.h
@@ -87,6 +87,8 @@ template <typename... LAMBDAS> visitors(LAMBDAS... x) -> visitors<LAMBDAS...>;
 // To disable, compile with '-DCHECK=(void)'
 #ifndef CHECK
 #define CHECK(x) ((x) || (DIE("CHECK(" #x ") failed"), false))
+// Same as above, but with a custom error message.
+#define CHECK_MSG(x, y) ((x) || (DIE("CHECK(" #x ") failed: " #y), false))
 #endif
 
 // User-defined type traits that default to false:
diff --git a/flang/lib/Evaluate/constant.cpp b/flang/lib/Evaluate/constant.cpp
index a3bdefb76a41..990339958399 100644
--- a/flang/lib/Evaluate/constant.cpp
+++ b/flang/lib/Evaluate/constant.cpp
@@ -160,7 +160,7 @@ template <typename RESULT, typename ELEMENT>
 auto ConstantBase<RESULT, ELEMENT>::Reshape(
     const ConstantSubscripts &dims) const -> std::vector<Element> {
   std::optional<uint64_t> optN{TotalElementCount(dims)};
-  CHECK(optN);
+  CHECK_MSG(optN, "Overflow in TotalElementCount");
   uint64_t n{*optN};
   CHECK(!empty() || n == 0);
   std::vector<Element> elements;
-- 
GitLab


From ca594fedeb3d6bb8ee0f80c92b0089147fb5e757 Mon Sep 17 00:00:00 2001
From: Tim Creech <timothy.m.creech@intel.com>
Date: Tue, 26 Mar 2024 09:52:36 -0400
Subject: [PATCH 280/404] Update documentation and release notes for
 llvm-profgen COFF support (#84864)

This change:
- Updates the existing Clang User's Manual section on SPGO so that it
describes how to use llvm-profgen to perform SPGO on Windows. This is
new functionality implemented in #83972.
- Fixes a minor typo in the existing llvm-profgen invocation example.
- Adds an LLVM release note on this new functionality in llvm-profgen.
---
 clang/docs/UsersManual.rst | 60 +++++++++++++++++++++++++++++---------
 llvm/docs/ReleaseNotes.rst |  5 ++++
 2 files changed, 52 insertions(+), 13 deletions(-)

diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst
index 129e75fc9a78..c464bc3a69ad 100644
--- a/clang/docs/UsersManual.rst
+++ b/clang/docs/UsersManual.rst
@@ -2441,20 +2441,39 @@ usual build cycle when using sample profilers for optimization:
 
 1. Build the code with source line table information. You can use all the
    usual build flags that you always build your application with. The only
-   requirement is that you add ``-gline-tables-only`` or ``-g`` to the
-   command line. This is important for the profiler to be able to map
-   instructions back to source line locations.
+   requirement is that DWARF debug info including source line information is
+   generated. This DWARF information is important for the profiler to be able
+   to map instructions back to source line locations.
+
+   On Linux, ``-g`` or just ``-gline-tables-only`` is sufficient:
 
    .. code-block:: console
 
      $ clang++ -O2 -gline-tables-only code.cc -o code
 
+   While MSVC-style targets default to CodeView debug information, DWARF debug
+   information is required to generate source-level LLVM profiles. Use
+   ``-gdwarf`` to include DWARF debug information:
+
+   .. code-block:: console
+
+     $ clang-cl -O2 -gdwarf -gline-tables-only coff-profile.cpp -fuse-ld=lld -link -debug:dwarf
+
 2. Run the executable under a sampling profiler. The specific profiler
    you use does not really matter, as long as its output can be converted
-   into the format that the LLVM optimizer understands. Currently, there
-   exists a conversion tool for the Linux Perf profiler
-   (https://perf.wiki.kernel.org/), so these examples assume that you
-   are using Linux Perf to profile your code.
+   into the format that the LLVM optimizer understands.
+
+   Two such profilers are the the Linux Perf profiler
+   (https://perf.wiki.kernel.org/) and Intel's Sampling Enabling Product (SEP),
+   available as part of `Intel VTune
+   <https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/vtune-profiler.html>`_.
+   While Perf is Linux-specific, SEP can be used on Linux, Windows, and FreeBSD.
+
+   The LLVM tool ``llvm-profgen`` can convert output of either Perf or SEP. An
+   external project, `AutoFDO <https://github.com/google/autofdo>`_, also
+   provides a ``create_llvm_prof`` tool which supports Linux Perf output.
+
+   When using Perf:
 
    .. code-block:: console
 
@@ -2465,11 +2484,19 @@ usual build cycle when using sample profilers for optimization:
    it provides better call information, which improves the accuracy of
    the profile data.
 
-3. Convert the collected profile data to LLVM's sample profile format.
-   This is currently supported via the AutoFDO converter ``create_llvm_prof``.
-   It is available at https://github.com/google/autofdo. Once built and
-   installed, you can convert the ``perf.data`` file to LLVM using
-   the command:
+   When using SEP:
+
+   .. code-block:: console
+
+     $ sep -start -out code.tb7 -ec BR_INST_RETIRED.NEAR_TAKEN:precise=yes:pdir -lbr no_filter:usr -perf-script brstack -app ./code
+
+   This produces a ``code.perf.data.script`` output which can be used with
+   ``llvm-profgen``'s ``--perfscript`` input option.
+
+3. Convert the collected profile data to LLVM's sample profile format. This is
+   currently supported via the `AutoFDO <https://github.com/google/autofdo>`_
+   converter ``create_llvm_prof``. Once built and installed, you can convert
+   the ``perf.data`` file to LLVM using the command:
 
    .. code-block:: console
 
@@ -2485,7 +2512,14 @@ usual build cycle when using sample profilers for optimization:
 
    .. code-block:: console
 
-     $ llvm-profgen --binary=./code --output=code.prof--perfdata=perf.data
+     $ llvm-profgen --binary=./code --output=code.prof --perfdata=perf.data
+
+   When using SEP the output is in the textual format corresponding to
+   ``llvm-profgen --perfscript``. For example:
+
+   .. code-block:: console
+
+     $ llvm-profgen --binary=./code --output=code.prof --perfscript=code.perf.data.script
 
 
 4. Build the code again using the collected profile. This step feeds
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index 08b496a909a7..c2b1a9d3d738 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -181,6 +181,11 @@ Changes to the LLVM tools
   for ELF input to skip the specified symbols when executing other options
   that can change a symbol's name, binding or visibility.
 
+* llvm-profgen now supports COFF+DWARF binaries. This enables Sample-based PGO
+  on Windows using Intel VTune's SEP. For details on usage, see the `end-user
+  documentation for SPGO
+  <https://clang.llvm.org/docs/UsersManual.html#using-sampling-profilers>`_.
+
 Changes to LLDB
 ---------------------------------
 
-- 
GitLab


From 89ef3130cf16f1965475396ad3a50760558cc08a Mon Sep 17 00:00:00 2001
From: Christian Sigg <chsigg@users.noreply.github.com>
Date: Tue, 26 Mar 2024 15:01:11 +0100
Subject: [PATCH 281/404] [mlir][bazel] Create a separate target for
 BytecodeOpInterface. (#86655)

This matches the CMake targets and reduces the number of headers that
need to be included in multiple targets.
---
 .../llvm-project-overlay/mlir/BUILD.bazel     | 45 ++++++++++++++-----
 .../mlir/test/BUILD.bazel                     |  3 ++
 2 files changed, 38 insertions(+), 10 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 1d9260e022c5..5dcaf0265c92 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -326,16 +326,15 @@ cc_library(
         "lib/IR/*.h",
         "lib/IR/PDL/*.cpp",
         "lib/Bytecode/Writer/*.h",
+        "include/mlir/Bytecode/*.h",
     ]) + [
         "include/mlir/IR/PDLPatternMatch.h.inc",
         "include/mlir/Interfaces/CallInterfaces.h",
         "include/mlir/Interfaces/DataLayoutInterfaces.h",
         "include/mlir/Interfaces/SideEffectInterfaces.h",
-        "lib/Bytecode/BytecodeOpInterface.cpp",
     ],
     hdrs = glob([
         "include/mlir/IR/*.h",
-        "include/mlir/Bytecode/*.h",
     ]) + [
         "include/mlir/Interfaces/FoldInterfaces.h",
     ],
@@ -3123,6 +3122,7 @@ cc_library(
     deps = [
         ":ArithDialect",
         ":BufferizationInterfaces",
+        ":BytecodeOpInterface",
         ":DialectUtils",
         ":IR",
         ":InferTypeOpInterface",
@@ -3755,6 +3755,7 @@ cc_library(
     hdrs = ["include/mlir/Dialect/XeGPU/IR/XeGPU.h"],
     includes = ["include"],
     deps = [
+        ":BytecodeOpInterface",
         ":DialectUtils",
         ":IR",
         ":ShapedOpInterfaces",
@@ -4700,6 +4701,7 @@ cc_library(
     deps = [
         ":ArithDialect",
         ":BufferizationInterfaces",
+        ":BytecodeOpInterface",
         ":CommonFolders",
         ":ControlFlowInterfaces",
         ":ControlFlowOpsIncGen",
@@ -5222,13 +5224,11 @@ cc_library(
     srcs = glob([
         "lib/Bytecode/Reader/*.cpp",
     ]),
-    hdrs = glob([
-        "include/mlir/Bytecode/*.h",
-    ]),
+    hdrs = ["include/mlir/Bytecode/BytecodeReader.h"],
     includes = ["include"],
     deps = [
         ":AsmParser",
-        ":BytecodeOpInterfaceIncGen",
+        ":BytecodeOpInterface",
         ":IR",
         ":Support",
         "//llvm:Support",
@@ -5241,12 +5241,10 @@ cc_library(
         "lib/Bytecode/Writer/*.cpp",
         "lib/Bytecode/Writer/*.h",
     ]),
-    hdrs = glob([
-        "include/mlir/Bytecode/*.h",
-    ]),
+    hdrs = ["include/mlir/Bytecode/BytecodeWriter.h"],
     includes = ["include"],
     deps = [
-        ":BytecodeOpInterfaceIncGen",
+        ":BytecodeOpInterface",
         ":IR",
         ":Support",
         "//llvm:Support",
@@ -6608,6 +6606,7 @@ cc_library(
     ]),
     includes = ["include"],
     deps = [
+        ":BytecodeOpInterface",
         ":IR",
         ":InferTypeOpInterface",
         ":PDLOpsIncGen",
@@ -7308,6 +7307,7 @@ cc_library(
         ":ArithDialect",
         ":ArithUtils",
         ":BufferizationInterfaces",
+        ":BytecodeOpInterface",
         ":CastInterfaces",
         ":ComplexDialect",
         ":ControlFlowInterfaces",
@@ -8436,6 +8436,23 @@ gentbl_cc_library(
     deps = [":BytecodeOpInterfaceTdFiles"],
 )
 
+cc_library(
+    name = "BytecodeOpInterface",
+    srcs = ["lib/Bytecode/BytecodeOpInterface.cpp"],
+    hdrs = [
+        "include/mlir/Bytecode/BytecodeImplementation.h",
+        "include/mlir/Bytecode/BytecodeOpInterface.h",
+        "include/mlir/Bytecode/Encoding.h",
+    ],
+    includes = ["include"],
+    deps = [
+        ":BytecodeOpInterfaceIncGen",
+        ":IR",
+        ":Support",
+        "//llvm:Support",
+    ],
+)
+
 gentbl_cc_library(
     name = "CallOpInterfacesIncGen",
     tbl_outs = [
@@ -10386,6 +10403,7 @@ cc_library(
     includes = ["include"],
     deps = [
         ":ArithDialect",
+        ":BytecodeOpInterface",
         ":FuncDialect",
         ":IR",
         ":InferTypeOpInterface",
@@ -10970,6 +10988,7 @@ cc_library(
         ":AsmParser",
         ":BufferizationDialect",
         ":BufferizationInterfaces",
+        ":BytecodeOpInterface",
         ":ComplexDialect",
         ":ControlFlowInterfaces",
         ":CopyOpInterface",
@@ -12356,6 +12375,7 @@ cc_library(
     includes = ["include"],
     deps = [
         ":ArithDialect",
+        ":BytecodeOpInterface",
         ":ComplexAttributesIncGen",
         ":ComplexBaseIncGen",
         ":ComplexOpsIncGen",
@@ -12621,6 +12641,7 @@ cc_library(
         ":ArithOpsIncGen",
         ":ArithOpsInterfacesIncGen",
         ":BufferizationInterfaces",
+        ":BytecodeOpInterface",
         ":CallOpInterfaces",
         ":CastInterfaces",
         ":CommonFolders",
@@ -12791,6 +12812,7 @@ cc_library(
     includes = ["include"],
     deps = [
         ":ArithDialect",
+        ":BytecodeOpInterface",
         ":CommonFolders",
         ":ConvertToLLVMInterface",
         ":IR",
@@ -12935,6 +12957,7 @@ cc_library(
         ":ArithDialect",
         ":ArithUtils",
         ":BufferizationInterfaces",
+        ":BytecodeOpInterface",
         ":CallOpInterfaces",
         ":CastInterfaces",
         ":ComplexDialect",
@@ -13199,6 +13222,7 @@ cc_library(
     hdrs = glob(["include/mlir/Dialect/MLProgram/IR/*.h"]),
     includes = ["include"],
     deps = [
+        ":BytecodeOpInterface",
         ":CallOpInterfaces",
         ":ControlFlowInterfaces",
         ":FunctionInterfaces",
@@ -13595,6 +13619,7 @@ cc_library(
         ":BufferizationBaseIncGen",
         ":BufferizationInterfaces",
         ":BufferizationOpsIncGen",
+        ":BytecodeOpInterface",
         ":CallOpInterfaces",
         ":ControlFlowInterfaces",
         ":CopyOpInterface",
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
index 57b29eb46e08..821b0446f64d 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
@@ -358,6 +358,7 @@ cc_library(
     deps = [
         ":TestTransformDialectExtensionIncGen",
         "//llvm:Support",
+        "//mlir:BytecodeOpInterface",
         "//mlir:IR",
         "//mlir:PDLDialect",
         "//mlir:Pass",
@@ -388,6 +389,7 @@ cc_library(
         ":TestTypeDefsIncGen",
         "//llvm:Support",
         "//mlir:ArithDialect",
+        "//mlir:BytecodeOpInterface",
         "//mlir:CallOpInterfaces",
         "//mlir:ControlFlowInterfaces",
         "//mlir:CopyOpInterface",
@@ -439,6 +441,7 @@ cc_library(
         "//llvm:Support",
         "//mlir:Analysis",
         "//mlir:ArithDialect",
+        "//mlir:BytecodeOpInterface",
         "//mlir:BytecodeReader",
         "//mlir:BytecodeWriter",
         "//mlir:FuncDialect",
-- 
GitLab


From 2e38c50e5c53d66d4968fbd47b78e71a220a28ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= <stefan.graenitz@gmail.com>
Date: Tue, 26 Mar 2024 15:07:05 +0100
Subject: [PATCH 282/404] [clang-repl] Fix remove invalidates iterators in
 CleanUpPTU() (#85378)

Using remove() on DeclContext::lookup_result list invalidates iterators.

This assertion failure was one (fortunate) symptom:
```
clang/include/clang/AST/DeclBase.h:1337: reference clang::DeclListNode::iterator::operator*() const: Assertion `Ptr && "dereferencing end() iterator"' failed.
```
---
 clang/include/clang/AST/DeclContextInternals.h |  2 +-
 clang/lib/Interpreter/IncrementalParser.cpp    | 18 ++++++++++++------
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/clang/include/clang/AST/DeclContextInternals.h b/clang/include/clang/AST/DeclContextInternals.h
index 903cdb7bfcc8..c4734ab57895 100644
--- a/clang/include/clang/AST/DeclContextInternals.h
+++ b/clang/include/clang/AST/DeclContextInternals.h
@@ -205,7 +205,7 @@ public:
     Data.setPointer(Head);
   }
 
-  /// Return an array of all the decls that this list represents.
+  /// Return the list of all the decls.
   DeclContext::lookup_result getLookupResult() const {
     return DeclContext::lookup_result(Data.getPointer());
   }
diff --git a/clang/lib/Interpreter/IncrementalParser.cpp b/clang/lib/Interpreter/IncrementalParser.cpp
index 370bcbfee8b0..5eec2a2fd6d1 100644
--- a/clang/lib/Interpreter/IncrementalParser.cpp
+++ b/clang/lib/Interpreter/IncrementalParser.cpp
@@ -375,16 +375,22 @@ void IncrementalParser::CleanUpPTU(PartialTranslationUnit &PTU) {
   TranslationUnitDecl *MostRecentTU = PTU.TUPart;
   TranslationUnitDecl *FirstTU = MostRecentTU->getFirstDecl();
   if (StoredDeclsMap *Map = FirstTU->getPrimaryContext()->getLookupPtr()) {
-    for (auto I = Map->begin(); I != Map->end(); ++I) {
-      StoredDeclsList &List = I->second;
+    for (auto &&[Key, List] : *Map) {
       DeclContextLookupResult R = List.getLookupResult();
+      std::vector<NamedDecl *> NamedDeclsToRemove;
+      bool RemoveAll = true;
       for (NamedDecl *D : R) {
-        if (D->getTranslationUnitDecl() == MostRecentTU) {
+        if (D->getTranslationUnitDecl() == MostRecentTU)
+          NamedDeclsToRemove.push_back(D);
+        else
+          RemoveAll = false;
+      }
+      if (LLVM_LIKELY(RemoveAll)) {
+        Map->erase(Key);
+      } else {
+        for (NamedDecl *D : NamedDeclsToRemove)
           List.remove(D);
-        }
       }
-      if (List.isNull())
-        Map->erase(I);
     }
   }
 }
-- 
GitLab


From b3117564508ce53b3af408bf2b8ab643a6030bc4 Mon Sep 17 00:00:00 2001
From: Akira Hatanaka <ahatanak@gmail.com>
Date: Tue, 26 Mar 2024 07:37:57 -0700
Subject: [PATCH 283/404] Revert "[CodeGen][arm64e] Add methods and data
 members to Address, which are needed to authenticate signed pointers
 (#67454)" (#86674)

This reverts commit 8bd1f9116aab879183f34707e6d21c7051d083b6.

It appears that the commit broke msan bots.
---
 clang/lib/CodeGen/ABIInfoImpl.cpp        |  10 +-
 clang/lib/CodeGen/Address.h              | 195 +++--------------
 clang/lib/CodeGen/CGAtomic.cpp           |  53 +++--
 clang/lib/CodeGen/CGBlocks.cpp           |  34 ++-
 clang/lib/CodeGen/CGBlocks.h             |   3 +-
 clang/lib/CodeGen/CGBuilder.h            | 234 +++++++--------------
 clang/lib/CodeGen/CGBuiltin.cpp          | 173 ++++++++-------
 clang/lib/CodeGen/CGCUDANV.cpp           |  19 +-
 clang/lib/CodeGen/CGCXXABI.cpp           |  21 +-
 clang/lib/CodeGen/CGCXXABI.h             |  14 +-
 clang/lib/CodeGen/CGCall.cpp             | 171 +++++++--------
 clang/lib/CodeGen/CGCall.h               |   1 -
 clang/lib/CodeGen/CGClass.cpp            |  76 +++----
 clang/lib/CodeGen/CGCleanup.cpp          | 110 ++++++----
 clang/lib/CodeGen/CGCleanup.h            |   2 +-
 clang/lib/CodeGen/CGCoroutine.cpp        |   4 +-
 clang/lib/CodeGen/CGDecl.cpp             |  28 ++-
 clang/lib/CodeGen/CGException.cpp        |  19 +-
 clang/lib/CodeGen/CGExpr.cpp             | 227 ++++++++++----------
 clang/lib/CodeGen/CGExprAgg.cpp          |  29 ++-
 clang/lib/CodeGen/CGExprCXX.cpp          | 111 +++++-----
 clang/lib/CodeGen/CGExprConstant.cpp     |   4 +-
 clang/lib/CodeGen/CGExprScalar.cpp       |  23 +-
 clang/lib/CodeGen/CGNonTrivialStruct.cpp |   8 +-
 clang/lib/CodeGen/CGObjC.cpp             |  43 ++--
 clang/lib/CodeGen/CGObjCGNU.cpp          |  42 ++--
 clang/lib/CodeGen/CGObjCMac.cpp          |  95 +++++----
 clang/lib/CodeGen/CGObjCRuntime.cpp      |   6 +-
 clang/lib/CodeGen/CGOpenMPRuntime.cpp    | 194 ++++++++---------
 clang/lib/CodeGen/CGOpenMPRuntime.h      |   5 +-
 clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp |  76 ++++---
 clang/lib/CodeGen/CGStmt.cpp             |   8 +-
 clang/lib/CodeGen/CGStmtOpenMP.cpp       |  87 ++++----
 clang/lib/CodeGen/CGVTables.cpp          |   9 +-
 clang/lib/CodeGen/CGValue.h              | 249 +++++++++++-----------
 clang/lib/CodeGen/CodeGenFunction.cpp    |  71 +++----
 clang/lib/CodeGen/CodeGenFunction.h      | 257 +++++++----------------
 clang/lib/CodeGen/CodeGenModule.cpp      |   2 +-
 clang/lib/CodeGen/CodeGenPGO.cpp         |  10 +-
 clang/lib/CodeGen/CodeGenPGO.h           |   6 +-
 clang/lib/CodeGen/ItaniumCXXABI.cpp      |  52 +++--
 clang/lib/CodeGen/MicrosoftCXXABI.cpp    |  58 +++--
 clang/lib/CodeGen/TargetInfo.h           |   5 -
 clang/lib/CodeGen/Targets/NVPTX.cpp      |   2 +-
 clang/lib/CodeGen/Targets/PPC.cpp        |  11 +-
 clang/lib/CodeGen/Targets/Sparc.cpp      |   2 +-
 clang/lib/CodeGen/Targets/SystemZ.cpp    |   9 +-
 clang/lib/CodeGen/Targets/XCore.cpp      |   2 +-
 clang/utils/TableGen/MveEmitter.cpp      |   2 +-
 llvm/include/llvm/IR/IRBuilder.h         |   1 -
 50 files changed, 1234 insertions(+), 1639 deletions(-)

diff --git a/clang/lib/CodeGen/ABIInfoImpl.cpp b/clang/lib/CodeGen/ABIInfoImpl.cpp
index 1dc76065a355..2b20d5a13346 100644
--- a/clang/lib/CodeGen/ABIInfoImpl.cpp
+++ b/clang/lib/CodeGen/ABIInfoImpl.cpp
@@ -187,7 +187,7 @@ CodeGen::emitVoidPtrDirectVAArg(CodeGenFunction &CGF, Address VAListAddr,
   CharUnits FullDirectSize = DirectSize.alignTo(SlotSize);
   Address NextPtr =
       CGF.Builder.CreateConstInBoundsByteGEP(Addr, FullDirectSize, "argp.next");
-  CGF.Builder.CreateStore(NextPtr.emitRawPointer(CGF), VAListAddr);
+  CGF.Builder.CreateStore(NextPtr.getPointer(), VAListAddr);
 
   // If the argument is smaller than a slot, and this is a big-endian
   // target, the argument will be right-adjusted in its slot.
@@ -239,8 +239,8 @@ Address CodeGen::emitMergePHI(CodeGenFunction &CGF, Address Addr1,
                               const llvm::Twine &Name) {
   assert(Addr1.getType() == Addr2.getType());
   llvm::PHINode *PHI = CGF.Builder.CreatePHI(Addr1.getType(), 2, Name);
-  PHI->addIncoming(Addr1.emitRawPointer(CGF), Block1);
-  PHI->addIncoming(Addr2.emitRawPointer(CGF), Block2);
+  PHI->addIncoming(Addr1.getPointer(), Block1);
+  PHI->addIncoming(Addr2.getPointer(), Block2);
   CharUnits Align = std::min(Addr1.getAlignment(), Addr2.getAlignment());
   return Address(PHI, Addr1.getElementType(), Align);
 }
@@ -400,7 +400,7 @@ Address CodeGen::EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr,
     llvm::Type *ElementTy = CGF.ConvertTypeForMem(Ty);
     llvm::Type *BaseTy = llvm::PointerType::getUnqual(ElementTy);
     llvm::Value *Addr =
-        CGF.Builder.CreateVAArg(VAListAddr.emitRawPointer(CGF), BaseTy);
+        CGF.Builder.CreateVAArg(VAListAddr.getPointer(), BaseTy);
     return Address(Addr, ElementTy, TyAlignForABI);
   } else {
     assert((AI.isDirect() || AI.isExtend()) &&
@@ -416,7 +416,7 @@ Address CodeGen::EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr,
            "Unexpected CoerceToType seen in arginfo in generic VAArg emitter!");
 
     Address Temp = CGF.CreateMemTemp(Ty, "varet");
-    Val = CGF.Builder.CreateVAArg(VAListAddr.emitRawPointer(CGF),
+    Val = CGF.Builder.CreateVAArg(VAListAddr.getPointer(),
                                   CGF.ConvertTypeForMem(Ty));
     CGF.Builder.CreateStore(Val, Temp);
     return Temp;
diff --git a/clang/lib/CodeGen/Address.h b/clang/lib/CodeGen/Address.h
index 35ec370a139c..cf48df8f5e73 100644
--- a/clang/lib/CodeGen/Address.h
+++ b/clang/lib/CodeGen/Address.h
@@ -15,7 +15,6 @@
 #define LLVM_CLANG_LIB_CODEGEN_ADDRESS_H
 
 #include "clang/AST/CharUnits.h"
-#include "clang/AST/Type.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/Support/MathExtras.h"
@@ -23,41 +22,28 @@
 namespace clang {
 namespace CodeGen {
 
-class Address;
-class CGBuilderTy;
-class CodeGenFunction;
-class CodeGenModule;
-
 // Indicates whether a pointer is known not to be null.
 enum KnownNonNull_t { NotKnownNonNull, KnownNonNull };
 
-/// An abstract representation of an aligned address. This is designed to be an
-/// IR-level abstraction, carrying just the information necessary to perform IR
-/// operations on an address like loads and stores.  In particular, it doesn't
-/// carry C type information or allow the representation of things like
-/// bit-fields; clients working at that level should generally be using
-/// `LValue`.
-/// The pointer contained in this class is known to be unsigned.
-class RawAddress {
+/// An aligned address.
+class Address {
   llvm::PointerIntPair<llvm::Value *, 1, bool> PointerAndKnownNonNull;
   llvm::Type *ElementType;
   CharUnits Alignment;
 
 protected:
-  RawAddress(std::nullptr_t) : ElementType(nullptr) {}
+  Address(std::nullptr_t) : ElementType(nullptr) {}
 
 public:
-  RawAddress(llvm::Value *Pointer, llvm::Type *ElementType, CharUnits Alignment,
-             KnownNonNull_t IsKnownNonNull = NotKnownNonNull)
+  Address(llvm::Value *Pointer, llvm::Type *ElementType, CharUnits Alignment,
+          KnownNonNull_t IsKnownNonNull = NotKnownNonNull)
       : PointerAndKnownNonNull(Pointer, IsKnownNonNull),
         ElementType(ElementType), Alignment(Alignment) {
     assert(Pointer != nullptr && "Pointer cannot be null");
     assert(ElementType != nullptr && "Element type cannot be null");
   }
 
-  inline RawAddress(Address Addr);
-
-  static RawAddress invalid() { return RawAddress(nullptr); }
+  static Address invalid() { return Address(nullptr); }
   bool isValid() const {
     return PointerAndKnownNonNull.getPointer() != nullptr;
   }
@@ -94,133 +80,6 @@ public:
     return Alignment;
   }
 
-  /// Return address with different element type, but same pointer and
-  /// alignment.
-  RawAddress withElementType(llvm::Type *ElemTy) const {
-    return RawAddress(getPointer(), ElemTy, getAlignment(), isKnownNonNull());
-  }
-
-  KnownNonNull_t isKnownNonNull() const {
-    assert(isValid());
-    return (KnownNonNull_t)PointerAndKnownNonNull.getInt();
-  }
-};
-
-/// Like RawAddress, an abstract representation of an aligned address, but the
-/// pointer contained in this class is possibly signed.
-class Address {
-  friend class CGBuilderTy;
-
-  // The boolean flag indicates whether the pointer is known to be non-null.
-  llvm::PointerIntPair<llvm::Value *, 1, bool> Pointer;
-
-  /// The expected IR type of the pointer. Carrying accurate element type
-  /// information in Address makes it more convenient to work with Address
-  /// values and allows frontend assertions to catch simple mistakes.
-  llvm::Type *ElementType = nullptr;
-
-  CharUnits Alignment;
-
-  /// Offset from the base pointer.
-  llvm::Value *Offset = nullptr;
-
-  llvm::Value *emitRawPointerSlow(CodeGenFunction &CGF) const;
-
-protected:
-  Address(std::nullptr_t) : ElementType(nullptr) {}
-
-public:
-  Address(llvm::Value *pointer, llvm::Type *elementType, CharUnits alignment,
-          KnownNonNull_t IsKnownNonNull = NotKnownNonNull)
-      : Pointer(pointer, IsKnownNonNull), ElementType(elementType),
-        Alignment(alignment) {
-    assert(pointer != nullptr && "Pointer cannot be null");
-    assert(elementType != nullptr && "Element type cannot be null");
-    assert(!alignment.isZero() && "Alignment cannot be zero");
-  }
-
-  Address(llvm::Value *BasePtr, llvm::Type *ElementType, CharUnits Alignment,
-          llvm::Value *Offset, KnownNonNull_t IsKnownNonNull = NotKnownNonNull)
-      : Pointer(BasePtr, IsKnownNonNull), ElementType(ElementType),
-        Alignment(Alignment), Offset(Offset) {}
-
-  Address(RawAddress RawAddr)
-      : Pointer(RawAddr.isValid() ? RawAddr.getPointer() : nullptr),
-        ElementType(RawAddr.isValid() ? RawAddr.getElementType() : nullptr),
-        Alignment(RawAddr.isValid() ? RawAddr.getAlignment()
-                                    : CharUnits::Zero()) {}
-
-  static Address invalid() { return Address(nullptr); }
-  bool isValid() const { return Pointer.getPointer() != nullptr; }
-
-  /// This function is used in situations where the caller is doing some sort of
-  /// opaque "laundering" of the pointer.
-  void replaceBasePointer(llvm::Value *P) {
-    assert(isValid() && "pointer isn't valid");
-    assert(P->getType() == Pointer.getPointer()->getType() &&
-           "Pointer's type changed");
-    Pointer.setPointer(P);
-    assert(isValid() && "pointer is invalid after replacement");
-  }
-
-  CharUnits getAlignment() const { return Alignment; }
-
-  void setAlignment(CharUnits Value) { Alignment = Value; }
-
-  llvm::Value *getBasePointer() const {
-    assert(isValid() && "pointer isn't valid");
-    return Pointer.getPointer();
-  }
-
-  /// Return the type of the pointer value.
-  llvm::PointerType *getType() const {
-    return llvm::PointerType::get(
-        ElementType,
-        llvm::cast<llvm::PointerType>(Pointer.getPointer()->getType())
-            ->getAddressSpace());
-  }
-
-  /// Return the type of the values stored in this address.
-  llvm::Type *getElementType() const {
-    assert(isValid());
-    return ElementType;
-  }
-
-  /// Return the address space that this address resides in.
-  unsigned getAddressSpace() const { return getType()->getAddressSpace(); }
-
-  /// Return the IR name of the pointer value.
-  llvm::StringRef getName() const { return Pointer.getPointer()->getName(); }
-
-  // This function is called only in CGBuilderBaseTy::CreateElementBitCast.
-  void setElementType(llvm::Type *Ty) {
-    assert(hasOffset() &&
-           "this funcion shouldn't be called when there is no offset");
-    ElementType = Ty;
-  }
-
-  /// Whether the pointer is known not to be null.
-  KnownNonNull_t isKnownNonNull() const {
-    assert(isValid());
-    return (KnownNonNull_t)Pointer.getInt();
-  }
-
-  Address setKnownNonNull() {
-    assert(isValid());
-    Pointer.setInt(KnownNonNull);
-    return *this;
-  }
-
-  bool hasOffset() const { return Offset; }
-
-  llvm::Value *getOffset() const { return Offset; }
-
-  /// Return the pointer contained in this class after authenticating it and
-  /// adding offset to it if necessary.
-  llvm::Value *emitRawPointer(CodeGenFunction &CGF) const {
-    return getBasePointer();
-  }
-
   /// Return address with different pointer, but same element type and
   /// alignment.
   Address withPointer(llvm::Value *NewPointer,
@@ -232,59 +91,61 @@ public:
   /// Return address with different alignment, but same pointer and element
   /// type.
   Address withAlignment(CharUnits NewAlignment) const {
-    return Address(Pointer.getPointer(), getElementType(), NewAlignment,
+    return Address(getPointer(), getElementType(), NewAlignment,
                    isKnownNonNull());
   }
 
   /// Return address with different element type, but same pointer and
   /// alignment.
   Address withElementType(llvm::Type *ElemTy) const {
-    if (!hasOffset())
-      return Address(getBasePointer(), ElemTy, getAlignment(), nullptr,
-                     isKnownNonNull());
-    Address A(*this);
-    A.ElementType = ElemTy;
-    return A;
+    return Address(getPointer(), ElemTy, getAlignment(), isKnownNonNull());
   }
-};
 
-inline RawAddress::RawAddress(Address Addr)
-    : PointerAndKnownNonNull(Addr.isValid() ? Addr.getBasePointer() : nullptr,
-                             Addr.isValid() ? Addr.isKnownNonNull()
-                                            : NotKnownNonNull),
-      ElementType(Addr.isValid() ? Addr.getElementType() : nullptr),
-      Alignment(Addr.isValid() ? Addr.getAlignment() : CharUnits::Zero()) {}
+  /// Whether the pointer is known not to be null.
+  KnownNonNull_t isKnownNonNull() const {
+    assert(isValid());
+    return (KnownNonNull_t)PointerAndKnownNonNull.getInt();
+  }
+
+  /// Set the non-null bit.
+  Address setKnownNonNull() {
+    assert(isValid());
+    PointerAndKnownNonNull.setInt(true);
+    return *this;
+  }
+};
 
 /// A specialization of Address that requires the address to be an
 /// LLVM Constant.
-class ConstantAddress : public RawAddress {
-  ConstantAddress(std::nullptr_t) : RawAddress(nullptr) {}
+class ConstantAddress : public Address {
+  ConstantAddress(std::nullptr_t) : Address(nullptr) {}
 
 public:
   ConstantAddress(llvm::Constant *pointer, llvm::Type *elementType,
                   CharUnits alignment)
-      : RawAddress(pointer, elementType, alignment) {}
+      : Address(pointer, elementType, alignment) {}
 
   static ConstantAddress invalid() {
     return ConstantAddress(nullptr);
   }
 
   llvm::Constant *getPointer() const {
-    return llvm::cast<llvm::Constant>(RawAddress::getPointer());
+    return llvm::cast<llvm::Constant>(Address::getPointer());
   }
 
   ConstantAddress withElementType(llvm::Type *ElemTy) const {
     return ConstantAddress(getPointer(), ElemTy, getAlignment());
   }
 
-  static bool isaImpl(RawAddress addr) {
+  static bool isaImpl(Address addr) {
     return llvm::isa<llvm::Constant>(addr.getPointer());
   }
-  static ConstantAddress castImpl(RawAddress addr) {
+  static ConstantAddress castImpl(Address addr) {
     return ConstantAddress(llvm::cast<llvm::Constant>(addr.getPointer()),
                            addr.getElementType(), addr.getAlignment());
   }
 };
+
 }
 
 // Present a minimal LLVM-like casting interface.
diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp
index 56198385de9d..fb03d013e8af 100644
--- a/clang/lib/CodeGen/CGAtomic.cpp
+++ b/clang/lib/CodeGen/CGAtomic.cpp
@@ -80,7 +80,7 @@ namespace {
         AtomicSizeInBits = C.toBits(
             C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
                 .alignTo(lvalue.getAlignment()));
-        llvm::Value *BitFieldPtr = lvalue.getRawBitFieldPointer(CGF);
+        llvm::Value *BitFieldPtr = lvalue.getBitFieldPointer();
         auto OffsetInChars =
             (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
             lvalue.getAlignment();
@@ -139,13 +139,13 @@ namespace {
     const LValue &getAtomicLValue() const { return LVal; }
     llvm::Value *getAtomicPointer() const {
       if (LVal.isSimple())
-        return LVal.emitRawPointer(CGF);
+        return LVal.getPointer(CGF);
       else if (LVal.isBitField())
-        return LVal.getRawBitFieldPointer(CGF);
+        return LVal.getBitFieldPointer();
       else if (LVal.isVectorElt())
-        return LVal.getRawVectorPointer(CGF);
+        return LVal.getVectorPointer();
       assert(LVal.isExtVectorElt());
-      return LVal.getRawExtVectorPointer(CGF);
+      return LVal.getExtVectorPointer();
     }
     Address getAtomicAddress() const {
       llvm::Type *ElTy;
@@ -368,7 +368,7 @@ bool AtomicInfo::emitMemSetZeroIfNecessary() const {
     return false;
 
   CGF.Builder.CreateMemSet(
-      addr.emitRawPointer(CGF), llvm::ConstantInt::get(CGF.Int8Ty, 0),
+      addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0),
       CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
       LVal.getAlignment().getAsAlign());
   return true;
@@ -1055,8 +1055,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
       return getTargetHooks().performAddrSpaceCast(
           *this, V, AS, LangAS::opencl_generic, DestType, false);
     };
-
-    Args.add(RValue::get(CastToGenericAddrSpace(Ptr.emitRawPointer(*this),
+    Args.add(RValue::get(CastToGenericAddrSpace(Ptr.getPointer(),
                                                 E->getPtr()->getType())),
              getContext().VoidPtrTy);
 
@@ -1087,10 +1086,10 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
       LibCallName = "__atomic_compare_exchange";
       RetTy = getContext().BoolTy;
       HaveRetTy = true;
-      Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this),
+      Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(),
                                                   E->getVal1()->getType())),
                getContext().VoidPtrTy);
-      Args.add(RValue::get(CastToGenericAddrSpace(Val2.emitRawPointer(*this),
+      Args.add(RValue::get(CastToGenericAddrSpace(Val2.getPointer(),
                                                   E->getVal2()->getType())),
                getContext().VoidPtrTy);
       Args.add(RValue::get(Order), getContext().IntTy);
@@ -1106,7 +1105,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
     case AtomicExpr::AO__scoped_atomic_exchange:
     case AtomicExpr::AO__scoped_atomic_exchange_n:
       LibCallName = "__atomic_exchange";
-      Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this),
+      Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(),
                                                   E->getVal1()->getType())),
                getContext().VoidPtrTy);
       break;
@@ -1121,7 +1120,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
       LibCallName = "__atomic_store";
       RetTy = getContext().VoidTy;
       HaveRetTy = true;
-      Args.add(RValue::get(CastToGenericAddrSpace(Val1.emitRawPointer(*this),
+      Args.add(RValue::get(CastToGenericAddrSpace(Val1.getPointer(),
                                                   E->getVal1()->getType())),
                getContext().VoidPtrTy);
       break;
@@ -1200,8 +1199,7 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
     if (!HaveRetTy) {
       // Value is returned through parameter before the order.
       RetTy = getContext().VoidTy;
-      Args.add(RValue::get(
-                   CastToGenericAddrSpace(Dest.emitRawPointer(*this), RetTy)),
+      Args.add(RValue::get(CastToGenericAddrSpace(Dest.getPointer(), RetTy)),
                getContext().VoidPtrTy);
     }
     // Order is always the last parameter.
@@ -1515,7 +1513,7 @@ RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
     } else
       TempAddr = CreateTempAlloca();
 
-    EmitAtomicLoadLibcall(TempAddr.emitRawPointer(CGF), AO, IsVolatile);
+    EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
 
     // Okay, turn that back into the original value or whole atomic (for
     // non-simple lvalues) type.
@@ -1675,9 +1673,9 @@ std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
   if (shouldUseLibcall()) {
     // Produce a source address.
     Address ExpectedAddr = materializeRValue(Expected);
-    llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
-    llvm::Value *DesiredPtr = materializeRValue(Desired).emitRawPointer(CGF);
-    auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr,
+    Address DesiredAddr = materializeRValue(Desired);
+    auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
+                                                 DesiredAddr.getPointer(),
                                                  Success, Failure);
     return std::make_pair(
         convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
@@ -1759,7 +1757,7 @@ void AtomicInfo::EmitAtomicUpdateLibcall(
 
   Address ExpectedAddr = CreateTempAlloca();
 
-  EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile);
+  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
   auto *ContBB = CGF.createBasicBlock("atomic_cont");
   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
   CGF.EmitBlock(ContBB);
@@ -1773,10 +1771,10 @@ void AtomicInfo::EmitAtomicUpdateLibcall(
                                            AggValueSlot::ignored(),
                                            SourceLocation(), /*AsValue=*/false);
   EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
-  llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
-  llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF);
   auto *Res =
-      EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure);
+      EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
+                                       DesiredAddr.getPointer(),
+                                       AO, Failure);
   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
 }
@@ -1845,7 +1843,7 @@ void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
 
   Address ExpectedAddr = CreateTempAlloca();
 
-  EmitAtomicLoadLibcall(ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile);
+  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
   auto *ContBB = CGF.createBasicBlock("atomic_cont");
   auto *ExitBB = CGF.createBasicBlock("atomic_exit");
   CGF.EmitBlock(ContBB);
@@ -1856,10 +1854,10 @@ void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
     CGF.Builder.CreateStore(OldVal, DesiredAddr);
   }
   EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
-  llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
-  llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF);
   auto *Res =
-      EmitAtomicCompareExchangeLibcall(ExpectedPtr, DesiredPtr, AO, Failure);
+      EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
+                                       DesiredAddr.getPointer(),
+                                       AO, Failure);
   CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
   CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
 }
@@ -1959,8 +1957,7 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
       args.add(RValue::get(atomics.getAtomicSizeValue()),
                getContext().getSizeType());
       args.add(RValue::get(atomics.getAtomicPointer()), getContext().VoidPtrTy);
-      args.add(RValue::get(srcAddr.emitRawPointer(*this)),
-               getContext().VoidPtrTy);
+      args.add(RValue::get(srcAddr.getPointer()), getContext().VoidPtrTy);
       args.add(
           RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
           getContext().IntTy);
diff --git a/clang/lib/CodeGen/CGBlocks.cpp b/clang/lib/CodeGen/CGBlocks.cpp
index a01f2c7c9798..ad0b50d79961 100644
--- a/clang/lib/CodeGen/CGBlocks.cpp
+++ b/clang/lib/CodeGen/CGBlocks.cpp
@@ -36,8 +36,7 @@ CGBlockInfo::CGBlockInfo(const BlockDecl *block, StringRef name)
     : Name(name), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false),
       NoEscape(false), HasCXXObject(false), UsesStret(false),
       HasCapturedVariableLayout(false), CapturesNonExternalType(false),
-      LocalAddress(RawAddress::invalid()), StructureType(nullptr),
-      Block(block) {
+      LocalAddress(Address::invalid()), StructureType(nullptr), Block(block) {
 
   // Skip asm prefix, if any.  'name' is usually taken directly from
   // the mangled name of the enclosing function.
@@ -795,7 +794,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
 
   // Otherwise, we have to emit this as a local block.
 
-  RawAddress blockAddr = blockInfo.LocalAddress;
+  Address blockAddr = blockInfo.LocalAddress;
   assert(blockAddr.isValid() && "block has no address!");
 
   llvm::Constant *isa;
@@ -940,7 +939,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
       if (CI.isNested())
         byrefPointer = Builder.CreateLoad(src, "byref.capture");
       else
-        byrefPointer = src.emitRawPointer(*this);
+        byrefPointer = src.getPointer();
 
       // Write that void* into the capture field.
       Builder.CreateStore(byrefPointer, blockField);
@@ -962,10 +961,10 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
       }
 
     // If it's a reference variable, copy the reference into the block field.
-    } else if (auto refType = type->getAs<ReferenceType>()) {
-      Builder.CreateStore(src.emitRawPointer(*this), blockField);
+    } else if (type->isReferenceType()) {
+      Builder.CreateStore(src.getPointer(), blockField);
 
-      // If type is const-qualified, copy the value into the block field.
+    // If type is const-qualified, copy the value into the block field.
     } else if (type.isConstQualified() &&
                type.getObjCLifetime() == Qualifiers::OCL_Strong &&
                CGM.getCodeGenOpts().OptimizationLevel != 0) {
@@ -1378,7 +1377,7 @@ void CodeGenFunction::setBlockContextParameter(const ImplicitParamDecl *D,
 
   // Allocate a stack slot like for any local variable to guarantee optimal
   // debug info at -O0. The mem2reg pass will eliminate it when optimizing.
-  RawAddress alloc = CreateMemTemp(D->getType(), D->getName() + ".addr");
+  Address alloc = CreateMemTemp(D->getType(), D->getName() + ".addr");
   Builder.CreateStore(arg, alloc);
   if (CGDebugInfo *DI = getDebugInfo()) {
     if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
@@ -1498,7 +1497,7 @@ llvm::Function *CodeGenFunction::GenerateBlockFunction(
     // frame setup instruction by llvm::DwarfDebug::beginFunction().
     auto NL = ApplyDebugLocation::CreateEmpty(*this);
     Builder.CreateStore(BlockPointer, Alloca);
-    BlockPointerDbgLoc = Alloca.emitRawPointer(*this);
+    BlockPointerDbgLoc = Alloca.getPointer();
   }
 
   // If we have a C++ 'this' reference, go ahead and force it into
@@ -1558,8 +1557,8 @@ llvm::Function *CodeGenFunction::GenerateBlockFunction(
         const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);
         if (capture.isConstant()) {
           auto addr = LocalDeclMap.find(variable)->second;
-          (void)DI->EmitDeclareOfAutoVariable(
-              variable, addr.emitRawPointer(*this), Builder);
+          (void)DI->EmitDeclareOfAutoVariable(variable, addr.getPointer(),
+                                              Builder);
           continue;
         }
 
@@ -1663,7 +1662,7 @@ struct CallBlockRelease final : EHScopeStack::Cleanup {
     if (LoadBlockVarAddr) {
       BlockVarAddr = CGF.Builder.CreateLoad(Addr);
     } else {
-      BlockVarAddr = Addr.emitRawPointer(CGF);
+      BlockVarAddr = Addr.getPointer();
     }
 
     CGF.BuildBlockRelease(BlockVarAddr, FieldFlags, CanThrow);
@@ -1963,15 +1962,13 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
         // it. It's not quite worth the annoyance to avoid creating it in the
         // first place.
         if (!needsEHCleanup(captureType.isDestructedType()))
-          if (auto *I =
-                  cast_or_null<llvm::Instruction>(dstField.getBasePointer()))
-            I->eraseFromParent();
+          cast<llvm::Instruction>(dstField.getPointer())->eraseFromParent();
       }
       break;
     }
     case BlockCaptureEntityKind::BlockObject: {
       llvm::Value *srcValue = Builder.CreateLoad(srcField, "blockcopy.src");
-      llvm::Value *dstAddr = dstField.emitRawPointer(*this);
+      llvm::Value *dstAddr = dstField.getPointer();
       llvm::Value *args[] = {
         dstAddr, srcValue, llvm::ConstantInt::get(Int32Ty, flags.getBitMask())
       };
@@ -2142,7 +2139,7 @@ public:
     llvm::Value *flagsVal = llvm::ConstantInt::get(CGF.Int32Ty, flags);
     llvm::FunctionCallee fn = CGF.CGM.getBlockObjectAssign();
 
-    llvm::Value *args[] = {destField.emitRawPointer(CGF), srcValue, flagsVal};
+    llvm::Value *args[] = { destField.getPointer(), srcValue, flagsVal };
     CGF.EmitNounwindRuntimeCall(fn, args);
   }
 
@@ -2699,8 +2696,7 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) {
   storeHeaderField(V, getPointerSize(), "byref.isa");
 
   // Store the address of the variable into its own forwarding pointer.
-  storeHeaderField(addr.emitRawPointer(*this), getPointerSize(),
-                   "byref.forwarding");
+  storeHeaderField(addr.getPointer(), getPointerSize(), "byref.forwarding");
 
   // Blocks ABI:
   //   c) the flags field is set to either 0 if no helper functions are
diff --git a/clang/lib/CodeGen/CGBlocks.h b/clang/lib/CodeGen/CGBlocks.h
index 8d10c4f69b20..4ef1ae9f3365 100644
--- a/clang/lib/CodeGen/CGBlocks.h
+++ b/clang/lib/CodeGen/CGBlocks.h
@@ -271,8 +271,7 @@ public:
   /// The block's captures. Non-constant captures are sorted by their offsets.
   llvm::SmallVector<Capture, 4> SortedCaptures;
 
-  // Currently we assume that block-pointer types are never signed.
-  RawAddress LocalAddress;
+  Address LocalAddress;
   llvm::StructType *StructureType;
   const BlockDecl *Block;
   const BlockExpr *BlockExpression;
diff --git a/clang/lib/CodeGen/CGBuilder.h b/clang/lib/CodeGen/CGBuilder.h
index 6dd9da7c4cad..bf5ab171d720 100644
--- a/clang/lib/CodeGen/CGBuilder.h
+++ b/clang/lib/CodeGen/CGBuilder.h
@@ -10,9 +10,7 @@
 #define LLVM_CLANG_LIB_CODEGEN_CGBUILDER_H
 
 #include "Address.h"
-#include "CGValue.h"
 #include "CodeGenTypeCache.h"
-#include "llvm/Analysis/Utils/Local.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Type.h"
@@ -20,15 +18,12 @@
 namespace clang {
 namespace CodeGen {
 
-class CGBuilderTy;
 class CodeGenFunction;
 
 /// This is an IRBuilder insertion helper that forwards to
 /// CodeGenFunction::InsertHelper, which adds necessary metadata to
 /// instructions.
 class CGBuilderInserter final : public llvm::IRBuilderDefaultInserter {
-  friend CGBuilderTy;
-
 public:
   CGBuilderInserter() = default;
   explicit CGBuilderInserter(CodeGenFunction *CGF) : CGF(CGF) {}
@@ -48,42 +43,10 @@ typedef llvm::IRBuilder<llvm::ConstantFolder, CGBuilderInserterTy>
     CGBuilderBaseTy;
 
 class CGBuilderTy : public CGBuilderBaseTy {
-  friend class Address;
-
   /// Storing a reference to the type cache here makes it a lot easier
   /// to build natural-feeling, target-specific IR.
   const CodeGenTypeCache &TypeCache;
 
-  CodeGenFunction *getCGF() const { return getInserter().CGF; }
-
-  llvm::Value *emitRawPointerFromAddress(Address Addr) const {
-    return Addr.getBasePointer();
-  }
-
-  template <bool IsInBounds>
-  Address createConstGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1,
-                             const llvm::Twine &Name) {
-    const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout();
-    llvm::GetElementPtrInst *GEP;
-    if (IsInBounds)
-      GEP = cast<llvm::GetElementPtrInst>(CreateConstInBoundsGEP2_32(
-          Addr.getElementType(), emitRawPointerFromAddress(Addr), Idx0, Idx1,
-          Name));
-    else
-      GEP = cast<llvm::GetElementPtrInst>(CreateConstGEP2_32(
-          Addr.getElementType(), emitRawPointerFromAddress(Addr), Idx0, Idx1,
-          Name));
-    llvm::APInt Offset(
-        DL.getIndexSizeInBits(Addr.getType()->getPointerAddressSpace()), 0,
-        /*isSigned=*/true);
-    if (!GEP->accumulateConstantOffset(DL, Offset))
-      llvm_unreachable("offset of GEP with constants is always computable");
-    return Address(GEP, GEP->getResultElementType(),
-                   Addr.getAlignment().alignmentAtOffset(
-                       CharUnits::fromQuantity(Offset.getSExtValue())),
-                   IsInBounds ? Addr.isKnownNonNull() : NotKnownNonNull);
-  }
-
 public:
   CGBuilderTy(const CodeGenTypeCache &TypeCache, llvm::LLVMContext &C)
       : CGBuilderBaseTy(C), TypeCache(TypeCache) {}
@@ -106,22 +69,20 @@ public:
   // Note that we intentionally hide the CreateLoad APIs that don't
   // take an alignment.
   llvm::LoadInst *CreateLoad(Address Addr, const llvm::Twine &Name = "") {
-    return CreateAlignedLoad(Addr.getElementType(),
-                             emitRawPointerFromAddress(Addr),
+    return CreateAlignedLoad(Addr.getElementType(), Addr.getPointer(),
                              Addr.getAlignment().getAsAlign(), Name);
   }
   llvm::LoadInst *CreateLoad(Address Addr, const char *Name) {
     // This overload is required to prevent string literals from
     // ending up in the IsVolatile overload.
-    return CreateAlignedLoad(Addr.getElementType(),
-                             emitRawPointerFromAddress(Addr),
+    return CreateAlignedLoad(Addr.getElementType(), Addr.getPointer(),
                              Addr.getAlignment().getAsAlign(), Name);
   }
   llvm::LoadInst *CreateLoad(Address Addr, bool IsVolatile,
                              const llvm::Twine &Name = "") {
-    return CreateAlignedLoad(
-        Addr.getElementType(), emitRawPointerFromAddress(Addr),
-        Addr.getAlignment().getAsAlign(), IsVolatile, Name);
+    return CreateAlignedLoad(Addr.getElementType(), Addr.getPointer(),
+                             Addr.getAlignment().getAsAlign(), IsVolatile,
+                             Name);
   }
 
   using CGBuilderBaseTy::CreateAlignedLoad;
@@ -135,7 +96,7 @@ public:
   // take an alignment.
   llvm::StoreInst *CreateStore(llvm::Value *Val, Address Addr,
                                bool IsVolatile = false) {
-    return CreateAlignedStore(Val, emitRawPointerFromAddress(Addr),
+    return CreateAlignedStore(Val, Addr.getPointer(),
                               Addr.getAlignment().getAsAlign(), IsVolatile);
   }
 
@@ -171,41 +132,33 @@ public:
                       llvm::AtomicOrdering FailureOrdering,
                       llvm::SyncScope::ID SSID = llvm::SyncScope::System) {
     return CGBuilderBaseTy::CreateAtomicCmpXchg(
-        Addr.emitRawPointer(*getCGF()), Cmp, New,
-        Addr.getAlignment().getAsAlign(), SuccessOrdering, FailureOrdering,
-        SSID);
+        Addr.getPointer(), Cmp, New, Addr.getAlignment().getAsAlign(),
+        SuccessOrdering, FailureOrdering, SSID);
   }
 
   llvm::AtomicRMWInst *
   CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val,
                   llvm::AtomicOrdering Ordering,
                   llvm::SyncScope::ID SSID = llvm::SyncScope::System) {
-    return CGBuilderBaseTy::CreateAtomicRMW(
-        Op, Addr.emitRawPointer(*getCGF()), Val,
-        Addr.getAlignment().getAsAlign(), Ordering, SSID);
+    return CGBuilderBaseTy::CreateAtomicRMW(Op, Addr.getPointer(), Val,
+                                            Addr.getAlignment().getAsAlign(),
+                                            Ordering, SSID);
   }
 
   using CGBuilderBaseTy::CreateAddrSpaceCast;
   Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty,
-                              llvm::Type *ElementTy,
                               const llvm::Twine &Name = "") {
-    if (!Addr.hasOffset())
-      return Address(CreateAddrSpaceCast(Addr.getBasePointer(), Ty, Name),
-                     ElementTy, Addr.getAlignment(), nullptr,
-                     Addr.isKnownNonNull());
-    // Eagerly force a raw address if these is an offset.
-    return RawAddress(
-        CreateAddrSpaceCast(Addr.emitRawPointer(*getCGF()), Ty, Name),
-        ElementTy, Addr.getAlignment(), Addr.isKnownNonNull());
+    return Addr.withPointer(CreateAddrSpaceCast(Addr.getPointer(), Ty, Name),
+                            Addr.isKnownNonNull());
   }
 
   using CGBuilderBaseTy::CreatePointerBitCastOrAddrSpaceCast;
   Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty,
                                               llvm::Type *ElementTy,
                                               const llvm::Twine &Name = "") {
-    if (Addr.getType()->getAddressSpace() == Ty->getPointerAddressSpace())
-      return Addr.withElementType(ElementTy);
-    return CreateAddrSpaceCast(Addr, Ty, ElementTy, Name);
+    llvm::Value *Ptr =
+        CreatePointerBitCastOrAddrSpaceCast(Addr.getPointer(), Ty, Name);
+    return Address(Ptr, ElementTy, Addr.getAlignment(), Addr.isKnownNonNull());
   }
 
   /// Given
@@ -223,11 +176,10 @@ public:
     const llvm::StructLayout *Layout = DL.getStructLayout(ElTy);
     auto Offset = CharUnits::fromQuantity(Layout->getElementOffset(Index));
 
-    return Address(CreateStructGEP(Addr.getElementType(), Addr.getBasePointer(),
-                                   Index, Name),
-                   ElTy->getElementType(Index),
-                   Addr.getAlignment().alignmentAtOffset(Offset),
-                   Addr.isKnownNonNull());
+    return Address(
+        CreateStructGEP(Addr.getElementType(), Addr.getPointer(), Index, Name),
+        ElTy->getElementType(Index),
+        Addr.getAlignment().alignmentAtOffset(Offset), Addr.isKnownNonNull());
   }
 
   /// Given
@@ -246,7 +198,7 @@ public:
         CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy->getElementType()));
 
     return Address(
-        CreateInBoundsGEP(Addr.getElementType(), Addr.getBasePointer(),
+        CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(),
                           {getSize(CharUnits::Zero()), getSize(Index)}, Name),
         ElTy->getElementType(),
         Addr.getAlignment().alignmentAtOffset(Index * EltSize),
@@ -264,10 +216,10 @@ public:
     const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout();
     CharUnits EltSize = CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy));
 
-    return Address(
-        CreateInBoundsGEP(ElTy, Addr.getBasePointer(), getSize(Index), Name),
-        ElTy, Addr.getAlignment().alignmentAtOffset(Index * EltSize),
-        Addr.isKnownNonNull());
+    return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(),
+                                     getSize(Index), Name),
+                   ElTy, Addr.getAlignment().alignmentAtOffset(Index * EltSize),
+                   Addr.isKnownNonNull());
   }
 
   /// Given
@@ -277,133 +229,110 @@ public:
   /// where i64 is actually the target word size.
   Address CreateConstGEP(Address Addr, uint64_t Index,
                          const llvm::Twine &Name = "") {
-    llvm::Type *ElTy = Addr.getElementType();
     const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout();
-    CharUnits EltSize = CharUnits::fromQuantity(DL.getTypeAllocSize(ElTy));
+    CharUnits EltSize =
+        CharUnits::fromQuantity(DL.getTypeAllocSize(Addr.getElementType()));
 
-    return Address(CreateGEP(ElTy, Addr.getBasePointer(), getSize(Index), Name),
+    return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(),
+                             getSize(Index), Name),
                    Addr.getElementType(),
-                   Addr.getAlignment().alignmentAtOffset(Index * EltSize));
+                   Addr.getAlignment().alignmentAtOffset(Index * EltSize),
+                   NotKnownNonNull);
   }
 
   /// Create GEP with single dynamic index. The address alignment is reduced
   /// according to the element size.
   using CGBuilderBaseTy::CreateGEP;
-  Address CreateGEP(CodeGenFunction &CGF, Address Addr, llvm::Value *Index,
+  Address CreateGEP(Address Addr, llvm::Value *Index,
                     const llvm::Twine &Name = "") {
     const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout();
     CharUnits EltSize =
         CharUnits::fromQuantity(DL.getTypeAllocSize(Addr.getElementType()));
 
     return Address(
-        CreateGEP(Addr.getElementType(), Addr.emitRawPointer(CGF), Index, Name),
+        CreateGEP(Addr.getElementType(), Addr.getPointer(), Index, Name),
         Addr.getElementType(),
-        Addr.getAlignment().alignmentOfArrayElement(EltSize));
+        Addr.getAlignment().alignmentOfArrayElement(EltSize), NotKnownNonNull);
   }
 
   /// Given a pointer to i8, adjust it by a given constant offset.
   Address CreateConstInBoundsByteGEP(Address Addr, CharUnits Offset,
                                      const llvm::Twine &Name = "") {
     assert(Addr.getElementType() == TypeCache.Int8Ty);
-    return Address(
-        CreateInBoundsGEP(Addr.getElementType(), Addr.getBasePointer(),
-                          getSize(Offset), Name),
-        Addr.getElementType(), Addr.getAlignment().alignmentAtOffset(Offset),
-        Addr.isKnownNonNull());
+    return Address(CreateInBoundsGEP(Addr.getElementType(), Addr.getPointer(),
+                                     getSize(Offset), Name),
+                   Addr.getElementType(),
+                   Addr.getAlignment().alignmentAtOffset(Offset),
+                   Addr.isKnownNonNull());
   }
-
   Address CreateConstByteGEP(Address Addr, CharUnits Offset,
                              const llvm::Twine &Name = "") {
     assert(Addr.getElementType() == TypeCache.Int8Ty);
-    return Address(CreateGEP(Addr.getElementType(), Addr.getBasePointer(),
+    return Address(CreateGEP(Addr.getElementType(), Addr.getPointer(),
                              getSize(Offset), Name),
                    Addr.getElementType(),
-                   Addr.getAlignment().alignmentAtOffset(Offset));
+                   Addr.getAlignment().alignmentAtOffset(Offset),
+                   NotKnownNonNull);
   }
 
   using CGBuilderBaseTy::CreateConstInBoundsGEP2_32;
   Address CreateConstInBoundsGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1,
                                      const llvm::Twine &Name = "") {
-    return createConstGEP2_32<true>(Addr, Idx0, Idx1, Name);
-  }
-
-  using CGBuilderBaseTy::CreateConstGEP2_32;
-  Address CreateConstGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1,
-                             const llvm::Twine &Name = "") {
-    return createConstGEP2_32<false>(Addr, Idx0, Idx1, Name);
-  }
-
-  Address CreateGEP(Address Addr, ArrayRef<llvm::Value *> IdxList,
-                    llvm::Type *ElementType, CharUnits Align,
-                    const Twine &Name = "") {
-    llvm::Value *Ptr = emitRawPointerFromAddress(Addr);
-    return RawAddress(CreateGEP(Addr.getElementType(), Ptr, IdxList, Name),
-                      ElementType, Align);
-  }
-
-  using CGBuilderBaseTy::CreateInBoundsGEP;
-  Address CreateInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList,
-                            llvm::Type *ElementType, CharUnits Align,
-                            const Twine &Name = "") {
-    return RawAddress(CreateInBoundsGEP(Addr.getElementType(),
-                                        emitRawPointerFromAddress(Addr),
-                                        IdxList, Name),
-                      ElementType, Align, Addr.isKnownNonNull());
-  }
+    const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout();
 
-  using CGBuilderBaseTy::CreateIsNull;
-  llvm::Value *CreateIsNull(Address Addr, const Twine &Name = "") {
-    if (!Addr.hasOffset())
-      return CreateIsNull(Addr.getBasePointer(), Name);
-    // The pointer isn't null if Addr has an offset since offsets can always
-    // be applied inbound.
-    return llvm::ConstantInt::getFalse(Context);
+    auto *GEP = cast<llvm::GetElementPtrInst>(CreateConstInBoundsGEP2_32(
+        Addr.getElementType(), Addr.getPointer(), Idx0, Idx1, Name));
+    llvm::APInt Offset(
+        DL.getIndexSizeInBits(Addr.getType()->getPointerAddressSpace()), 0,
+        /*isSigned=*/true);
+    if (!GEP->accumulateConstantOffset(DL, Offset))
+      llvm_unreachable("offset of GEP with constants is always computable");
+    return Address(GEP, GEP->getResultElementType(),
+                   Addr.getAlignment().alignmentAtOffset(
+                       CharUnits::fromQuantity(Offset.getSExtValue())),
+                   Addr.isKnownNonNull());
   }
 
   using CGBuilderBaseTy::CreateMemCpy;
   llvm::CallInst *CreateMemCpy(Address Dest, Address Src, llvm::Value *Size,
                                bool IsVolatile = false) {
-    llvm::Value *DestPtr = emitRawPointerFromAddress(Dest);
-    llvm::Value *SrcPtr = emitRawPointerFromAddress(Src);
-    return CreateMemCpy(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr,
-                        Src.getAlignment().getAsAlign(), Size, IsVolatile);
+    return CreateMemCpy(Dest.getPointer(), Dest.getAlignment().getAsAlign(),
+                        Src.getPointer(), Src.getAlignment().getAsAlign(), Size,
+                        IsVolatile);
   }
   llvm::CallInst *CreateMemCpy(Address Dest, Address Src, uint64_t Size,
                                bool IsVolatile = false) {
-    llvm::Value *DestPtr = emitRawPointerFromAddress(Dest);
-    llvm::Value *SrcPtr = emitRawPointerFromAddress(Src);
-    return CreateMemCpy(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr,
-                        Src.getAlignment().getAsAlign(), Size, IsVolatile);
+    return CreateMemCpy(Dest.getPointer(), Dest.getAlignment().getAsAlign(),
+                        Src.getPointer(), Src.getAlignment().getAsAlign(), Size,
+                        IsVolatile);
   }
 
   using CGBuilderBaseTy::CreateMemCpyInline;
   llvm::CallInst *CreateMemCpyInline(Address Dest, Address Src, uint64_t Size) {
-    llvm::Value *DestPtr = emitRawPointerFromAddress(Dest);
-    llvm::Value *SrcPtr = emitRawPointerFromAddress(Src);
-    return CreateMemCpyInline(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr,
-                              Src.getAlignment().getAsAlign(), getInt64(Size));
+    return CreateMemCpyInline(
+        Dest.getPointer(), Dest.getAlignment().getAsAlign(), Src.getPointer(),
+        Src.getAlignment().getAsAlign(), getInt64(Size));
   }
 
   using CGBuilderBaseTy::CreateMemMove;
   llvm::CallInst *CreateMemMove(Address Dest, Address Src, llvm::Value *Size,
                                 bool IsVolatile = false) {
-    llvm::Value *DestPtr = emitRawPointerFromAddress(Dest);
-    llvm::Value *SrcPtr = emitRawPointerFromAddress(Src);
-    return CreateMemMove(DestPtr, Dest.getAlignment().getAsAlign(), SrcPtr,
-                         Src.getAlignment().getAsAlign(), Size, IsVolatile);
+    return CreateMemMove(Dest.getPointer(), Dest.getAlignment().getAsAlign(),
+                         Src.getPointer(), Src.getAlignment().getAsAlign(),
+                         Size, IsVolatile);
   }
 
   using CGBuilderBaseTy::CreateMemSet;
   llvm::CallInst *CreateMemSet(Address Dest, llvm::Value *Value,
                                llvm::Value *Size, bool IsVolatile = false) {
-    return CreateMemSet(emitRawPointerFromAddress(Dest), Value, Size,
+    return CreateMemSet(Dest.getPointer(), Value, Size,
                         Dest.getAlignment().getAsAlign(), IsVolatile);
   }
 
   using CGBuilderBaseTy::CreateMemSetInline;
   llvm::CallInst *CreateMemSetInline(Address Dest, llvm::Value *Value,
                                      uint64_t Size) {
-    return CreateMemSetInline(emitRawPointerFromAddress(Dest),
+    return CreateMemSetInline(Dest.getPointer(),
                               Dest.getAlignment().getAsAlign(), Value,
                               getInt64(Size));
   }
@@ -417,31 +346,16 @@ public:
     const llvm::StructLayout *Layout = DL.getStructLayout(ElTy);
     auto Offset = CharUnits::fromQuantity(Layout->getElementOffset(Index));
 
-    return Address(
-        CreatePreserveStructAccessIndex(ElTy, emitRawPointerFromAddress(Addr),
-                                        Index, FieldIndex, DbgInfo),
-        ElTy->getElementType(Index),
-        Addr.getAlignment().alignmentAtOffset(Offset));
-  }
-
-  using CGBuilderBaseTy::CreatePreserveUnionAccessIndex;
-  Address CreatePreserveUnionAccessIndex(Address Addr, unsigned FieldIndex,
-                                         llvm::MDNode *DbgInfo) {
-    Addr.replaceBasePointer(CreatePreserveUnionAccessIndex(
-        Addr.getBasePointer(), FieldIndex, DbgInfo));
-    return Addr;
+    return Address(CreatePreserveStructAccessIndex(ElTy, Addr.getPointer(),
+                                                   Index, FieldIndex, DbgInfo),
+                   ElTy->getElementType(Index),
+                   Addr.getAlignment().alignmentAtOffset(Offset));
   }
 
   using CGBuilderBaseTy::CreateLaunderInvariantGroup;
   Address CreateLaunderInvariantGroup(Address Addr) {
-    Addr.replaceBasePointer(CreateLaunderInvariantGroup(Addr.getBasePointer()));
-    return Addr;
-  }
-
-  using CGBuilderBaseTy::CreateStripInvariantGroup;
-  Address CreateStripInvariantGroup(Address Addr) {
-    Addr.replaceBasePointer(CreateStripInvariantGroup(Addr.getBasePointer()));
-    return Addr;
+    return Addr.withPointer(CreateLaunderInvariantGroup(Addr.getPointer()),
+                            Addr.isKnownNonNull());
   }
 };
 
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 9ed7b0c57092..46a815155e7b 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2116,9 +2116,9 @@ llvm::Function *CodeGenFunction::generateBuiltinOSLogHelperFunction(
   auto AL = ApplyDebugLocation::CreateArtificial(*this);
 
   CharUnits Offset;
-  Address BufAddr = makeNaturalAddressForPointer(
-      Builder.CreateLoad(GetAddrOfLocalVar(Args[0]), "buf"), Ctx.VoidTy,
-      BufferAlignment);
+  Address BufAddr =
+      Address(Builder.CreateLoad(GetAddrOfLocalVar(Args[0]), "buf"), Int8Ty,
+              BufferAlignment);
   Builder.CreateStore(Builder.getInt8(Layout.getSummaryByte()),
                       Builder.CreateConstByteGEP(BufAddr, Offset++, "summary"));
   Builder.CreateStore(Builder.getInt8(Layout.getNumArgsByte()),
@@ -2161,7 +2161,7 @@ RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) {
 
   // Ignore argument 1, the format string. It is not currently used.
   CallArgList Args;
-  Args.add(RValue::get(BufAddr.emitRawPointer(*this)), Ctx.VoidPtrTy);
+  Args.add(RValue::get(BufAddr.getPointer()), Ctx.VoidPtrTy);
 
   for (const auto &Item : Layout.Items) {
     int Size = Item.getSizeByte();
@@ -2201,8 +2201,8 @@ RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) {
         if (!isa<Constant>(ArgVal)) {
           CleanupKind Cleanup = getARCCleanupKind();
           QualType Ty = TheExpr->getType();
-          RawAddress Alloca = RawAddress::invalid();
-          RawAddress Addr = CreateMemTemp(Ty, "os.log.arg", &Alloca);
+          Address Alloca = Address::invalid();
+          Address Addr = CreateMemTemp(Ty, "os.log.arg", &Alloca);
           ArgVal = EmitARCRetain(Ty, ArgVal);
           Builder.CreateStore(ArgVal, Addr);
           pushLifetimeExtendedDestroy(Cleanup, Alloca, Ty,
@@ -2235,7 +2235,7 @@ RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) {
   llvm::Function *F = CodeGenFunction(CGM).generateBuiltinOSLogHelperFunction(
       Layout, BufAddr.getAlignment());
   EmitCall(FI, CGCallee::forDirect(F), ReturnValueSlot(), Args);
-  return RValue::get(BufAddr, *this);
+  return RValue::get(BufAddr.getPointer());
 }
 
 static bool isSpecialUnsignedMultiplySignedResult(
@@ -2983,7 +2983,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   // Check NonnullAttribute/NullabilityArg and Alignment.
   auto EmitArgCheck = [&](TypeCheckKind Kind, Address A, const Expr *Arg,
                           unsigned ParmNum) {
-    Value *Val = A.emitRawPointer(*this);
+    Value *Val = A.getPointer();
     EmitNonNullArgCheck(RValue::get(Val), Arg->getType(), Arg->getExprLoc(), FD,
                         ParmNum);
 
@@ -3012,12 +3012,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   case Builtin::BI__builtin_va_end:
     EmitVAStartEnd(BuiltinID == Builtin::BI__va_start
                        ? EmitScalarExpr(E->getArg(0))
-                       : EmitVAListRef(E->getArg(0)).emitRawPointer(*this),
+                       : EmitVAListRef(E->getArg(0)).getPointer(),
                    BuiltinID != Builtin::BI__builtin_va_end);
     return RValue::get(nullptr);
   case Builtin::BI__builtin_va_copy: {
-    Value *DstPtr = EmitVAListRef(E->getArg(0)).emitRawPointer(*this);
-    Value *SrcPtr = EmitVAListRef(E->getArg(1)).emitRawPointer(*this);
+    Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer();
+    Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer();
     Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy), {DstPtr, SrcPtr});
     return RValue::get(nullptr);
   }
@@ -3847,13 +3847,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
 
     Address Src = EmitPointerWithAlignment(E->getArg(0));
-    EmitNonNullArgCheck(RValue::get(Src.emitRawPointer(*this)),
-                        E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD,
-                        0);
+    EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(0)->getType(),
+                        E->getArg(0)->getExprLoc(), FD, 0);
     Value *Result = MB.CreateColumnMajorLoad(
-        Src.getElementType(), Src.emitRawPointer(*this),
+        Src.getElementType(), Src.getPointer(),
         Align(Src.getAlignment().getQuantity()), Stride, IsVolatile,
-        ResultTy->getNumRows(), ResultTy->getNumColumns(), "matrix");
+        ResultTy->getNumRows(), ResultTy->getNumColumns(),
+        "matrix");
     return RValue::get(Result);
   }
 
@@ -3868,13 +3868,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     assert(PtrTy && "arg1 must be of pointer type");
     bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
 
-    EmitNonNullArgCheck(RValue::get(Dst.emitRawPointer(*this)),
-                        E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD,
-                        0);
+    EmitNonNullArgCheck(RValue::get(Dst.getPointer()), E->getArg(1)->getType(),
+                        E->getArg(1)->getExprLoc(), FD, 0);
     Value *Result = MB.CreateColumnMajorStore(
-        Matrix, Dst.emitRawPointer(*this),
-        Align(Dst.getAlignment().getQuantity()), Stride, IsVolatile,
-        MatrixTy->getNumRows(), MatrixTy->getNumColumns());
+        Matrix, Dst.getPointer(), Align(Dst.getAlignment().getQuantity()),
+        Stride, IsVolatile, MatrixTy->getNumRows(), MatrixTy->getNumColumns());
     return RValue::get(Result);
   }
 
@@ -4033,7 +4031,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   case Builtin::BI__builtin_bzero: {
     Address Dest = EmitPointerWithAlignment(E->getArg(0));
     Value *SizeVal = EmitScalarExpr(E->getArg(1));
-    EmitNonNullArgCheck(Dest, E->getArg(0)->getType(),
+    EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
                         E->getArg(0)->getExprLoc(), FD, 0);
     Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false);
     return RValue::get(nullptr);
@@ -4044,12 +4042,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Address Src = EmitPointerWithAlignment(E->getArg(0));
     Address Dest = EmitPointerWithAlignment(E->getArg(1));
     Value *SizeVal = EmitScalarExpr(E->getArg(2));
-    EmitNonNullArgCheck(RValue::get(Src.emitRawPointer(*this)),
-                        E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD,
-                        0);
-    EmitNonNullArgCheck(RValue::get(Dest.emitRawPointer(*this)),
-                        E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD,
-                        0);
+    EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(0)->getType(),
+                        E->getArg(0)->getExprLoc(), FD, 0);
+    EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(1)->getType(),
+                        E->getArg(1)->getExprLoc(), FD, 0);
     Builder.CreateMemMove(Dest, Src, SizeVal, false);
     return RValue::get(nullptr);
   }
@@ -4066,10 +4062,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Builder.CreateMemCpy(Dest, Src, SizeVal, false);
     if (BuiltinID == Builtin::BImempcpy ||
         BuiltinID == Builtin::BI__builtin_mempcpy)
-      return RValue::get(Builder.CreateInBoundsGEP(
-          Dest.getElementType(), Dest.emitRawPointer(*this), SizeVal));
+      return RValue::get(Builder.CreateInBoundsGEP(Dest.getElementType(),
+                                                   Dest.getPointer(), SizeVal));
     else
-      return RValue::get(Dest, *this);
+      return RValue::get(Dest.getPointer());
   }
 
   case Builtin::BI__builtin_memcpy_inline: {
@@ -4101,7 +4097,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Address Src = EmitPointerWithAlignment(E->getArg(1));
     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
     Builder.CreateMemCpy(Dest, Src, SizeVal, false);
-    return RValue::get(Dest, *this);
+    return RValue::get(Dest.getPointer());
   }
 
   case Builtin::BI__builtin_objc_memmove_collectable: {
@@ -4110,7 +4106,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Value *SizeVal = EmitScalarExpr(E->getArg(2));
     CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this,
                                                   DestAddr, SrcAddr, SizeVal);
-    return RValue::get(DestAddr, *this);
+    return RValue::get(DestAddr.getPointer());
   }
 
   case Builtin::BI__builtin___memmove_chk: {
@@ -4127,7 +4123,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Address Src = EmitPointerWithAlignment(E->getArg(1));
     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
     Builder.CreateMemMove(Dest, Src, SizeVal, false);
-    return RValue::get(Dest, *this);
+    return RValue::get(Dest.getPointer());
   }
 
   case Builtin::BImemmove:
@@ -4138,7 +4134,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     EmitArgCheck(TCK_Store, Dest, E->getArg(0), 0);
     EmitArgCheck(TCK_Load, Src, E->getArg(1), 1);
     Builder.CreateMemMove(Dest, Src, SizeVal, false);
-    return RValue::get(Dest, *this);
+    return RValue::get(Dest.getPointer());
   }
   case Builtin::BImemset:
   case Builtin::BI__builtin_memset: {
@@ -4146,10 +4142,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
                                          Builder.getInt8Ty());
     Value *SizeVal = EmitScalarExpr(E->getArg(2));
-    EmitNonNullArgCheck(Dest, E->getArg(0)->getType(),
+    EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
                         E->getArg(0)->getExprLoc(), FD, 0);
     Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
-    return RValue::get(Dest, *this);
+    return RValue::get(Dest.getPointer());
   }
   case Builtin::BI__builtin_memset_inline: {
     Address Dest = EmitPointerWithAlignment(E->getArg(0));
@@ -4157,9 +4153,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
         Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), Builder.getInt8Ty());
     uint64_t Size =
         E->getArg(2)->EvaluateKnownConstInt(getContext()).getZExtValue();
-    EmitNonNullArgCheck(RValue::get(Dest.emitRawPointer(*this)),
-                        E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD,
-                        0);
+    EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
+                        E->getArg(0)->getExprLoc(), FD, 0);
     Builder.CreateMemSetInline(Dest, ByteVal, Size);
     return RValue::get(nullptr);
   }
@@ -4178,7 +4173,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
                                          Builder.getInt8Ty());
     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
     Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
-    return RValue::get(Dest, *this);
+    return RValue::get(Dest.getPointer());
   }
   case Builtin::BI__builtin_wmemchr: {
     // The MSVC runtime library does not provide a definition of wmemchr, so we
@@ -4400,14 +4395,14 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
 
     // Store the stack pointer to the setjmp buffer.
     Value *StackAddr = Builder.CreateStackSave();
-    assert(Buf.emitRawPointer(*this)->getType() == StackAddr->getType());
+    assert(Buf.getPointer()->getType() == StackAddr->getType());
 
     Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Buf, 2);
     Builder.CreateStore(StackAddr, StackSaveSlot);
 
     // Call LLVM's EH setjmp, which is lightweight.
     Function *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
-    return RValue::get(Builder.CreateCall(F, Buf.emitRawPointer(*this)));
+    return RValue::get(Builder.CreateCall(F, Buf.getPointer()));
   }
   case Builtin::BI__builtin_longjmp: {
     Value *Buf = EmitScalarExpr(E->getArg(0));
@@ -5580,7 +5575,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     llvm::Value *Queue = EmitScalarExpr(E->getArg(0));
     llvm::Value *Flags = EmitScalarExpr(E->getArg(1));
     LValue NDRangeL = EmitAggExprToLValue(E->getArg(2));
-    llvm::Value *Range = NDRangeL.getAddress(*this).emitRawPointer(*this);
+    llvm::Value *Range = NDRangeL.getAddress(*this).getPointer();
     llvm::Type *RangeTy = NDRangeL.getAddress(*this).getType();
 
     if (NumArgs == 4) {
@@ -5689,10 +5684,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
               getContext(), Expr::NPC_ValueDependentIsNotNull)) {
         EventWaitList = llvm::ConstantPointerNull::get(PtrTy);
       } else {
-        EventWaitList =
-            E->getArg(4)->getType()->isArrayType()
-                ? EmitArrayToPointerDecay(E->getArg(4)).emitRawPointer(*this)
-                : EmitScalarExpr(E->getArg(4));
+        EventWaitList = E->getArg(4)->getType()->isArrayType()
+                        ? EmitArrayToPointerDecay(E->getArg(4)).getPointer()
+                        : EmitScalarExpr(E->getArg(4));
         // Convert to generic address space.
         EventWaitList = Builder.CreatePointerCast(EventWaitList, PtrTy);
       }
@@ -5788,7 +5782,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
         getContext().getTargetAddressSpace(LangAS::opencl_generic));
     LValue NDRangeL = EmitAggExprToLValue(E->getArg(0));
-    llvm::Value *NDRange = NDRangeL.getAddress(*this).emitRawPointer(*this);
+    llvm::Value *NDRange = NDRangeL.getAddress(*this).getPointer();
     auto Info =
         CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(1));
     Value *Kernel =
@@ -5873,7 +5867,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     auto PTy0 = FTy->getParamType(0);
     if (PTy0 != Arg0Val->getType()) {
       if (Arg0Ty->isArrayType())
-        Arg0Val = EmitArrayToPointerDecay(Arg0).emitRawPointer(*this);
+        Arg0Val = EmitArrayToPointerDecay(Arg0).getPointer();
       else
         Arg0Val = Builder.CreatePointerCast(Arg0Val, PTy0);
     }
@@ -5911,7 +5905,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     auto PTy1 = FTy->getParamType(1);
     if (PTy1 != Arg1Val->getType()) {
       if (Arg1Ty->isArrayType())
-        Arg1Val = EmitArrayToPointerDecay(Arg1).emitRawPointer(*this);
+        Arg1Val = EmitArrayToPointerDecay(Arg1).getPointer();
       else
         Arg1Val = Builder.CreatePointerCast(Arg1Val, PTy1);
     }
@@ -5925,7 +5919,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   case Builtin::BI__builtin_ms_va_start:
   case Builtin::BI__builtin_ms_va_end:
     return RValue::get(
-        EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).emitRawPointer(*this),
+        EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).getPointer(),
                        BuiltinID == Builtin::BI__builtin_ms_va_start));
 
   case Builtin::BI__builtin_ms_va_copy: {
@@ -5967,8 +5961,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   // If this is a predefined lib function (e.g. malloc), emit the call
   // using exactly the normal call path.
   if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID))
-    return emitLibraryCall(
-        *this, FD, E, cast<llvm::Constant>(EmitScalarExpr(E->getCallee())));
+    return emitLibraryCall(*this, FD, E,
+                      cast<llvm::Constant>(EmitScalarExpr(E->getCallee())));
 
   // Check that a call to a target specific builtin has the correct target
   // features.
@@ -6085,7 +6079,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
         return RValue::get(nullptr);
       return RValue::get(V);
     case TEK_Aggregate:
-      return RValue::getAggregate(ReturnValue.getAddress(),
+      return RValue::getAggregate(ReturnValue.getValue(),
                                   ReturnValue.isVolatile());
     case TEK_Complex:
       llvm_unreachable("No current target builtin returns complex");
@@ -8855,7 +8849,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
         // Get the alignment for the argument in addition to the value;
         // we'll use it later.
         PtrOp0 = EmitPointerWithAlignment(E->getArg(0));
-        Ops.push_back(PtrOp0.emitRawPointer(*this));
+        Ops.push_back(PtrOp0.getPointer());
         continue;
       }
     }
@@ -8882,7 +8876,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
         // Get the alignment for the argument in addition to the value;
         // we'll use it later.
         PtrOp1 = EmitPointerWithAlignment(E->getArg(1));
-        Ops.push_back(PtrOp1.emitRawPointer(*this));
+        Ops.push_back(PtrOp1.getPointer());
         continue;
       }
     }
@@ -9303,7 +9297,7 @@ Value *CodeGenFunction::EmitARMMVEBuiltinExpr(unsigned BuiltinID,
     if (ReturnValue.isNull())
       return MvecOut;
     else
-      return Builder.CreateStore(MvecOut, ReturnValue.getAddress());
+      return Builder.CreateStore(MvecOut, ReturnValue.getValue());
   }
 
   case CustomCodeGen::VST24: {
@@ -11483,7 +11477,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
         // Get the alignment for the argument in addition to the value;
         // we'll use it later.
         PtrOp0 = EmitPointerWithAlignment(E->getArg(0));
-        Ops.push_back(PtrOp0.emitRawPointer(*this));
+        Ops.push_back(PtrOp0.getPointer());
         continue;
       }
     }
@@ -13349,15 +13343,15 @@ Value *CodeGenFunction::EmitBPFBuiltinExpr(unsigned BuiltinID,
     if (!getDebugInfo()) {
       CGM.Error(E->getExprLoc(),
                 "using __builtin_preserve_field_info() without -g");
-      return IsBitField ? EmitLValue(Arg).getRawBitFieldPointer(*this)
-                        : EmitLValue(Arg).emitRawPointer(*this);
+      return IsBitField ? EmitLValue(Arg).getBitFieldPointer()
+                        : EmitLValue(Arg).getPointer(*this);
     }
 
     // Enable underlying preserve_*_access_index() generation.
     bool OldIsInPreservedAIRegion = IsInPreservedAIRegion;
     IsInPreservedAIRegion = true;
-    Value *FieldAddr = IsBitField ? EmitLValue(Arg).getRawBitFieldPointer(*this)
-                                  : EmitLValue(Arg).emitRawPointer(*this);
+    Value *FieldAddr = IsBitField ? EmitLValue(Arg).getBitFieldPointer()
+                                  : EmitLValue(Arg).getPointer(*this);
     IsInPreservedAIRegion = OldIsInPreservedAIRegion;
 
     ConstantInt *C = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
@@ -14349,14 +14343,14 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
   }
   case X86::BI_mm_setcsr:
   case X86::BI__builtin_ia32_ldmxcsr: {
-    RawAddress Tmp = CreateMemTemp(E->getArg(0)->getType());
+    Address Tmp = CreateMemTemp(E->getArg(0)->getType());
     Builder.CreateStore(Ops[0], Tmp);
     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr),
                               Tmp.getPointer());
   }
   case X86::BI_mm_getcsr:
   case X86::BI__builtin_ia32_stmxcsr: {
-    RawAddress Tmp = CreateMemTemp(E->getType());
+    Address Tmp = CreateMemTemp(E->getType());
     Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr),
                        Tmp.getPointer());
     return Builder.CreateLoad(Tmp, "stmxcsr");
@@ -17631,8 +17625,7 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
     SmallVector<Value *, 4> Ops;
     for (unsigned i = 0, e = E->getNumArgs(); i != e; i++)
       if (E->getArg(i)->getType()->isArrayType())
-        Ops.push_back(
-            EmitArrayToPointerDecay(E->getArg(i)).emitRawPointer(*this));
+        Ops.push_back(EmitArrayToPointerDecay(E->getArg(i)).getPointer());
       else
         Ops.push_back(EmitScalarExpr(E->getArg(i)));
     // The first argument of these two builtins is a pointer used to store their
@@ -20106,14 +20099,14 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
     // Save returned values.
     assert(II.NumResults);
     if (II.NumResults == 1) {
-      Builder.CreateAlignedStore(Result, Dst.emitRawPointer(*this),
+      Builder.CreateAlignedStore(Result, Dst.getPointer(),
                                  CharUnits::fromQuantity(4));
     } else {
       for (unsigned i = 0; i < II.NumResults; ++i) {
         Builder.CreateAlignedStore(
             Builder.CreateBitCast(Builder.CreateExtractValue(Result, i),
                                   Dst.getElementType()),
-            Builder.CreateGEP(Dst.getElementType(), Dst.emitRawPointer(*this),
+            Builder.CreateGEP(Dst.getElementType(), Dst.getPointer(),
                               llvm::ConstantInt::get(IntTy, i)),
             CharUnits::fromQuantity(4));
       }
@@ -20153,7 +20146,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
     for (unsigned i = 0; i < II.NumResults; ++i) {
       Value *V = Builder.CreateAlignedLoad(
           Src.getElementType(),
-          Builder.CreateGEP(Src.getElementType(), Src.emitRawPointer(*this),
+          Builder.CreateGEP(Src.getElementType(), Src.getPointer(),
                             llvm::ConstantInt::get(IntTy, i)),
           CharUnits::fromQuantity(4));
       Values.push_back(Builder.CreateBitCast(V, ParamType));
@@ -20225,7 +20218,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
     for (unsigned i = 0; i < MI.NumEltsA; ++i) {
       Value *V = Builder.CreateAlignedLoad(
           SrcA.getElementType(),
-          Builder.CreateGEP(SrcA.getElementType(), SrcA.emitRawPointer(*this),
+          Builder.CreateGEP(SrcA.getElementType(), SrcA.getPointer(),
                             llvm::ConstantInt::get(IntTy, i)),
           CharUnits::fromQuantity(4));
       Values.push_back(Builder.CreateBitCast(V, AType));
@@ -20235,7 +20228,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
     for (unsigned i = 0; i < MI.NumEltsB; ++i) {
       Value *V = Builder.CreateAlignedLoad(
           SrcB.getElementType(),
-          Builder.CreateGEP(SrcB.getElementType(), SrcB.emitRawPointer(*this),
+          Builder.CreateGEP(SrcB.getElementType(), SrcB.getPointer(),
                             llvm::ConstantInt::get(IntTy, i)),
           CharUnits::fromQuantity(4));
       Values.push_back(Builder.CreateBitCast(V, BType));
@@ -20246,7 +20239,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
     for (unsigned i = 0; i < MI.NumEltsC; ++i) {
       Value *V = Builder.CreateAlignedLoad(
           SrcC.getElementType(),
-          Builder.CreateGEP(SrcC.getElementType(), SrcC.emitRawPointer(*this),
+          Builder.CreateGEP(SrcC.getElementType(), SrcC.getPointer(),
                             llvm::ConstantInt::get(IntTy, i)),
           CharUnits::fromQuantity(4));
       Values.push_back(Builder.CreateBitCast(V, CType));
@@ -20256,7 +20249,7 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
     for (unsigned i = 0; i < MI.NumEltsD; ++i)
       Builder.CreateAlignedStore(
           Builder.CreateBitCast(Builder.CreateExtractValue(Result, i), DType),
-          Builder.CreateGEP(Dst.getElementType(), Dst.emitRawPointer(*this),
+          Builder.CreateGEP(Dst.getElementType(), Dst.getPointer(),
                             llvm::ConstantInt::get(IntTy, i)),
           CharUnits::fromQuantity(4));
     return Result;
@@ -20514,7 +20507,7 @@ struct BuiltinAlignArgs {
   BuiltinAlignArgs(const CallExpr *E, CodeGenFunction &CGF) {
     QualType AstType = E->getArg(0)->getType();
     if (AstType->isArrayType())
-      Src = CGF.EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(CGF);
+      Src = CGF.EmitArrayToPointerDecay(E->getArg(0)).getPointer();
     else
       Src = CGF.EmitScalarExpr(E->getArg(0));
     SrcType = Src->getType();
@@ -21132,7 +21125,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
   }
   case WebAssembly::BI__builtin_wasm_table_get: {
     assert(E->getArg(0)->getType()->isArrayType());
-    Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
+    Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer();
     Value *Index = EmitScalarExpr(E->getArg(1));
     Function *Callee;
     if (E->getType().isWebAssemblyExternrefType())
@@ -21146,7 +21139,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
   }
   case WebAssembly::BI__builtin_wasm_table_set: {
     assert(E->getArg(0)->getType()->isArrayType());
-    Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
+    Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer();
     Value *Index = EmitScalarExpr(E->getArg(1));
     Value *Val = EmitScalarExpr(E->getArg(2));
     Function *Callee;
@@ -21161,13 +21154,13 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
   }
   case WebAssembly::BI__builtin_wasm_table_size: {
     assert(E->getArg(0)->getType()->isArrayType());
-    Value *Value = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
+    Value *Value = EmitArrayToPointerDecay(E->getArg(0)).getPointer();
     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_table_size);
     return Builder.CreateCall(Callee, Value);
   }
   case WebAssembly::BI__builtin_wasm_table_grow: {
     assert(E->getArg(0)->getType()->isArrayType());
-    Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
+    Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer();
     Value *Val = EmitScalarExpr(E->getArg(1));
     Value *NElems = EmitScalarExpr(E->getArg(2));
 
@@ -21184,7 +21177,7 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
   }
   case WebAssembly::BI__builtin_wasm_table_fill: {
     assert(E->getArg(0)->getType()->isArrayType());
-    Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
+    Value *Table = EmitArrayToPointerDecay(E->getArg(0)).getPointer();
     Value *Index = EmitScalarExpr(E->getArg(1));
     Value *Val = EmitScalarExpr(E->getArg(2));
     Value *NElems = EmitScalarExpr(E->getArg(3));
@@ -21202,8 +21195,8 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
   }
   case WebAssembly::BI__builtin_wasm_table_copy: {
     assert(E->getArg(0)->getType()->isArrayType());
-    Value *TableX = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
-    Value *TableY = EmitArrayToPointerDecay(E->getArg(1)).emitRawPointer(*this);
+    Value *TableX = EmitArrayToPointerDecay(E->getArg(0)).getPointer();
+    Value *TableY = EmitArrayToPointerDecay(E->getArg(1)).getPointer();
     Value *DstIdx = EmitScalarExpr(E->getArg(2));
     Value *SrcIdx = EmitScalarExpr(E->getArg(3));
     Value *NElems = EmitScalarExpr(E->getArg(4));
@@ -21282,7 +21275,7 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID,
   auto MakeCircOp = [this, E](unsigned IntID, bool IsLoad) {
     // The base pointer is passed by address, so it needs to be loaded.
     Address A = EmitPointerWithAlignment(E->getArg(0));
-    Address BP = Address(A.emitRawPointer(*this), Int8PtrTy, A.getAlignment());
+    Address BP = Address(A.getPointer(), Int8PtrTy, A.getAlignment());
     llvm::Value *Base = Builder.CreateLoad(BP);
     // The treatment of both loads and stores is the same: the arguments for
     // the builtin are the same as the arguments for the intrinsic.
@@ -21323,8 +21316,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID,
     // EmitPointerWithAlignment and EmitScalarExpr evaluates the expression
     // per call.
     Address DestAddr = EmitPointerWithAlignment(E->getArg(1));
-    DestAddr = DestAddr.withElementType(Int8Ty);
-    llvm::Value *DestAddress = DestAddr.emitRawPointer(*this);
+    DestAddr = Address(DestAddr.getPointer(), Int8Ty, DestAddr.getAlignment());
+    llvm::Value *DestAddress = DestAddr.getPointer();
 
     // Operands are Base, Dest, Modifier.
     // The intrinsic format in LLVM IR is defined as
@@ -21375,8 +21368,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID,
         {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn});
 
     llvm::Value *PredOut = Builder.CreateExtractValue(Result, 1);
-    Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.emitRawPointer(*this),
-                               PredAddr.getAlignment());
+    Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.getPointer(),
+        PredAddr.getAlignment());
     return Builder.CreateExtractValue(Result, 0);
   }
   // These are identical to the builtins above, except they don't consume
@@ -21394,8 +21387,8 @@ Value *CodeGenFunction::EmitHexagonBuiltinExpr(unsigned BuiltinID,
         {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
 
     llvm::Value *PredOut = Builder.CreateExtractValue(Result, 1);
-    Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.emitRawPointer(*this),
-                               PredAddr.getAlignment());
+    Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.getPointer(),
+        PredAddr.getAlignment());
     return Builder.CreateExtractValue(Result, 0);
   }
 
diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp
index 0cb5b06a519c..b756318c46a9 100644
--- a/clang/lib/CodeGen/CGCUDANV.cpp
+++ b/clang/lib/CodeGen/CGCUDANV.cpp
@@ -331,11 +331,11 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF,
       llvm::ConstantInt::get(SizeTy, std::max<size_t>(1, Args.size())));
   // Store pointers to the arguments in a locally allocated launch_args.
   for (unsigned i = 0; i < Args.size(); ++i) {
-    llvm::Value *VarPtr = CGF.GetAddrOfLocalVar(Args[i]).emitRawPointer(CGF);
+    llvm::Value* VarPtr = CGF.GetAddrOfLocalVar(Args[i]).getPointer();
     llvm::Value *VoidVarPtr = CGF.Builder.CreatePointerCast(VarPtr, PtrTy);
     CGF.Builder.CreateDefaultAlignedStore(
-        VoidVarPtr, CGF.Builder.CreateConstGEP1_32(
-                        PtrTy, KernelArgs.emitRawPointer(CGF), i));
+        VoidVarPtr,
+        CGF.Builder.CreateConstGEP1_32(PtrTy, KernelArgs.getPointer(), i));
   }
 
   llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end");
@@ -393,10 +393,9 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF,
                               /*isVarArg=*/false),
       addUnderscoredPrefixToName("PopCallConfiguration"));
 
-  CGF.EmitRuntimeCallOrInvoke(cudaPopConfigFn, {GridDim.emitRawPointer(CGF),
-                                                BlockDim.emitRawPointer(CGF),
-                                                ShmemSize.emitRawPointer(CGF),
-                                                Stream.emitRawPointer(CGF)});
+  CGF.EmitRuntimeCallOrInvoke(cudaPopConfigFn,
+                              {GridDim.getPointer(), BlockDim.getPointer(),
+                               ShmemSize.getPointer(), Stream.getPointer()});
 
   // Emit the call to cudaLaunch
   llvm::Value *Kernel =
@@ -406,7 +405,7 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF,
                        cudaLaunchKernelFD->getParamDecl(0)->getType());
   LaunchKernelArgs.add(RValue::getAggregate(GridDim), Dim3Ty);
   LaunchKernelArgs.add(RValue::getAggregate(BlockDim), Dim3Ty);
-  LaunchKernelArgs.add(RValue::get(KernelArgs, CGF),
+  LaunchKernelArgs.add(RValue::get(KernelArgs.getPointer()),
                        cudaLaunchKernelFD->getParamDecl(3)->getType());
   LaunchKernelArgs.add(RValue::get(CGF.Builder.CreateLoad(ShmemSize)),
                        cudaLaunchKernelFD->getParamDecl(4)->getType());
@@ -439,8 +438,8 @@ void CGNVCUDARuntime::emitDeviceStubBodyLegacy(CodeGenFunction &CGF,
     auto TInfo = CGM.getContext().getTypeInfoInChars(A->getType());
     Offset = Offset.alignTo(TInfo.Align);
     llvm::Value *Args[] = {
-        CGF.Builder.CreatePointerCast(
-            CGF.GetAddrOfLocalVar(A).emitRawPointer(CGF), PtrTy),
+        CGF.Builder.CreatePointerCast(CGF.GetAddrOfLocalVar(A).getPointer(),
+                                      PtrTy),
         llvm::ConstantInt::get(SizeTy, TInfo.Width.getQuantity()),
         llvm::ConstantInt::get(SizeTy, Offset.getQuantity()),
     };
diff --git a/clang/lib/CodeGen/CGCXXABI.cpp b/clang/lib/CodeGen/CGCXXABI.cpp
index 7c6dfc3e59d8..a8bf57a277e9 100644
--- a/clang/lib/CodeGen/CGCXXABI.cpp
+++ b/clang/lib/CodeGen/CGCXXABI.cpp
@@ -20,12 +20,6 @@ using namespace CodeGen;
 
 CGCXXABI::~CGCXXABI() { }
 
-Address CGCXXABI::getThisAddress(CodeGenFunction &CGF) {
-  return CGF.makeNaturalAddressForPointer(
-      CGF.CXXABIThisValue, CGF.CXXABIThisDecl->getType()->getPointeeType(),
-      CGF.CXXABIThisAlignment);
-}
-
 void CGCXXABI::ErrorUnsupportedABI(CodeGenFunction &CGF, StringRef S) {
   DiagnosticsEngine &Diags = CGF.CGM.getDiags();
   unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error,
@@ -50,12 +44,8 @@ CGCallee CGCXXABI::EmitLoadOfMemberFunctionPointer(
     llvm::Value *MemPtr, const MemberPointerType *MPT) {
   ErrorUnsupportedABI(CGF, "calls through member pointers");
 
-  const auto *RD =
-      cast<CXXRecordDecl>(MPT->getClass()->castAs<RecordType>()->getDecl());
-  ThisPtrForCall =
-      CGF.getAsNaturalPointerTo(This, CGF.getContext().getRecordType(RD));
-  const FunctionProtoType *FPT =
-      MPT->getPointeeType()->getAs<FunctionProtoType>();
+  ThisPtrForCall = This.getPointer();
+  const auto *FPT = MPT->getPointeeType()->castAs<FunctionProtoType>();
   llvm::Constant *FnPtr = llvm::Constant::getNullValue(
       llvm::PointerType::getUnqual(CGM.getLLVMContext()));
   return CGCallee::forDirect(FnPtr, FPT);
@@ -261,15 +251,16 @@ void CGCXXABI::ReadArrayCookie(CodeGenFunction &CGF, Address ptr,
 
   // If we don't need an array cookie, bail out early.
   if (!requiresArrayCookie(expr, eltTy)) {
-    allocPtr = ptr.emitRawPointer(CGF);
+    allocPtr = ptr.getPointer();
     numElements = nullptr;
     cookieSize = CharUnits::Zero();
     return;
   }
 
   cookieSize = getArrayCookieSizeImpl(eltTy);
-  Address allocAddr = CGF.Builder.CreateConstInBoundsByteGEP(ptr, -cookieSize);
-  allocPtr = allocAddr.emitRawPointer(CGF);
+  Address allocAddr =
+    CGF.Builder.CreateConstInBoundsByteGEP(ptr, -cookieSize);
+  allocPtr = allocAddr.getPointer();
   numElements = readArrayCookieImpl(CGF, allocAddr, cookieSize);
 }
 
diff --git a/clang/lib/CodeGen/CGCXXABI.h b/clang/lib/CodeGen/CGCXXABI.h
index c7eccbd0095a..ad1ad08d0856 100644
--- a/clang/lib/CodeGen/CGCXXABI.h
+++ b/clang/lib/CodeGen/CGCXXABI.h
@@ -57,8 +57,12 @@ protected:
   llvm::Value *getThisValue(CodeGenFunction &CGF) {
     return CGF.CXXABIThisValue;
   }
-
-  Address getThisAddress(CodeGenFunction &CGF);
+  Address getThisAddress(CodeGenFunction &CGF) {
+    return Address(
+        CGF.CXXABIThisValue,
+        CGF.ConvertTypeForMem(CGF.CXXABIThisDecl->getType()->getPointeeType()),
+        CGF.CXXABIThisAlignment);
+  }
 
   /// Issue a diagnostic about unsupported features in the ABI.
   void ErrorUnsupportedABI(CodeGenFunction &CGF, StringRef S);
@@ -471,6 +475,12 @@ public:
                                   BaseSubobject Base,
                                   const CXXRecordDecl *NearestVBase) = 0;
 
+  /// Get the address point of the vtable for the given base subobject while
+  /// building a constexpr.
+  virtual llvm::Constant *
+  getVTableAddressPointForConstExpr(BaseSubobject Base,
+                                    const CXXRecordDecl *VTableClass) = 0;
+
   /// Get the address of the vtable for the given record decl which should be
   /// used for the vptr at the given offset in RD.
   virtual llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD,
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index e21ed396dc8d..a28d7888715d 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -1031,9 +1031,15 @@ static void forConstantArrayExpansion(CodeGenFunction &CGF,
                                       ConstantArrayExpansion *CAE,
                                       Address BaseAddr,
                                       llvm::function_ref<void(Address)> Fn) {
+  CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy);
+  CharUnits EltAlign =
+    BaseAddr.getAlignment().alignmentOfArrayElement(EltSize);
+  llvm::Type *EltTy = CGF.ConvertTypeForMem(CAE->EltTy);
+
   for (int i = 0, n = CAE->NumElts; i < n; i++) {
-    Address EltAddr = CGF.Builder.CreateConstGEP2_32(BaseAddr, 0, i);
-    Fn(EltAddr);
+    llvm::Value *EltAddr = CGF.Builder.CreateConstGEP2_32(
+        BaseAddr.getElementType(), BaseAddr.getPointer(), 0, i);
+    Fn(Address(EltAddr, EltTy, EltAlign));
   }
 }
 
@@ -1148,10 +1154,9 @@ void CodeGenFunction::ExpandTypeToArgs(
 }
 
 /// Create a temporary allocation for the purposes of coercion.
-static RawAddress CreateTempAllocaForCoercion(CodeGenFunction &CGF,
-                                              llvm::Type *Ty,
-                                              CharUnits MinAlign,
-                                              const Twine &Name = "tmp") {
+static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty,
+                                           CharUnits MinAlign,
+                                           const Twine &Name = "tmp") {
   // Don't use an alignment that's worse than what LLVM would prefer.
   auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(Ty);
   CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign));
@@ -1321,11 +1326,11 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
   }
 
   // Otherwise do coercion through memory. This is stupid, but simple.
-  RawAddress Tmp =
+  Address Tmp =
       CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment(), Src.getName());
   CGF.Builder.CreateMemCpy(
-      Tmp.getPointer(), Tmp.getAlignment().getAsAlign(),
-      Src.emitRawPointer(CGF), Src.getAlignment().getAsAlign(),
+      Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), Src.getPointer(),
+      Src.getAlignment().getAsAlign(),
       llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize.getKnownMinValue()));
   return CGF.Builder.CreateLoad(Tmp);
 }
@@ -1409,12 +1414,11 @@ static void CreateCoercedStore(llvm::Value *Src,
     //
     // FIXME: Assert that we aren't truncating non-padding bits when have access
     // to that information.
-    RawAddress Tmp =
-        CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment());
+    Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment());
     CGF.Builder.CreateStore(Src, Tmp);
     CGF.Builder.CreateMemCpy(
-        Dst.emitRawPointer(CGF), Dst.getAlignment().getAsAlign(),
-        Tmp.getPointer(), Tmp.getAlignment().getAsAlign(),
+        Dst.getPointer(), Dst.getAlignment().getAsAlign(), Tmp.getPointer(),
+        Tmp.getAlignment().getAsAlign(),
         llvm::ConstantInt::get(CGF.IntPtrTy, DstSize.getFixedValue()));
   }
 }
@@ -3014,17 +3018,17 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
     case ABIArgInfo::Indirect:
     case ABIArgInfo::IndirectAliased: {
       assert(NumIRArgs == 1);
-      Address ParamAddr = makeNaturalAddressForPointer(
-          Fn->getArg(FirstIRArg), Ty, ArgI.getIndirectAlign(), false, nullptr,
-          nullptr, KnownNonNull);
+      Address ParamAddr = Address(Fn->getArg(FirstIRArg), ConvertTypeForMem(Ty),
+                                  ArgI.getIndirectAlign(), KnownNonNull);
 
       if (!hasScalarEvaluationKind(Ty)) {
         // Aggregates and complex variables are accessed by reference. All we
         // need to do is realign the value, if requested. Also, if the address
         // may be aliased, copy it to ensure that the parameter variable is
         // mutable and has a unique adress, as C requires.
+        Address V = ParamAddr;
         if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) {
-          RawAddress AlignedTemp = CreateMemTemp(Ty, "coerce");
+          Address AlignedTemp = CreateMemTemp(Ty, "coerce");
 
           // Copy from the incoming argument pointer to the temporary with the
           // appropriate alignment.
@@ -3034,12 +3038,11 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
           CharUnits Size = getContext().getTypeSizeInChars(Ty);
           Builder.CreateMemCpy(
               AlignedTemp.getPointer(), AlignedTemp.getAlignment().getAsAlign(),
-              ParamAddr.emitRawPointer(*this),
-              ParamAddr.getAlignment().getAsAlign(),
+              ParamAddr.getPointer(), ParamAddr.getAlignment().getAsAlign(),
               llvm::ConstantInt::get(IntPtrTy, Size.getQuantity()));
-          ParamAddr = AlignedTemp;
+          V = AlignedTemp;
         }
-        ArgVals.push_back(ParamValue::forIndirect(ParamAddr));
+        ArgVals.push_back(ParamValue::forIndirect(V));
       } else {
         // Load scalar value from indirect argument.
         llvm::Value *V =
@@ -3153,10 +3156,10 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
               == ParameterABI::SwiftErrorResult) {
           QualType pointeeTy = Ty->getPointeeType();
           assert(pointeeTy->isPointerType());
-          RawAddress temp =
-              CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
-          Address arg = makeNaturalAddressForPointer(
-              V, pointeeTy, getContext().getTypeAlignInChars(pointeeTy));
+          Address temp =
+            CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
+          Address arg(V, ConvertTypeForMem(pointeeTy),
+                      getContext().getTypeAlignInChars(pointeeTy));
           llvm::Value *incomingErrorValue = Builder.CreateLoad(arg);
           Builder.CreateStore(incomingErrorValue, temp);
           V = temp.getPointer();
@@ -3493,7 +3496,7 @@ static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
   llvm::LoadInst *load =
     dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());
   if (!load || load->isAtomic() || load->isVolatile() ||
-      load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getBasePointer())
+      load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer())
     return nullptr;
 
   // Okay!  Burn it all down.  This relies for correctness on the
@@ -3530,15 +3533,12 @@ static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
 
 /// Heuristically search for a dominating store to the return-value slot.
 static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
-  llvm::Value *ReturnValuePtr = CGF.ReturnValue.getBasePointer();
-
   // Check if a User is a store which pointerOperand is the ReturnValue.
   // We are looking for stores to the ReturnValue, not for stores of the
   // ReturnValue to some other location.
-  auto GetStoreIfValid = [&CGF,
-                          ReturnValuePtr](llvm::User *U) -> llvm::StoreInst * {
+  auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * {
     auto *SI = dyn_cast<llvm::StoreInst>(U);
-    if (!SI || SI->getPointerOperand() != ReturnValuePtr ||
+    if (!SI || SI->getPointerOperand() != CGF.ReturnValue.getPointer() ||
         SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType())
       return nullptr;
     // These aren't actually possible for non-coerced returns, and we
@@ -3552,7 +3552,7 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
   // for something immediately preceding the IP.  Sometimes this can
   // happen with how we generate implicit-returns; it can also happen
   // with noreturn cleanups.
-  if (!ReturnValuePtr->hasOneUse()) {
+  if (!CGF.ReturnValue.getPointer()->hasOneUse()) {
     llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
     if (IP->empty()) return nullptr;
 
@@ -3570,7 +3570,8 @@ static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
     return nullptr;
   }
 
-  llvm::StoreInst *store = GetStoreIfValid(ReturnValuePtr->user_back());
+  llvm::StoreInst *store =
+      GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back());
   if (!store) return nullptr;
 
   // Now do a first-and-dirty dominance check: just walk up the
@@ -4114,11 +4115,7 @@ void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
 }
 
 static bool isProvablyNull(llvm::Value *addr) {
-  return llvm::isa_and_nonnull<llvm::ConstantPointerNull>(addr);
-}
-
-static bool isProvablyNonNull(Address Addr, CodeGenFunction &CGF) {
-  return llvm::isKnownNonZero(Addr.getBasePointer(), CGF.CGM.getDataLayout());
+  return isa<llvm::ConstantPointerNull>(addr);
 }
 
 /// Emit the actual writing-back of a writeback.
@@ -4126,20 +4123,21 @@ static void emitWriteback(CodeGenFunction &CGF,
                           const CallArgList::Writeback &writeback) {
   const LValue &srcLV = writeback.Source;
   Address srcAddr = srcLV.getAddress(CGF);
-  assert(!isProvablyNull(srcAddr.getBasePointer()) &&
+  assert(!isProvablyNull(srcAddr.getPointer()) &&
          "shouldn't have writeback for provably null argument");
 
   llvm::BasicBlock *contBB = nullptr;
 
   // If the argument wasn't provably non-null, we need to null check
   // before doing the store.
-  bool provablyNonNull = isProvablyNonNull(srcAddr, CGF);
-
+  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
+                                              CGF.CGM.getDataLayout());
   if (!provablyNonNull) {
     llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
     contBB = CGF.createBasicBlock("icr.done");
 
-    llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
+    llvm::Value *isNull =
+      CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
     CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
     CGF.EmitBlock(writebackBB);
   }
@@ -4243,7 +4241,7 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
       CGF.ConvertTypeForMem(CRE->getType()->getPointeeType());
 
   // If the address is a constant null, just pass the appropriate null.
-  if (isProvablyNull(srcAddr.getBasePointer())) {
+  if (isProvablyNull(srcAddr.getPointer())) {
     args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
              CRE->getType());
     return;
@@ -4272,16 +4270,17 @@ static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
   // If the address is *not* known to be non-null, we need to switch.
   llvm::Value *finalArgument;
 
-  bool provablyNonNull = isProvablyNonNull(srcAddr, CGF);
-
+  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
+                                              CGF.CGM.getDataLayout());
   if (provablyNonNull) {
-    finalArgument = temp.emitRawPointer(CGF);
+    finalArgument = temp.getPointer();
   } else {
-    llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
+    llvm::Value *isNull =
+      CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
 
-    finalArgument = CGF.Builder.CreateSelect(
-        isNull, llvm::ConstantPointerNull::get(destType),
-        temp.emitRawPointer(CGF), "icr.argument");
+    finalArgument = CGF.Builder.CreateSelect(isNull,
+                                   llvm::ConstantPointerNull::get(destType),
+                                             temp.getPointer(), "icr.argument");
 
     // If we need to copy, then the load has to be conditional, which
     // means we need control flow.
@@ -4405,16 +4404,6 @@ void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
   EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, std::nullopt);
 }
 
-void CodeGenFunction::EmitNonNullArgCheck(Address Addr, QualType ArgType,
-                                          SourceLocation ArgLoc,
-                                          AbstractCallee AC, unsigned ParmNum) {
-  if (!AC.getDecl() || !(SanOpts.has(SanitizerKind::NonnullAttribute) ||
-                         SanOpts.has(SanitizerKind::NullabilityArg)))
-    return;
-
-  EmitNonNullArgCheck(RValue::get(Addr, *this), ArgType, ArgLoc, AC, ParmNum);
-}
-
 // Check if the call is going to use the inalloca convention. This needs to
 // agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
 // later, so we can't check it directly.
@@ -4755,20 +4744,10 @@ CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
 llvm::CallInst *
 CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
                                          const llvm::Twine &name) {
-  return EmitNounwindRuntimeCall(callee, ArrayRef<llvm::Value *>(), name);
+  return EmitNounwindRuntimeCall(callee, std::nullopt, name);
 }
 
 /// Emits a call to the given nounwind runtime function.
-llvm::CallInst *
-CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
-                                         ArrayRef<Address> args,
-                                         const llvm::Twine &name) {
-  SmallVector<llvm::Value *, 3> values;
-  for (auto arg : args)
-    values.push_back(arg.emitRawPointer(*this));
-  return EmitNounwindRuntimeCall(callee, values, name);
-}
-
 llvm::CallInst *
 CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
                                          ArrayRef<llvm::Value *> args,
@@ -5047,7 +5026,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
 
   // If we're using inalloca, insert the allocation after the stack save.
   // FIXME: Do this earlier rather than hacking it in here!
-  RawAddress ArgMemory = RawAddress::invalid();
+  Address ArgMemory = Address::invalid();
   if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
     const llvm::DataLayout &DL = CGM.getDataLayout();
     llvm::Instruction *IP = CallArgs.getStackBase();
@@ -5063,7 +5042,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
     AI->setAlignment(Align.getAsAlign());
     AI->setUsedWithInAlloca(true);
     assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
-    ArgMemory = RawAddress(AI, ArgStruct, Align);
+    ArgMemory = Address(AI, ArgStruct, Align);
   }
 
   ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
@@ -5072,11 +5051,11 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
   // If the call returns a temporary with struct return, create a temporary
   // alloca to hold the result, unless one is given to us.
   Address SRetPtr = Address::invalid();
-  RawAddress SRetAlloca = RawAddress::invalid();
+  Address SRetAlloca = Address::invalid();
   llvm::Value *UnusedReturnSizePtr = nullptr;
   if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) {
     if (!ReturnValue.isNull()) {
-      SRetPtr = ReturnValue.getAddress();
+      SRetPtr = ReturnValue.getValue();
     } else {
       SRetPtr = CreateMemTemp(RetTy, "tmp", &SRetAlloca);
       if (HaveInsertPoint() && ReturnValue.isUnused()) {
@@ -5086,16 +5065,15 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       }
     }
     if (IRFunctionArgs.hasSRetArg()) {
-      IRCallArgs[IRFunctionArgs.getSRetArgNo()] =
-          getAsNaturalPointerTo(SRetPtr, RetTy);
+      IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer();
     } else if (RetAI.isInAlloca()) {
       Address Addr =
           Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex());
-      Builder.CreateStore(getAsNaturalPointerTo(SRetPtr, RetTy), Addr);
+      Builder.CreateStore(SRetPtr.getPointer(), Addr);
     }
   }
 
-  RawAddress swiftErrorTemp = RawAddress::invalid();
+  Address swiftErrorTemp = Address::invalid();
   Address swiftErrorArg = Address::invalid();
 
   // When passing arguments using temporary allocas, we need to add the
@@ -5128,9 +5106,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       assert(NumIRArgs == 0);
       assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
       if (I->isAggregate()) {
-        RawAddress Addr = I->hasLValue()
-                              ? I->getKnownLValue().getAddress(*this)
-                              : I->getKnownRValue().getAggregateAddress();
+        Address Addr = I->hasLValue()
+                           ? I->getKnownLValue().getAddress(*this)
+                           : I->getKnownRValue().getAggregateAddress();
         llvm::Instruction *Placeholder =
             cast<llvm::Instruction>(Addr.getPointer());
 
@@ -5154,7 +5132,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       } else if (ArgInfo.getInAllocaIndirect()) {
         // Make a temporary alloca and store the address of it into the argument
         // struct.
-        RawAddress Addr = CreateMemTempWithoutCast(
+        Address Addr = CreateMemTempWithoutCast(
             I->Ty, getContext().getTypeAlignInChars(I->Ty),
             "indirect-arg-temp");
         I->copyInto(*this, Addr);
@@ -5176,12 +5154,12 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       assert(NumIRArgs == 1);
       if (!I->isAggregate()) {
         // Make a temporary alloca to pass the argument.
-        RawAddress Addr = CreateMemTempWithoutCast(
+        Address Addr = CreateMemTempWithoutCast(
             I->Ty, ArgInfo.getIndirectAlign(), "indirect-arg-temp");
 
-        llvm::Value *Val = getAsNaturalPointerTo(Addr, I->Ty);
+        llvm::Value *Val = Addr.getPointer();
         if (ArgHasMaybeUndefAttr)
-          Val = Builder.CreateFreeze(Val);
+          Val = Builder.CreateFreeze(Addr.getPointer());
         IRCallArgs[FirstIRArg] = Val;
 
         I->copyInto(*this, Addr);
@@ -5197,6 +5175,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
         Address Addr = I->hasLValue()
                            ? I->getKnownLValue().getAddress(*this)
                            : I->getKnownRValue().getAggregateAddress();
+        llvm::Value *V = Addr.getPointer();
         CharUnits Align = ArgInfo.getIndirectAlign();
         const llvm::DataLayout *TD = &CGM.getDataLayout();
 
@@ -5207,9 +5186,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
 
         bool NeedCopy = false;
         if (Addr.getAlignment() < Align &&
-            llvm::getOrEnforceKnownAlignment(Addr.emitRawPointer(*this),
-                                             Align.getAsAlign(),
-                                             *TD) < Align.getAsAlign()) {
+            llvm::getOrEnforceKnownAlignment(V, Align.getAsAlign(), *TD) <
+                Align.getAsAlign()) {
           NeedCopy = true;
         } else if (I->hasLValue()) {
           auto LV = I->getKnownLValue();
@@ -5240,11 +5218,11 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
 
         if (NeedCopy) {
           // Create an aligned temporary, and copy to it.
-          RawAddress AI = CreateMemTempWithoutCast(
+          Address AI = CreateMemTempWithoutCast(
               I->Ty, ArgInfo.getIndirectAlign(), "byval-temp");
-          llvm::Value *Val = getAsNaturalPointerTo(AI, I->Ty);
+          llvm::Value *Val = AI.getPointer();
           if (ArgHasMaybeUndefAttr)
-            Val = Builder.CreateFreeze(Val);
+            Val = Builder.CreateFreeze(AI.getPointer());
           IRCallArgs[FirstIRArg] = Val;
 
           // Emit lifetime markers for the temporary alloca.
@@ -5261,7 +5239,6 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
           I->copyInto(*this, AI);
         } else {
           // Skip the extra memcpy call.
-          llvm::Value *V = getAsNaturalPointerTo(Addr, I->Ty);
           auto *T = llvm::PointerType::get(
               CGM.getLLVMContext(), CGM.getDataLayout().getAllocaAddrSpace());
 
@@ -5301,8 +5278,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
           assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
 
           QualType pointeeTy = I->Ty->getPointeeType();
-          swiftErrorArg = makeNaturalAddressForPointer(
-              V, pointeeTy, getContext().getTypeAlignInChars(pointeeTy));
+          swiftErrorArg = Address(V, ConvertTypeForMem(pointeeTy),
+                                  getContext().getTypeAlignInChars(pointeeTy));
 
           swiftErrorTemp =
             CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
@@ -5439,7 +5416,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
 
       llvm::Value *tempSize = nullptr;
       Address addr = Address::invalid();
-      RawAddress AllocaAddr = RawAddress::invalid();
+      Address AllocaAddr = Address::invalid();
       if (I->isAggregate()) {
         addr = I->hasLValue() ? I->getKnownLValue().getAddress(*this)
                               : I->getKnownRValue().getAggregateAddress();
@@ -5873,7 +5850,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
           return RValue::getComplex(std::make_pair(Real, Imag));
         }
         case TEK_Aggregate: {
-          Address DestPtr = ReturnValue.getAddress();
+          Address DestPtr = ReturnValue.getValue();
           bool DestIsVolatile = ReturnValue.isVolatile();
 
           if (!DestPtr.isValid()) {
diff --git a/clang/lib/CodeGen/CGCall.h b/clang/lib/CodeGen/CGCall.h
index 6b676ac196db..1bd48a072593 100644
--- a/clang/lib/CodeGen/CGCall.h
+++ b/clang/lib/CodeGen/CGCall.h
@@ -377,7 +377,6 @@ public:
   Address getValue() const { return Addr; }
   bool isUnused() const { return IsUnused; }
   bool isExternallyDestructed() const { return IsExternallyDestructed; }
-  Address getAddress() const { return Addr; }
 };
 
 /// Adds attributes to \p F according to our \p CodeGenOpts and \p LangOpts, as
diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp
index 8c1c8ee455d2..34319381901a 100644
--- a/clang/lib/CodeGen/CGClass.cpp
+++ b/clang/lib/CodeGen/CGClass.cpp
@@ -139,9 +139,8 @@ Address CodeGenFunction::LoadCXXThisAddress() {
     CXXThisAlignment = CGM.getClassPointerAlignment(MD->getParent());
   }
 
-  return makeNaturalAddressForPointer(
-      LoadCXXThis(), MD->getFunctionObjectParameterType(), CXXThisAlignment,
-      false, nullptr, nullptr, KnownNonNull);
+  llvm::Type *Ty = ConvertType(MD->getFunctionObjectParameterType());
+  return Address(LoadCXXThis(), Ty, CXXThisAlignment, KnownNonNull);
 }
 
 /// Emit the address of a field using a member data pointer.
@@ -271,7 +270,7 @@ ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr,
   }
 
   // Apply the base offset.
-  llvm::Value *ptr = addr.emitRawPointer(CGF);
+  llvm::Value *ptr = addr.getPointer();
   ptr = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ptr, baseOffset, "add.ptr");
 
   // If we have a virtual component, the alignment of the result will
@@ -339,8 +338,8 @@ Address CodeGenFunction::GetAddressOfBaseClass(
     if (sanitizePerformTypeCheck()) {
       SanitizerSet SkippedChecks;
       SkippedChecks.set(SanitizerKind::Null, !NullCheckValue);
-      EmitTypeCheck(TCK_Upcast, Loc, Value.emitRawPointer(*this), DerivedTy,
-                    DerivedAlign, SkippedChecks);
+      EmitTypeCheck(TCK_Upcast, Loc, Value.getPointer(),
+                    DerivedTy, DerivedAlign, SkippedChecks);
     }
     return Value.withElementType(BaseValueTy);
   }
@@ -355,7 +354,7 @@ Address CodeGenFunction::GetAddressOfBaseClass(
     llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull");
     endBB = createBasicBlock("cast.end");
 
-    llvm::Value *isNull = Builder.CreateIsNull(Value);
+    llvm::Value *isNull = Builder.CreateIsNull(Value.getPointer());
     Builder.CreateCondBr(isNull, endBB, notNullBB);
     EmitBlock(notNullBB);
   }
@@ -364,15 +363,14 @@ Address CodeGenFunction::GetAddressOfBaseClass(
     SanitizerSet SkippedChecks;
     SkippedChecks.set(SanitizerKind::Null, true);
     EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc,
-                  Value.emitRawPointer(*this), DerivedTy, DerivedAlign,
-                  SkippedChecks);
+                  Value.getPointer(), DerivedTy, DerivedAlign, SkippedChecks);
   }
 
   // Compute the virtual offset.
   llvm::Value *VirtualOffset = nullptr;
   if (VBase) {
     VirtualOffset =
-        CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase);
+      CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase);
   }
 
   // Apply both offsets.
@@ -389,7 +387,7 @@ Address CodeGenFunction::GetAddressOfBaseClass(
     EmitBlock(endBB);
 
     llvm::PHINode *PHI = Builder.CreatePHI(PtrTy, 2, "cast.result");
-    PHI->addIncoming(Value.emitRawPointer(*this), notNullBB);
+    PHI->addIncoming(Value.getPointer(), notNullBB);
     PHI->addIncoming(llvm::Constant::getNullValue(PtrTy), origBB);
     Value = Value.withPointer(PHI, NotKnownNonNull);
   }
@@ -426,19 +424,15 @@ CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,
     CastNotNull = createBasicBlock("cast.notnull");
     CastEnd = createBasicBlock("cast.end");
 
-    llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr);
+    llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr.getPointer());
     Builder.CreateCondBr(IsNull, CastNull, CastNotNull);
     EmitBlock(CastNotNull);
   }
 
   // Apply the offset.
-  Address Addr = BaseAddr.withElementType(Int8Ty);
-  Addr = Builder.CreateInBoundsGEP(
-      Addr, Builder.CreateNeg(NonVirtualOffset), Int8Ty,
-      CGM.getClassPointerAlignment(Derived), "sub.ptr");
-
-  // Just cast.
-  Addr = Addr.withElementType(DerivedValueTy);
+  llvm::Value *Value = BaseAddr.getPointer();
+  Value = Builder.CreateInBoundsGEP(
+      Int8Ty, Value, Builder.CreateNeg(NonVirtualOffset), "sub.ptr");
 
   // Produce a PHI if we had a null-check.
   if (NullCheckValue) {
@@ -447,15 +441,13 @@ CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,
     Builder.CreateBr(CastEnd);
     EmitBlock(CastEnd);
 
-    llvm::Value *Value = Addr.emitRawPointer(*this);
     llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2);
     PHI->addIncoming(Value, CastNotNull);
     PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull);
-    return Address(PHI, Addr.getElementType(),
-                   CGM.getClassPointerAlignment(Derived));
+    Value = PHI;
   }
 
-  return Addr;
+  return Address(Value, DerivedValueTy, CGM.getClassPointerAlignment(Derived));
 }
 
 llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,
@@ -1727,7 +1719,7 @@ namespace {
       // Use the base class declaration location as inline DebugLocation. All
       // fields of the class are destroyed.
       DeclAsInlineDebugLocation InlineHere(CGF, *BaseClass);
-      EmitSanitizerDtorFieldsCallback(CGF, Addr.emitRawPointer(CGF),
+      EmitSanitizerDtorFieldsCallback(CGF, Addr.getPointer(),
                                       BaseSize.getQuantity());
 
       // Prevent the current stack frame from disappearing from the stack trace.
@@ -2030,7 +2022,7 @@ void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
 
   // Find the end of the array.
   llvm::Type *elementType = arrayBase.getElementType();
-  llvm::Value *arrayBegin = arrayBase.emitRawPointer(*this);
+  llvm::Value *arrayBegin = arrayBase.getPointer();
   llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(
       elementType, arrayBegin, numElements, "arrayctor.end");
 
@@ -2126,15 +2118,14 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
   Address This = ThisAVS.getAddress();
   LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace();
   LangAS ThisAS = D->getFunctionObjectParameterType().getAddressSpace();
-  llvm::Value *ThisPtr =
-      getAsNaturalPointerTo(This, D->getThisType()->getPointeeType());
+  llvm::Value *ThisPtr = This.getPointer();
 
   if (SlotAS != ThisAS) {
     unsigned TargetThisAS = getContext().getTargetAddressSpace(ThisAS);
     llvm::Type *NewType =
         llvm::PointerType::get(getLLVMContext(), TargetThisAS);
-    ThisPtr = getTargetHooks().performAddrSpaceCast(*this, ThisPtr, ThisAS,
-                                                    SlotAS, NewType);
+    ThisPtr = getTargetHooks().performAddrSpaceCast(*this, This.getPointer(),
+                                                    ThisAS, SlotAS, NewType);
   }
 
   // Push the this ptr.
@@ -2203,7 +2194,7 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
   const CXXRecordDecl *ClassDecl = D->getParent();
 
   if (!NewPointerIsChecked)
-    EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This,
+    EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This.getPointer(),
                   getContext().getRecordType(ClassDecl), CharUnits::Zero());
 
   if (D->isTrivial() && D->isDefaultConstructor()) {
@@ -2216,9 +2207,10 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
   // model that copy.
   if (isMemcpyEquivalentSpecialMember(D)) {
     assert(Args.size() == 2 && "unexpected argcount for trivial ctor");
+
     QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType();
-    Address Src = makeNaturalAddressForPointer(
-        Args[1].getRValue(*this).getScalarVal(), SrcTy);
+    Address Src = Address(Args[1].getRValue(*this).getScalarVal(), ConvertTypeForMem(SrcTy),
+                                      CGM.getNaturalTypeAlignment(SrcTy));
     LValue SrcLVal = MakeAddrLValue(Src, SrcTy);
     QualType DestTy = getContext().getTypeDeclType(ClassDecl);
     LValue DestLVal = MakeAddrLValue(This, DestTy);
@@ -2271,9 +2263,7 @@ void CodeGenFunction::EmitInheritedCXXConstructorCall(
     const CXXConstructorDecl *D, bool ForVirtualBase, Address This,
     bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) {
   CallArgList Args;
-  CallArg ThisArg(RValue::get(getAsNaturalPointerTo(
-                      This, D->getThisType()->getPointeeType())),
-                  D->getThisType());
+  CallArg ThisArg(RValue::get(This.getPointer()), D->getThisType());
 
   // Forward the parameters.
   if (InheritedFromVBase &&
@@ -2398,14 +2388,12 @@ CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
   CallArgList Args;
 
   // Push the this ptr.
-  Args.add(RValue::get(getAsNaturalPointerTo(This, D->getThisType())),
-           D->getThisType());
+  Args.add(RValue::get(This.getPointer()), D->getThisType());
 
   // Push the src ptr.
   QualType QT = *(FPT->param_type_begin());
   llvm::Type *t = CGM.getTypes().ConvertType(QT);
-  llvm::Value *Val = getAsNaturalPointerTo(Src, D->getThisType());
-  llvm::Value *SrcVal = Builder.CreateBitCast(Val, t);
+  llvm::Value *SrcVal = Builder.CreateBitCast(Src.getPointer(), t);
   Args.add(RValue::get(SrcVal), QT);
 
   // Skip over first argument (Src).
@@ -2430,9 +2418,7 @@ CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
 
   // this
   Address This = LoadCXXThisAddress();
-  DelegateArgs.add(RValue::get(getAsNaturalPointerTo(
-                       This, (*I)->getType()->getPointeeType())),
-                   (*I)->getType());
+  DelegateArgs.add(RValue::get(This.getPointer()), (*I)->getType());
   ++I;
 
   // FIXME: The location of the VTT parameter in the parameter list is
@@ -2789,7 +2775,7 @@ void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, Address Derived,
 
   if (MayBeNull) {
     llvm::Value *DerivedNotNull =
-        Builder.CreateIsNotNull(Derived.emitRawPointer(*this), "cast.nonnull");
+        Builder.CreateIsNotNull(Derived.getPointer(), "cast.nonnull");
 
     llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check");
     ContBlock = createBasicBlock("cast.cont");
@@ -2990,7 +2976,7 @@ void CodeGenFunction::EmitLambdaBlockInvokeBody() {
 
   QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
   Address ThisPtr = GetAddrOfBlockDecl(variable);
-  CallArgs.add(RValue::get(getAsNaturalPointerTo(ThisPtr, ThisType)), ThisType);
+  CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType);
 
   // Add the rest of the parameters.
   for (auto *param : BD->parameters())
@@ -3018,7 +3004,7 @@ void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) {
   QualType LambdaType = getContext().getRecordType(Lambda);
   QualType ThisType = getContext().getPointerType(LambdaType);
   Address ThisPtr = CreateMemTemp(LambdaType, "unused.capture");
-  CallArgs.add(RValue::get(ThisPtr.emitRawPointer(*this)), ThisType);
+  CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType);
 
   EmitLambdaDelegatingInvokeBody(MD, CallArgs);
 }
diff --git a/clang/lib/CodeGen/CGCleanup.cpp b/clang/lib/CodeGen/CGCleanup.cpp
index e6f8e6873004..f87caf050eea 100644
--- a/clang/lib/CodeGen/CGCleanup.cpp
+++ b/clang/lib/CodeGen/CGCleanup.cpp
@@ -27,7 +27,7 @@ bool DominatingValue<RValue>::saved_type::needsSaving(RValue rv) {
   if (rv.isScalar())
     return DominatingLLVMValue::needsSaving(rv.getScalarVal());
   if (rv.isAggregate())
-    return DominatingValue<Address>::needsSaving(rv.getAggregateAddress());
+    return DominatingLLVMValue::needsSaving(rv.getAggregatePointer());
   return true;
 }
 
@@ -35,40 +35,69 @@ DominatingValue<RValue>::saved_type
 DominatingValue<RValue>::saved_type::save(CodeGenFunction &CGF, RValue rv) {
   if (rv.isScalar()) {
     llvm::Value *V = rv.getScalarVal();
-    return saved_type(DominatingLLVMValue::save(CGF, V),
-                      DominatingLLVMValue::needsSaving(V) ? ScalarAddress
-                                                          : ScalarLiteral);
+
+    // These automatically dominate and don't need to be saved.
+    if (!DominatingLLVMValue::needsSaving(V))
+      return saved_type(V, nullptr, ScalarLiteral);
+
+    // Everything else needs an alloca.
+    Address addr =
+      CGF.CreateDefaultAlignTempAlloca(V->getType(), "saved-rvalue");
+    CGF.Builder.CreateStore(V, addr);
+    return saved_type(addr.getPointer(), nullptr, ScalarAddress);
   }
 
   if (rv.isComplex()) {
     CodeGenFunction::ComplexPairTy V = rv.getComplexVal();
-    return saved_type(DominatingLLVMValue::save(CGF, V.first),
-                      DominatingLLVMValue::save(CGF, V.second));
+    llvm::Type *ComplexTy =
+        llvm::StructType::get(V.first->getType(), V.second->getType());
+    Address addr = CGF.CreateDefaultAlignTempAlloca(ComplexTy, "saved-complex");
+    CGF.Builder.CreateStore(V.first, CGF.Builder.CreateStructGEP(addr, 0));
+    CGF.Builder.CreateStore(V.second, CGF.Builder.CreateStructGEP(addr, 1));
+    return saved_type(addr.getPointer(), nullptr, ComplexAddress);
   }
 
   assert(rv.isAggregate());
-  Address V = rv.getAggregateAddress();
-  return saved_type(
-      DominatingValue<Address>::save(CGF, V), rv.isVolatileQualified(),
-      DominatingValue<Address>::needsSaving(V) ? AggregateAddress
-                                               : AggregateLiteral);
+  Address V = rv.getAggregateAddress(); // TODO: volatile?
+  if (!DominatingLLVMValue::needsSaving(V.getPointer()))
+    return saved_type(V.getPointer(), V.getElementType(), AggregateLiteral,
+                      V.getAlignment().getQuantity());
+
+  Address addr =
+    CGF.CreateTempAlloca(V.getType(), CGF.getPointerAlign(), "saved-rvalue");
+  CGF.Builder.CreateStore(V.getPointer(), addr);
+  return saved_type(addr.getPointer(), V.getElementType(), AggregateAddress,
+                    V.getAlignment().getQuantity());
 }
 
 /// Given a saved r-value produced by SaveRValue, perform the code
 /// necessary to restore it to usability at the current insertion
 /// point.
 RValue DominatingValue<RValue>::saved_type::restore(CodeGenFunction &CGF) {
+  auto getSavingAddress = [&](llvm::Value *value) {
+    auto *AI = cast<llvm::AllocaInst>(value);
+    return Address(value, AI->getAllocatedType(),
+                   CharUnits::fromQuantity(AI->getAlign().value()));
+  };
   switch (K) {
   case ScalarLiteral:
+    return RValue::get(Value);
   case ScalarAddress:
-    return RValue::get(DominatingLLVMValue::restore(CGF, Vals.first));
+    return RValue::get(CGF.Builder.CreateLoad(getSavingAddress(Value)));
   case AggregateLiteral:
-  case AggregateAddress:
     return RValue::getAggregate(
-        DominatingValue<Address>::restore(CGF, AggregateAddr), IsVolatile);
+        Address(Value, ElementType, CharUnits::fromQuantity(Align)));
+  case AggregateAddress: {
+    auto addr = CGF.Builder.CreateLoad(getSavingAddress(Value));
+    return RValue::getAggregate(
+        Address(addr, ElementType, CharUnits::fromQuantity(Align)));
+  }
   case ComplexAddress: {
-    llvm::Value *real = DominatingLLVMValue::restore(CGF, Vals.first);
-    llvm::Value *imag = DominatingLLVMValue::restore(CGF, Vals.second);
+    Address address = getSavingAddress(Value);
+    llvm::Value *real =
+        CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(address, 0));
+    llvm::Value *imag =
+        CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(address, 1));
     return RValue::getComplex(real, imag);
   }
   }
@@ -265,14 +294,14 @@ void EHScopeStack::popNullFixups() {
     BranchFixups.pop_back();
 }
 
-RawAddress CodeGenFunction::createCleanupActiveFlag() {
+Address CodeGenFunction::createCleanupActiveFlag() {
   // Create a variable to decide whether the cleanup needs to be run.
-  RawAddress active = CreateTempAllocaWithoutCast(
+  Address active = CreateTempAllocaWithoutCast(
       Builder.getInt1Ty(), CharUnits::One(), "cleanup.cond");
 
   // Initialize it to false at a site that's guaranteed to be run
   // before each evaluation.
-  setBeforeOutermostConditional(Builder.getFalse(), active, *this);
+  setBeforeOutermostConditional(Builder.getFalse(), active);
 
   // Initialize it to true at the current location.
   Builder.CreateStore(Builder.getTrue(), active);
@@ -280,7 +309,7 @@ RawAddress CodeGenFunction::createCleanupActiveFlag() {
   return active;
 }
 
-void CodeGenFunction::initFullExprCleanupWithFlag(RawAddress ActiveFlag) {
+void CodeGenFunction::initFullExprCleanupWithFlag(Address ActiveFlag) {
   // Set that as the active flag in the cleanup.
   EHCleanupScope &cleanup = cast<EHCleanupScope>(*EHStack.begin());
   assert(!cleanup.hasActiveFlag() && "cleanup already has active flag?");
@@ -293,17 +322,15 @@ void CodeGenFunction::initFullExprCleanupWithFlag(RawAddress ActiveFlag) {
 void EHScopeStack::Cleanup::anchor() {}
 
 static void createStoreInstBefore(llvm::Value *value, Address addr,
-                                  llvm::Instruction *beforeInst,
-                                  CodeGenFunction &CGF) {
-  auto store = new llvm::StoreInst(value, addr.emitRawPointer(CGF), beforeInst);
+                                  llvm::Instruction *beforeInst) {
+  auto store = new llvm::StoreInst(value, addr.getPointer(), beforeInst);
   store->setAlignment(addr.getAlignment().getAsAlign());
 }
 
 static llvm::LoadInst *createLoadInstBefore(Address addr, const Twine &name,
-                                            llvm::Instruction *beforeInst,
-                                            CodeGenFunction &CGF) {
-  return new llvm::LoadInst(addr.getElementType(), addr.emitRawPointer(CGF),
-                            name, false, addr.getAlignment().getAsAlign(),
+                                            llvm::Instruction *beforeInst) {
+  return new llvm::LoadInst(addr.getElementType(), addr.getPointer(), name,
+                            false, addr.getAlignment().getAsAlign(),
                             beforeInst);
 }
 
@@ -330,8 +357,8 @@ static void ResolveAllBranchFixups(CodeGenFunction &CGF,
     // entry which we're currently popping.
     if (Fixup.OptimisticBranchBlock == nullptr) {
       createStoreInstBefore(CGF.Builder.getInt32(Fixup.DestinationIndex),
-                            CGF.getNormalCleanupDestSlot(), Fixup.InitialBranch,
-                            CGF);
+                            CGF.getNormalCleanupDestSlot(),
+                            Fixup.InitialBranch);
       Fixup.InitialBranch->setSuccessor(0, CleanupEntry);
     }
 
@@ -358,7 +385,7 @@ static llvm::SwitchInst *TransitionToCleanupSwitch(CodeGenFunction &CGF,
   if (llvm::BranchInst *Br = dyn_cast<llvm::BranchInst>(Term)) {
     assert(Br->isUnconditional());
     auto Load = createLoadInstBefore(CGF.getNormalCleanupDestSlot(),
-                                     "cleanup.dest", Term, CGF);
+                                     "cleanup.dest", Term);
     llvm::SwitchInst *Switch =
       llvm::SwitchInst::Create(Load, Br->getSuccessor(0), 4, Block);
     Br->eraseFromParent();
@@ -486,8 +513,8 @@ void CodeGenFunction::PopCleanupBlocks(
     I += Header.getSize();
 
     if (Header.isConditional()) {
-      RawAddress ActiveFlag =
-          reinterpret_cast<RawAddress &>(LifetimeExtendedCleanupStack[I]);
+      Address ActiveFlag =
+          reinterpret_cast<Address &>(LifetimeExtendedCleanupStack[I]);
       initFullExprCleanupWithFlag(ActiveFlag);
       I += sizeof(ActiveFlag);
     }
@@ -861,7 +888,7 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
         if (NormalCleanupDestSlot->hasOneUse()) {
           NormalCleanupDestSlot->user_back()->eraseFromParent();
           NormalCleanupDestSlot->eraseFromParent();
-          NormalCleanupDest = RawAddress::invalid();
+          NormalCleanupDest = Address::invalid();
         }
 
         llvm::BasicBlock *BranchAfter = Scope.getBranchAfterBlock(0);
@@ -885,8 +912,9 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
         // pass the abnormal exit flag to Fn (SEH cleanup)
         cleanupFlags.setHasExitSwitch();
 
-        llvm::LoadInst *Load = createLoadInstBefore(
-            getNormalCleanupDestSlot(), "cleanup.dest", nullptr, *this);
+        llvm::LoadInst *Load =
+          createLoadInstBefore(getNormalCleanupDestSlot(), "cleanup.dest",
+                               nullptr);
         llvm::SwitchInst *Switch =
           llvm::SwitchInst::Create(Load, Default, SwitchCapacity);
 
@@ -933,8 +961,8 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
         if (!Fixup.Destination) continue;
         if (!Fixup.OptimisticBranchBlock) {
           createStoreInstBefore(Builder.getInt32(Fixup.DestinationIndex),
-                                getNormalCleanupDestSlot(), Fixup.InitialBranch,
-                                *this);
+                                getNormalCleanupDestSlot(),
+                                Fixup.InitialBranch);
           Fixup.InitialBranch->setSuccessor(0, NormalEntry);
         }
         Fixup.OptimisticBranchBlock = NormalExit;
@@ -1107,7 +1135,7 @@ void CodeGenFunction::EmitBranchThroughCleanup(JumpDest Dest) {
 
   // Store the index at the start.
   llvm::ConstantInt *Index = Builder.getInt32(Dest.getDestIndex());
-  createStoreInstBefore(Index, getNormalCleanupDestSlot(), BI, *this);
+  createStoreInstBefore(Index, getNormalCleanupDestSlot(), BI);
 
   // Adjust BI to point to the first cleanup block.
   {
@@ -1241,9 +1269,9 @@ static void SetupCleanupBlockActivation(CodeGenFunction &CGF,
     // If we're in a conditional block, ignore the dominating IP and
     // use the outermost conditional branch.
     if (CGF.isInConditionalBranch()) {
-      CGF.setBeforeOutermostConditional(value, var, CGF);
+      CGF.setBeforeOutermostConditional(value, var);
     } else {
-      createStoreInstBefore(value, var, dominatingIP, CGF);
+      createStoreInstBefore(value, var, dominatingIP);
     }
   }
 
@@ -1293,7 +1321,7 @@ void CodeGenFunction::DeactivateCleanupBlock(EHScopeStack::stable_iterator C,
   Scope.setActive(false);
 }
 
-RawAddress CodeGenFunction::getNormalCleanupDestSlot() {
+Address CodeGenFunction::getNormalCleanupDestSlot() {
   if (!NormalCleanupDest.isValid())
     NormalCleanupDest =
       CreateDefaultAlignTempAlloca(Builder.getInt32Ty(), "cleanup.dest.slot");
diff --git a/clang/lib/CodeGen/CGCleanup.h b/clang/lib/CodeGen/CGCleanup.h
index 03e4a29d7b3d..7a7344c07160 100644
--- a/clang/lib/CodeGen/CGCleanup.h
+++ b/clang/lib/CodeGen/CGCleanup.h
@@ -333,7 +333,7 @@ public:
   Address getActiveFlag() const {
     return ActiveFlag;
   }
-  void setActiveFlag(RawAddress Var) {
+  void setActiveFlag(Address Var) {
     assert(Var.getAlignment().isOne());
     ActiveFlag = Var;
   }
diff --git a/clang/lib/CodeGen/CGCoroutine.cpp b/clang/lib/CodeGen/CGCoroutine.cpp
index 93ca711f716f..b7142ec08af9 100644
--- a/clang/lib/CodeGen/CGCoroutine.cpp
+++ b/clang/lib/CodeGen/CGCoroutine.cpp
@@ -867,8 +867,8 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
     EmitStmt(S.getPromiseDeclStmt());
 
     Address PromiseAddr = GetAddrOfLocalVar(S.getPromiseDecl());
-    auto *PromiseAddrVoidPtr = new llvm::BitCastInst(
-        PromiseAddr.emitRawPointer(*this), VoidPtrTy, "", CoroId);
+    auto *PromiseAddrVoidPtr =
+        new llvm::BitCastInst(PromiseAddr.getPointer(), VoidPtrTy, "", CoroId);
     // Update CoroId to refer to the promise. We could not do it earlier because
     // promise local variable was not emitted yet.
     CoroId->setArgOperand(1, PromiseAddrVoidPtr);
diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index 267f2e40a7bb..2ef5ed04af30 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -1461,7 +1461,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
   bool EmitDebugInfo = DI && CGM.getCodeGenOpts().hasReducedDebugInfo();
 
   Address address = Address::invalid();
-  RawAddress AllocaAddr = RawAddress::invalid();
+  Address AllocaAddr = Address::invalid();
   Address OpenMPLocalAddr = Address::invalid();
   if (CGM.getLangOpts().OpenMPIRBuilder)
     OpenMPLocalAddr = OMPBuilderCBHelpers::getAddressOfLocalVariable(*this, &D);
@@ -1524,10 +1524,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
       // return slot, so that we can elide the copy when returning this
       // variable (C++0x [class.copy]p34).
       address = ReturnValue;
-      AllocaAddr =
-          RawAddress(ReturnValue.emitRawPointer(*this),
-                     ReturnValue.getElementType(), ReturnValue.getAlignment());
-      ;
+      AllocaAddr = ReturnValue;
 
       if (const RecordType *RecordTy = Ty->getAs<RecordType>()) {
         const auto *RD = RecordTy->getDecl();
@@ -1538,7 +1535,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
           // to this variable. Set it to zero to indicate that NRVO was not
           // applied.
           llvm::Value *Zero = Builder.getFalse();
-          RawAddress NRVOFlag =
+          Address NRVOFlag =
               CreateTempAlloca(Zero->getType(), CharUnits::One(), "nrvo");
           EnsureInsertPoint();
           Builder.CreateStore(Zero, NRVOFlag);
@@ -1681,7 +1678,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
   }
 
   if (D.hasAttr<AnnotateAttr>() && HaveInsertPoint())
-    EmitVarAnnotations(&D, address.emitRawPointer(*this));
+    EmitVarAnnotations(&D, address.getPointer());
 
   // Make sure we call @llvm.lifetime.end.
   if (emission.useLifetimeMarkers())
@@ -1854,13 +1851,12 @@ void CodeGenFunction::emitZeroOrPatternForAutoVarInit(QualType type,
     llvm::Value *BaseSizeInChars =
         llvm::ConstantInt::get(IntPtrTy, EltSize.getQuantity());
     Address Begin = Loc.withElementType(Int8Ty);
-    llvm::Value *End = Builder.CreateInBoundsGEP(Begin.getElementType(),
-                                                 Begin.emitRawPointer(*this),
-                                                 SizeVal, "vla.end");
+    llvm::Value *End = Builder.CreateInBoundsGEP(
+        Begin.getElementType(), Begin.getPointer(), SizeVal, "vla.end");
     llvm::BasicBlock *OriginBB = Builder.GetInsertBlock();
     EmitBlock(LoopBB);
     llvm::PHINode *Cur = Builder.CreatePHI(Begin.getType(), 2, "vla.cur");
-    Cur->addIncoming(Begin.emitRawPointer(*this), OriginBB);
+    Cur->addIncoming(Begin.getPointer(), OriginBB);
     CharUnits CurAlign = Loc.getAlignment().alignmentOfArrayElement(EltSize);
     auto *I =
         Builder.CreateMemCpy(Address(Cur, Int8Ty, CurAlign),
@@ -2287,7 +2283,7 @@ void CodeGenFunction::emitDestroy(Address addr, QualType type,
     checkZeroLength = false;
   }
 
-  llvm::Value *begin = addr.emitRawPointer(*this);
+  llvm::Value *begin = addr.getPointer();
   llvm::Value *end =
       Builder.CreateInBoundsGEP(addr.getElementType(), begin, length);
   emitArrayDestroy(begin, end, type, elementAlign, destroyer,
@@ -2547,7 +2543,7 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg,
   }
 
   Address DeclPtr = Address::invalid();
-  RawAddress AllocaPtr = Address::invalid();
+  Address AllocaPtr = Address::invalid();
   bool DoStore = false;
   bool IsScalar = hasScalarEvaluationKind(Ty);
   bool UseIndirectDebugAddress = false;
@@ -2559,8 +2555,8 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg,
     // Indirect argument is in alloca address space, which may be different
     // from the default address space.
     auto AllocaAS = CGM.getASTAllocaAddressSpace();
-    auto *V = DeclPtr.emitRawPointer(*this);
-    AllocaPtr = RawAddress(V, DeclPtr.getElementType(), DeclPtr.getAlignment());
+    auto *V = DeclPtr.getPointer();
+    AllocaPtr = DeclPtr;
 
     // For truly ABI indirect arguments -- those that are not `byval` -- store
     // the address of the argument on the stack to preserve debug information.
@@ -2699,7 +2695,7 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg,
   }
 
   if (D.hasAttr<AnnotateAttr>())
-    EmitVarAnnotations(&D, DeclPtr.emitRawPointer(*this));
+    EmitVarAnnotations(&D, DeclPtr.getPointer());
 
   // We can only check return value nullability if all arguments to the
   // function satisfy their nullability preconditions. This makes it necessary
diff --git a/clang/lib/CodeGen/CGException.cpp b/clang/lib/CodeGen/CGException.cpp
index 34f289334a7d..5a9d06da12de 100644
--- a/clang/lib/CodeGen/CGException.cpp
+++ b/clang/lib/CodeGen/CGException.cpp
@@ -397,7 +397,7 @@ namespace {
 void CodeGenFunction::EmitAnyExprToExn(const Expr *e, Address addr) {
   // Make sure the exception object is cleaned up if there's an
   // exception during initialization.
-  pushFullExprCleanup<FreeException>(EHCleanup, addr.emitRawPointer(*this));
+  pushFullExprCleanup<FreeException>(EHCleanup, addr.getPointer());
   EHScopeStack::stable_iterator cleanup = EHStack.stable_begin();
 
   // __cxa_allocate_exception returns a void*;  we need to cast this
@@ -416,8 +416,8 @@ void CodeGenFunction::EmitAnyExprToExn(const Expr *e, Address addr) {
                    /*IsInit*/ true);
 
   // Deactivate the cleanup block.
-  DeactivateCleanupBlock(
-      cleanup, cast<llvm::Instruction>(typedAddr.emitRawPointer(*this)));
+  DeactivateCleanupBlock(cleanup,
+                         cast<llvm::Instruction>(typedAddr.getPointer()));
 }
 
 Address CodeGenFunction::getExceptionSlot() {
@@ -1834,8 +1834,7 @@ Address CodeGenFunction::recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,
                                                    llvm::Value *ParentFP) {
   llvm::CallInst *RecoverCall = nullptr;
   CGBuilderTy Builder(*this, AllocaInsertPt);
-  if (auto *ParentAlloca =
-          dyn_cast_or_null<llvm::AllocaInst>(ParentVar.getBasePointer())) {
+  if (auto *ParentAlloca = dyn_cast<llvm::AllocaInst>(ParentVar.getPointer())) {
     // Mark the variable escaped if nobody else referenced it and compute the
     // localescape index.
     auto InsertPair = ParentCGF.EscapedLocals.insert(
@@ -1852,8 +1851,8 @@ Address CodeGenFunction::recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,
     // If the parent didn't have an alloca, we're doing some nested outlining.
     // Just clone the existing localrecover call, but tweak the FP argument to
     // use our FP value. All other arguments are constants.
-    auto *ParentRecover = cast<llvm::IntrinsicInst>(
-        ParentVar.emitRawPointer(*this)->stripPointerCasts());
+    auto *ParentRecover =
+        cast<llvm::IntrinsicInst>(ParentVar.getPointer()->stripPointerCasts());
     assert(ParentRecover->getIntrinsicID() == llvm::Intrinsic::localrecover &&
            "expected alloca or localrecover in parent LocalDeclMap");
     RecoverCall = cast<llvm::CallInst>(ParentRecover->clone());
@@ -1926,8 +1925,7 @@ void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF,
         if (isa<ImplicitParamDecl>(D) &&
             D->getType() == getContext().VoidPtrTy) {
           assert(D->getName().starts_with("frame_pointer"));
-          FramePtrAddrAlloca =
-              cast<llvm::AllocaInst>(I.second.getBasePointer());
+          FramePtrAddrAlloca = cast<llvm::AllocaInst>(I.second.getPointer());
           break;
         }
       }
@@ -1988,8 +1986,7 @@ void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF,
         LValue ThisFieldLValue =
             EmitLValueForLambdaField(LambdaThisCaptureField);
         if (!LambdaThisCaptureField->getType()->isPointerType()) {
-          CXXThisValue =
-              ThisFieldLValue.getAddress(*this).emitRawPointer(*this);
+          CXXThisValue = ThisFieldLValue.getAddress(*this).getPointer();
         } else {
           CXXThisValue = EmitLoadOfLValue(ThisFieldLValue, SourceLocation())
                              .getScalarVal();
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 59f7cd452610..85f5d739cef4 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -65,21 +65,21 @@ static llvm::cl::opt<bool> ClSanitizeDebugDeoptimization(
 
 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
 /// block.
-RawAddress
-CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits Align,
-                                             const Twine &Name,
-                                             llvm::Value *ArraySize) {
+Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty,
+                                                     CharUnits Align,
+                                                     const Twine &Name,
+                                                     llvm::Value *ArraySize) {
   auto Alloca = CreateTempAlloca(Ty, Name, ArraySize);
   Alloca->setAlignment(Align.getAsAlign());
-  return RawAddress(Alloca, Ty, Align, KnownNonNull);
+  return Address(Alloca, Ty, Align, KnownNonNull);
 }
 
 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
 /// block. The alloca is casted to default address space if necessary.
-RawAddress CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align,
-                                             const Twine &Name,
-                                             llvm::Value *ArraySize,
-                                             RawAddress *AllocaAddr) {
+Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align,
+                                          const Twine &Name,
+                                          llvm::Value *ArraySize,
+                                          Address *AllocaAddr) {
   auto Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize);
   if (AllocaAddr)
     *AllocaAddr = Alloca;
@@ -101,7 +101,7 @@ RawAddress CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align,
         Ty->getPointerTo(DestAddrSpace), /*non-null*/ true);
   }
 
-  return RawAddress(V, Ty, Align, KnownNonNull);
+  return Address(V, Ty, Align, KnownNonNull);
 }
 
 /// CreateTempAlloca - This creates an alloca and inserts it into the entry
@@ -120,29 +120,28 @@ llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
 /// default alignment of the corresponding LLVM type, which is *not*
 /// guaranteed to be related in any way to the expected alignment of
 /// an AST type that might have been lowered to Ty.
-RawAddress CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty,
-                                                         const Twine &Name) {
+Address CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty,
+                                                      const Twine &Name) {
   CharUnits Align =
       CharUnits::fromQuantity(CGM.getDataLayout().getPrefTypeAlign(Ty));
   return CreateTempAlloca(Ty, Align, Name);
 }
 
-RawAddress CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) {
+Address CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) {
   CharUnits Align = getContext().getTypeAlignInChars(Ty);
   return CreateTempAlloca(ConvertType(Ty), Align, Name);
 }
 
-RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name,
-                                          RawAddress *Alloca) {
+Address CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name,
+                                       Address *Alloca) {
   // FIXME: Should we prefer the preferred type alignment here?
   return CreateMemTemp(Ty, getContext().getTypeAlignInChars(Ty), Name, Alloca);
 }
 
-RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align,
-                                          const Twine &Name,
-                                          RawAddress *Alloca) {
-  RawAddress Result = CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name,
-                                       /*ArraySize=*/nullptr, Alloca);
+Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align,
+                                       const Twine &Name, Address *Alloca) {
+  Address Result = CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name,
+                                    /*ArraySize=*/nullptr, Alloca);
 
   if (Ty->isConstantMatrixType()) {
     auto *ArrayTy = cast<llvm::ArrayType>(Result.getElementType());
@@ -155,14 +154,13 @@ RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align,
   return Result;
 }
 
-RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
-                                                     CharUnits Align,
-                                                     const Twine &Name) {
+Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, CharUnits Align,
+                                                  const Twine &Name) {
   return CreateTempAllocaWithoutCast(ConvertTypeForMem(Ty), Align, Name);
 }
 
-RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
-                                                     const Twine &Name) {
+Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
+                                                  const Twine &Name) {
   return CreateMemTempWithoutCast(Ty, getContext().getTypeAlignInChars(Ty),
                                   Name);
 }
@@ -361,7 +359,7 @@ pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M,
     } else {
       CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor(
           GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete));
-      CleanupArg = cast<llvm::Constant>(ReferenceTemporary.emitRawPointer(CGF));
+      CleanupArg = cast<llvm::Constant>(ReferenceTemporary.getPointer());
     }
     CGF.CGM.getCXXABI().registerGlobalDtor(
         CGF, *cast<VarDecl>(M->getExtendingDecl()), CleanupFn, CleanupArg);
@@ -386,10 +384,10 @@ pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M,
   }
 }
 
-static RawAddress createReferenceTemporary(CodeGenFunction &CGF,
-                                           const MaterializeTemporaryExpr *M,
-                                           const Expr *Inner,
-                                           RawAddress *Alloca = nullptr) {
+static Address createReferenceTemporary(CodeGenFunction &CGF,
+                                        const MaterializeTemporaryExpr *M,
+                                        const Expr *Inner,
+                                        Address *Alloca = nullptr) {
   auto &TCG = CGF.getTargetHooks();
   switch (M->getStorageDuration()) {
   case SD_FullExpression:
@@ -418,7 +416,7 @@ static RawAddress createReferenceTemporary(CodeGenFunction &CGF,
               GV->getValueType()->getPointerTo(
                   CGF.getContext().getTargetAddressSpace(LangAS::Default)));
         // FIXME: Should we put the new global into a COMDAT?
-        return RawAddress(C, GV->getValueType(), alignment);
+        return Address(C, GV->getValueType(), alignment);
       }
     return CGF.CreateMemTemp(Ty, "ref.tmp", Alloca);
   }
@@ -450,7 +448,7 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
   auto ownership = M->getType().getObjCLifetime();
   if (ownership != Qualifiers::OCL_None &&
       ownership != Qualifiers::OCL_ExplicitNone) {
-    RawAddress Object = createReferenceTemporary(*this, M, E);
+    Address Object = createReferenceTemporary(*this, M, E);
     if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) {
       llvm::Type *Ty = ConvertTypeForMem(E->getType());
       Object = Object.withElementType(Ty);
@@ -504,8 +502,8 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
   }
 
   // Create and initialize the reference temporary.
-  RawAddress Alloca = Address::invalid();
-  RawAddress Object = createReferenceTemporary(*this, M, E, &Alloca);
+  Address Alloca = Address::invalid();
+  Address Object = createReferenceTemporary(*this, M, E, &Alloca);
   if (auto *Var = dyn_cast<llvm::GlobalVariable>(
           Object.getPointer()->stripPointerCasts())) {
     llvm::Type *TemporaryType = ConvertTypeForMem(E->getType());
@@ -1113,12 +1111,12 @@ llvm::Value *CodeGenFunction::EmitCountedByFieldExpr(
   } else if (const MemberExpr *ME = dyn_cast<MemberExpr>(StructBase)) {
     LValue LV = EmitMemberExpr(ME);
     Address Addr = LV.getAddress(*this);
-    Res = Addr.emitRawPointer(*this);
+    Res = Addr.getPointer();
   } else if (StructBase->getType()->isPointerType()) {
     LValueBaseInfo BaseInfo;
     TBAAAccessInfo TBAAInfo;
     Address Addr = EmitPointerWithAlignment(StructBase, &BaseInfo, &TBAAInfo);
-    Res = Addr.emitRawPointer(*this);
+    Res = Addr.getPointer();
   } else {
     return nullptr;
   }
@@ -1284,7 +1282,8 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo,
           if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) {
             if (BaseInfo)
               BaseInfo->mergeForCast(TargetTypeBaseInfo);
-            Addr.setAlignment(Align);
+            Addr = Address(Addr.getPointer(), Addr.getElementType(), Align,
+                           IsKnownNonNull);
           }
         }
 
@@ -1301,8 +1300,8 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo,
             CGF.ConvertTypeForMem(E->getType()->getPointeeType());
         Addr = Addr.withElementType(ElemTy);
         if (CE->getCastKind() == CK_AddressSpaceConversion)
-          Addr = CGF.Builder.CreateAddrSpaceCast(
-              Addr, CGF.ConvertType(E->getType()), ElemTy);
+          Addr = CGF.Builder.CreateAddrSpaceCast(Addr,
+                                                 CGF.ConvertType(E->getType()));
         return Addr;
       }
       break;
@@ -1365,9 +1364,10 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo,
   // TODO: conditional operators, comma.
 
   // Otherwise, use the alignment of the type.
-  return CGF.makeNaturalAddressForPointer(
-      CGF.EmitScalarExpr(E), E->getType()->getPointeeType(), CharUnits(),
-      /*ForPointeeType=*/true, BaseInfo, TBAAInfo, IsKnownNonNull);
+  CharUnits Align =
+      CGF.CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo);
+  llvm::Type *ElemTy = CGF.ConvertTypeForMem(E->getType()->getPointeeType());
+  return Address(CGF.EmitScalarExpr(E), ElemTy, Align, IsKnownNonNull);
 }
 
 /// EmitPointerWithAlignment - Given an expression of pointer type, try to
@@ -1468,7 +1468,8 @@ LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
       if (IsBaseCXXThis || isa<DeclRefExpr>(ME->getBase()))
         SkippedChecks.set(SanitizerKind::Null, true);
     }
-    EmitTypeCheck(TCK, E->getExprLoc(), LV, E->getType(), SkippedChecks);
+    EmitTypeCheck(TCK, E->getExprLoc(), LV.getPointer(*this), E->getType(),
+                  LV.getAlignment(), SkippedChecks);
   }
   return LV;
 }
@@ -1580,11 +1581,11 @@ LValue CodeGenFunction::EmitLValueHelper(const Expr *E,
       // Defend against branches out of gnu statement expressions surrounded by
       // cleanups.
       Address Addr = LV.getAddress(*this);
-      llvm::Value *V = Addr.getBasePointer();
+      llvm::Value *V = Addr.getPointer();
       Scope.ForceCleanup({&V});
-      Addr.replaceBasePointer(V);
-      return LValue::MakeAddr(Addr, LV.getType(), getContext(),
-                              LV.getBaseInfo(), LV.getTBAAInfo());
+      return LValue::MakeAddr(Addr.withPointer(V, Addr.isKnownNonNull()),
+                              LV.getType(), getContext(), LV.getBaseInfo(),
+                              LV.getTBAAInfo());
     }
     // FIXME: Is it possible to create an ExprWithCleanups that produces a
     // bitfield lvalue or some other non-simple lvalue?
@@ -1928,7 +1929,7 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
                                                LValueBaseInfo BaseInfo,
                                                TBAAAccessInfo TBAAInfo,
                                                bool isNontemporal) {
-  if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getBasePointer()))
+  if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer()))
     if (GV->isThreadLocal())
       Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV),
                               NotKnownNonNull);
@@ -2038,9 +2039,8 @@ llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
 
 // Convert the pointer of \p Addr to a pointer to a vector (the value type of
 // MatrixType), if it points to a array (the memory type of MatrixType).
-static RawAddress MaybeConvertMatrixAddress(RawAddress Addr,
-                                            CodeGenFunction &CGF,
-                                            bool IsVector = true) {
+static Address MaybeConvertMatrixAddress(Address Addr, CodeGenFunction &CGF,
+                                         bool IsVector = true) {
   auto *ArrayTy = dyn_cast<llvm::ArrayType>(Addr.getElementType());
   if (ArrayTy && IsVector) {
     auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
@@ -2077,7 +2077,7 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr,
                                         LValueBaseInfo BaseInfo,
                                         TBAAAccessInfo TBAAInfo,
                                         bool isInit, bool isNontemporal) {
-  if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getBasePointer()))
+  if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer()))
     if (GV->isThreadLocal())
       Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV),
                               NotKnownNonNull);
@@ -2432,12 +2432,14 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
       assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
       llvm::Type *ResultType = IntPtrTy;
       Address dst = EmitPointerWithAlignment(Dst.getBaseIvarExp());
-      llvm::Value *RHS = dst.emitRawPointer(*this);
+      llvm::Value *RHS = dst.getPointer();
       RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast");
-      llvm::Value *LHS = Builder.CreatePtrToInt(LvalueDst.emitRawPointer(*this),
-                                                ResultType, "sub.ptr.lhs.cast");
+      llvm::Value *LHS =
+        Builder.CreatePtrToInt(LvalueDst.getPointer(), ResultType,
+                               "sub.ptr.lhs.cast");
       llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset");
-      CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst, BytesBetween);
+      CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst,
+                                              BytesBetween);
     } else if (Dst.isGlobalObjCRef()) {
       CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst,
                                                 Dst.isThreadLocalRef());
@@ -2768,9 +2770,12 @@ CodeGenFunction::EmitLoadOfReference(LValue RefLVal,
   llvm::LoadInst *Load =
       Builder.CreateLoad(RefLVal.getAddress(*this), RefLVal.isVolatile());
   CGM.DecorateInstructionWithTBAA(Load, RefLVal.getTBAAInfo());
-  return makeNaturalAddressForPointer(Load, RefLVal.getType()->getPointeeType(),
-                                      CharUnits(), /*ForPointeeType=*/true,
-                                      PointeeBaseInfo, PointeeTBAAInfo);
+
+  QualType PointeeType = RefLVal.getType()->getPointeeType();
+  CharUnits Align = CGM.getNaturalTypeAlignment(
+      PointeeType, PointeeBaseInfo, PointeeTBAAInfo,
+      /* forPointeeType= */ true);
+  return Address(Load, ConvertTypeForMem(PointeeType), Align);
 }
 
 LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) {
@@ -2787,9 +2792,10 @@ Address CodeGenFunction::EmitLoadOfPointer(Address Ptr,
                                            LValueBaseInfo *BaseInfo,
                                            TBAAAccessInfo *TBAAInfo) {
   llvm::Value *Addr = Builder.CreateLoad(Ptr);
-  return makeNaturalAddressForPointer(Addr, PtrTy->getPointeeType(),
-                                      CharUnits(), /*ForPointeeType=*/true,
-                                      BaseInfo, TBAAInfo);
+  return Address(Addr, ConvertTypeForMem(PtrTy->getPointeeType()),
+                 CGM.getNaturalTypeAlignment(PtrTy->getPointeeType(), BaseInfo,
+                                             TBAAInfo,
+                                             /*forPointeeType=*/true));
 }
 
 LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr,
@@ -2985,7 +2991,7 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
                                         /* BaseInfo= */ nullptr,
                                         /* TBAAInfo= */ nullptr,
                                         /* forPointeeType= */ true);
-        Addr = makeNaturalAddressForPointer(Val, T, Alignment);
+        Addr = Address(Val, ConvertTypeForMem(E->getType()), Alignment);
       }
       return MakeAddrLValue(Addr, T, AlignmentSource::Decl);
     }
@@ -3017,12 +3023,11 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
             EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD),
                                     CapturedStmtInfo->getContextValue());
         Address LValueAddress = CapLVal.getAddress(*this);
-        CapLVal = MakeAddrLValue(Address(LValueAddress.emitRawPointer(*this),
-                                         LValueAddress.getElementType(),
-                                         getContext().getDeclAlign(VD)),
-                                 CapLVal.getType(),
-                                 LValueBaseInfo(AlignmentSource::Decl),
-                                 CapLVal.getTBAAInfo());
+        CapLVal = MakeAddrLValue(
+            Address(LValueAddress.getPointer(), LValueAddress.getElementType(),
+                    getContext().getDeclAlign(VD)),
+            CapLVal.getType(), LValueBaseInfo(AlignmentSource::Decl),
+            CapLVal.getTBAAInfo());
         // Mark lvalue as nontemporal if the variable is marked as nontemporal
         // in simd context.
         if (getLangOpts().OpenMP &&
@@ -3078,8 +3083,7 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
     // Handle threadlocal function locals.
     if (VD->getTLSKind() != VarDecl::TLS_None)
       addr = addr.withPointer(
-          Builder.CreateThreadLocalAddress(addr.getBasePointer()),
-          NotKnownNonNull);
+          Builder.CreateThreadLocalAddress(addr.getPointer()), NotKnownNonNull);
 
     // Check for OpenMP threadprivate variables.
     if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&
@@ -3347,7 +3351,7 @@ llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {
 
   // Pointers are passed directly, everything else is passed by address.
   if (!V->getType()->isPointerTy()) {
-    RawAddress Ptr = CreateDefaultAlignTempAlloca(V->getType());
+    Address Ptr = CreateDefaultAlignTempAlloca(V->getType());
     Builder.CreateStore(V, Ptr);
     V = Ptr.getPointer();
   }
@@ -3903,21 +3907,6 @@ static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF,
   }
 }
 
-static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
-                                     ArrayRef<llvm::Value *> indices,
-                                     llvm::Type *elementType, bool inbounds,
-                                     bool signedIndices, SourceLocation loc,
-                                     CharUnits align,
-                                     const llvm::Twine &name = "arrayidx") {
-  if (inbounds) {
-    return CGF.EmitCheckedInBoundsGEP(addr, indices, elementType, signedIndices,
-                                      CodeGenFunction::NotSubtraction, loc,
-                                      align, name);
-  } else {
-    return CGF.Builder.CreateGEP(addr, indices, elementType, align, name);
-  }
-}
-
 static CharUnits getArrayElementAlign(CharUnits arrayAlign,
                                       llvm::Value *idx,
                                       CharUnits eltSize) {
@@ -3965,7 +3954,7 @@ static Address wrapWithBPFPreserveStaticOffset(CodeGenFunction &CGF,
 
   llvm::Function *Fn =
       CGF.CGM.getIntrinsic(llvm::Intrinsic::preserve_static_offset);
-  llvm::CallInst *Call = CGF.Builder.CreateCall(Fn, {Addr.emitRawPointer(CGF)});
+  llvm::CallInst *Call = CGF.Builder.CreateCall(Fn, {Addr.getPointer()});
   return Address(Call, Addr.getElementType(), Addr.getAlignment());
 }
 
@@ -4028,7 +4017,7 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
   // We can use that to compute the best alignment of the element.
   CharUnits eltSize = CGF.getContext().getTypeSizeInChars(eltType);
   CharUnits eltAlign =
-      getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize);
+    getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize);
 
   if (hasBPFPreserveStaticOffset(Base))
     addr = wrapWithBPFPreserveStaticOffset(CGF, addr);
@@ -4037,19 +4026,19 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
   auto LastIndex = dyn_cast<llvm::ConstantInt>(indices.back());
   if (!LastIndex ||
       (!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, Base))) {
-    addr = emitArraySubscriptGEP(CGF, addr, indices,
-                                 CGF.ConvertTypeForMem(eltType), inbounds,
-                                 signedIndices, loc, eltAlign, name);
-    return addr;
+    eltPtr = emitArraySubscriptGEP(
+        CGF, addr.getElementType(), addr.getPointer(), indices, inbounds,
+        signedIndices, loc, name);
   } else {
     // Remember the original array subscript for bpf target
     unsigned idx = LastIndex->getZExtValue();
     llvm::DIType *DbgInfo = nullptr;
     if (arrayType)
       DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(*arrayType, loc);
-    eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(
-        addr.getElementType(), addr.emitRawPointer(CGF), indices.size() - 1,
-        idx, DbgInfo);
+    eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(addr.getElementType(),
+                                                        addr.getPointer(),
+                                                        indices.size() - 1,
+                                                        idx, DbgInfo);
   }
 
   return Address(eltPtr, CGF.ConvertTypeForMem(eltType), eltAlign);
@@ -4218,8 +4207,8 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
     CharUnits EltAlign =
       getArrayElementAlign(Addr.getAlignment(), Idx, InterfaceSize);
     llvm::Value *EltPtr =
-        emitArraySubscriptGEP(*this, Int8Ty, Addr.emitRawPointer(*this),
-                              ScaledIdx, false, SignedIndices, E->getExprLoc());
+        emitArraySubscriptGEP(*this, Int8Ty, Addr.getPointer(), ScaledIdx,
+                              false, SignedIndices, E->getExprLoc());
     Addr = Address(EltPtr, OrigBaseElemTy, EltAlign);
   } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
     // If this is A[i] where A is an array, the frontend will have decayed the
@@ -4265,7 +4254,7 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
 
             llvm::Type *CountTy = ConvertType(CountFD->getType());
             llvm::Value *Res = Builder.CreateInBoundsGEP(
-                Int8Ty, Addr.emitRawPointer(*this),
+                Int8Ty, Addr.getPointer(),
                 Builder.getInt32(OffsetDiff.getQuantity()), ".counted_by.gep");
             Res = Builder.CreateAlignedLoad(CountTy, Res, getIntAlign(),
                                             ".counted_by.load");
@@ -4511,9 +4500,9 @@ LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E,
     BaseInfo = ArrayLV.getBaseInfo();
     TBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, ResultExprTy);
   } else {
-    Address Base =
-        emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, TBAAInfo, BaseTy,
-                                ResultExprTy, IsLowerBound);
+    Address Base = emitOMPArraySectionBase(*this, E->getBase(), BaseInfo,
+                                           TBAAInfo, BaseTy, ResultExprTy,
+                                           IsLowerBound);
     EltPtr = emitArraySubscriptGEP(*this, Base, Idx, ResultExprTy,
                                    !getLangOpts().isSignedOverflowDefined(),
                                    /*signedIndices=*/false, E->getExprLoc());
@@ -4600,7 +4589,7 @@ LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
       SkippedChecks.set(SanitizerKind::Alignment, true);
     if (IsBaseCXXThis || isa<DeclRefExpr>(BaseExpr))
       SkippedChecks.set(SanitizerKind::Null, true);
-    EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr, PtrTy,
+    EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr.getPointer(), PtrTy,
                   /*Alignment=*/CharUnits::Zero(), SkippedChecks);
     BaseLV = MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo);
   } else
@@ -4649,8 +4638,8 @@ LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field,
       LambdaLV = EmitLoadOfReferenceLValue(AddrOfExplicitObject, D->getType(),
                                            AlignmentSource::Decl);
     else
-      LambdaLV = MakeAddrLValue(AddrOfExplicitObject,
-                                D->getType().getNonReferenceType());
+      LambdaLV = MakeNaturalAlignAddrLValue(AddrOfExplicitObject.getPointer(),
+                                            D->getType().getNonReferenceType());
   } else {
     QualType LambdaTagType = getContext().getTagDeclType(Field->getParent());
     LambdaLV = MakeNaturalAlignAddrLValue(ThisValue, LambdaTagType);
@@ -4840,8 +4829,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base,
       // information provided by invariant.group.  This is because accessing
       // fields may leak the real address of dynamic object, which could result
       // in miscompilation when leaked pointer would be compared.
-      auto *stripped =
-          Builder.CreateStripInvariantGroup(addr.emitRawPointer(*this));
+      auto *stripped = Builder.CreateStripInvariantGroup(addr.getPointer());
       addr = Address(stripped, addr.getElementType(), addr.getAlignment());
     }
   }
@@ -4860,11 +4848,10 @@ LValue CodeGenFunction::EmitLValueForField(LValue base,
       // Remember the original union field index
       llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(base.getType(),
           rec->getLocation());
-      addr =
-          Address(Builder.CreatePreserveUnionAccessIndex(
-                      addr.emitRawPointer(*this),
-                      getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo),
-                  addr.getElementType(), addr.getAlignment());
+      addr = Address(
+          Builder.CreatePreserveUnionAccessIndex(
+              addr.getPointer(), getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo),
+          addr.getElementType(), addr.getAlignment());
     }
 
     if (FieldType->isReferenceType())
@@ -5101,9 +5088,11 @@ LValue CodeGenFunction::EmitConditionalOperatorLValue(
   if (Info.LHS && Info.RHS) {
     Address lhsAddr = Info.LHS->getAddress(*this);
     Address rhsAddr = Info.RHS->getAddress(*this);
-    Address result = mergeAddressesInConditionalExpr(
-        lhsAddr, rhsAddr, Info.lhsBlock, Info.rhsBlock,
-        Builder.GetInsertBlock(), expr->getType());
+    llvm::PHINode *phi = Builder.CreatePHI(lhsAddr.getType(), 2, "cond-lvalue");
+    phi->addIncoming(lhsAddr.getPointer(), Info.lhsBlock);
+    phi->addIncoming(rhsAddr.getPointer(), Info.rhsBlock);
+    Address result(phi, lhsAddr.getElementType(),
+                   std::min(lhsAddr.getAlignment(), rhsAddr.getAlignment()));
     AlignmentSource alignSource =
         std::max(Info.LHS->getBaseInfo().getAlignmentSource(),
                  Info.RHS->getBaseInfo().getAlignmentSource());
@@ -5190,7 +5179,7 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
     LValue LV = EmitLValue(E->getSubExpr());
     Address V = LV.getAddress(*this);
     const auto *DCE = cast<CXXDynamicCastExpr>(E);
-    return MakeNaturalAlignRawAddrLValue(EmitDynamicCast(V, DCE), E->getType());
+    return MakeNaturalAlignAddrLValue(EmitDynamicCast(V, DCE), E->getType());
   }
 
   case CK_ConstructorConversion:
@@ -5255,8 +5244,8 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
     // C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is
     // performed and the object is not of the derived type.
     if (sanitizePerformTypeCheck())
-      EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(), Derived,
-                    E->getType());
+      EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(),
+                    Derived.getPointer(), E->getType());
 
     if (SanOpts.has(SanitizerKind::CFIDerivedCast))
       EmitVTablePtrCheckForCast(E->getType(), Derived,
@@ -5612,7 +5601,7 @@ LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
 
 LValue
 CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
-  return MakeNaturalAlignRawAddrLValue(EmitCXXTypeidExpr(E), E->getType());
+  return MakeNaturalAlignAddrLValue(EmitCXXTypeidExpr(E), E->getType());
 }
 
 Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) {
diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp
index 143855aa84ca..5190b22bcc16 100644
--- a/clang/lib/CodeGen/CGExprAgg.cpp
+++ b/clang/lib/CodeGen/CGExprAgg.cpp
@@ -294,10 +294,10 @@ void AggExprEmitter::withReturnValueSlot(
   // Otherwise, EmitCall will emit its own, notice that it's "unused", and end
   // its lifetime before we have the chance to emit a proper destructor call.
   bool UseTemp = Dest.isPotentiallyAliased() || Dest.requiresGCollection() ||
-                 (RequiresDestruction && Dest.isIgnored());
+                 (RequiresDestruction && !Dest.getAddress().isValid());
 
   Address RetAddr = Address::invalid();
-  RawAddress RetAllocaAddr = RawAddress::invalid();
+  Address RetAllocaAddr = Address::invalid();
 
   EHScopeStack::stable_iterator LifetimeEndBlock;
   llvm::Value *LifetimeSizePtr = nullptr;
@@ -329,8 +329,7 @@ void AggExprEmitter::withReturnValueSlot(
   if (!UseTemp)
     return;
 
-  assert(Dest.isIgnored() || Dest.emitRawPointer(CGF) !=
-                                 Src.getAggregatePointer(E->getType(), CGF));
+  assert(Dest.isIgnored() || Dest.getPointer() != Src.getAggregatePointer());
   EmitFinalDestCopy(E->getType(), Src);
 
   if (!RequiresDestruction && LifetimeStartInst) {
@@ -449,8 +448,7 @@ AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) {
   llvm::Value *Zero = llvm::ConstantInt::get(CGF.PtrDiffTy, 0);
   llvm::Value *IdxStart[] = { Zero, Zero };
   llvm::Value *ArrayStart = Builder.CreateInBoundsGEP(
-      ArrayPtr.getElementType(), ArrayPtr.emitRawPointer(CGF), IdxStart,
-      "arraystart");
+      ArrayPtr.getElementType(), ArrayPtr.getPointer(), IdxStart, "arraystart");
   CGF.EmitStoreThroughLValue(RValue::get(ArrayStart), Start);
   ++Field;
 
@@ -467,8 +465,7 @@ AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) {
     // End pointer.
     llvm::Value *IdxEnd[] = { Zero, Size };
     llvm::Value *ArrayEnd = Builder.CreateInBoundsGEP(
-        ArrayPtr.getElementType(), ArrayPtr.emitRawPointer(CGF), IdxEnd,
-        "arrayend");
+        ArrayPtr.getElementType(), ArrayPtr.getPointer(), IdxEnd, "arrayend");
     CGF.EmitStoreThroughLValue(RValue::get(ArrayEnd), EndOrLength);
   } else if (Ctx.hasSameType(Field->getType(), Ctx.getSizeType())) {
     // Length.
@@ -519,9 +516,9 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
   // down a level.
   llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0);
   llvm::Value *indices[] = { zero, zero };
-  llvm::Value *begin = Builder.CreateInBoundsGEP(DestPtr.getElementType(),
-                                                 DestPtr.emitRawPointer(CGF),
-                                                 indices, "arrayinit.begin");
+  llvm::Value *begin = Builder.CreateInBoundsGEP(
+      DestPtr.getElementType(), DestPtr.getPointer(), indices,
+      "arrayinit.begin");
 
   CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType);
   CharUnits elementAlign =
@@ -1062,7 +1059,7 @@ void AggExprEmitter::VisitBinCmp(const BinaryOperator *E) {
     if (RV.isScalar())
       return {RV.getScalarVal(), nullptr};
     if (RV.isAggregate())
-      return {RV.getAggregatePointer(E->getType(), CGF), nullptr};
+      return {RV.getAggregatePointer(), nullptr};
     assert(RV.isComplex());
     return RV.getComplexVal();
   };
@@ -1821,7 +1818,7 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr(
     // else, clean it up for -O0 builds and general tidiness.
     if (!pushedCleanup && LV.isSimple())
       if (llvm::GetElementPtrInst *GEP =
-              dyn_cast<llvm::GetElementPtrInst>(LV.emitRawPointer(CGF)))
+              dyn_cast<llvm::GetElementPtrInst>(LV.getPointer(CGF)))
         if (GEP->use_empty())
           GEP->eraseFromParent();
   }
@@ -1852,9 +1849,9 @@ void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
   // destPtr is an array*. Construct an elementType* by drilling down a level.
   llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0);
   llvm::Value *indices[] = {zero, zero};
-  llvm::Value *begin = Builder.CreateInBoundsGEP(destPtr.getElementType(),
-                                                 destPtr.emitRawPointer(CGF),
-                                                 indices, "arrayinit.begin");
+  llvm::Value *begin = Builder.CreateInBoundsGEP(
+      destPtr.getElementType(), destPtr.getPointer(), indices,
+      "arrayinit.begin");
 
   // Prepare to special-case multidimensional array initialization: we avoid
   // emitting multiple destructor loops in that case.
diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp
index 1d2ca37a449b..2adbef6d5512 100644
--- a/clang/lib/CodeGen/CGExprCXX.cpp
+++ b/clang/lib/CodeGen/CGExprCXX.cpp
@@ -280,8 +280,7 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr(
     LValueBaseInfo BaseInfo;
     TBAAAccessInfo TBAAInfo;
     Address ThisValue = EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo);
-    This = MakeAddrLValue(ThisValue, Base->getType()->getPointeeType(),
-                          BaseInfo, TBAAInfo);
+    This = MakeAddrLValue(ThisValue, Base->getType(), BaseInfo, TBAAInfo);
   } else {
     This = EmitLValue(Base);
   }
@@ -354,8 +353,10 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr(
     if (IsImplicitObjectCXXThis || isa<DeclRefExpr>(IOA))
       SkippedChecks.set(SanitizerKind::Null, true);
   }
-  EmitTypeCheck(CodeGenFunction::TCK_MemberCall, CallLoc, This,
-                C.getRecordType(CalleeDecl->getParent()), SkippedChecks);
+  EmitTypeCheck(CodeGenFunction::TCK_MemberCall, CallLoc,
+                This.getPointer(*this),
+                C.getRecordType(CalleeDecl->getParent()),
+                /*Alignment=*/CharUnits::Zero(), SkippedChecks);
 
   // C++ [class.virtual]p12:
   //   Explicit qualification with the scope operator (5.1) suppresses the
@@ -454,7 +455,7 @@ CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
   else
     This = EmitLValue(BaseExpr, KnownNonNull).getAddress(*this);
 
-  EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This.emitRawPointer(*this),
+  EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This.getPointer(),
                 QualType(MPT->getClass(), 0));
 
   // Get the member function pointer.
@@ -1109,10 +1110,9 @@ void CodeGenFunction::EmitNewArrayInitializer(
       // alloca.
       EndOfInit = CreateTempAlloca(BeginPtr.getType(), getPointerAlign(),
                                    "array.init.end");
-      CleanupDominator =
-          Builder.CreateStore(BeginPtr.emitRawPointer(*this), EndOfInit);
-      pushIrregularPartialArrayCleanup(BeginPtr.emitRawPointer(*this),
-                                       EndOfInit, ElementType, ElementAlign,
+      CleanupDominator = Builder.CreateStore(BeginPtr.getPointer(), EndOfInit);
+      pushIrregularPartialArrayCleanup(BeginPtr.getPointer(), EndOfInit,
+                                       ElementType, ElementAlign,
                                        getDestroyer(DtorKind));
       Cleanup = EHStack.stable_begin();
     }
@@ -1124,17 +1124,16 @@ void CodeGenFunction::EmitNewArrayInitializer(
       // element.  TODO: some of these stores can be trivially
       // observed to be unnecessary.
       if (EndOfInit.isValid()) {
-        Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit);
+        Builder.CreateStore(CurPtr.getPointer(), EndOfInit);
       }
       // FIXME: If the last initializer is an incomplete initializer list for
       // an array, and we have an array filler, we can fold together the two
       // initialization loops.
       StoreAnyExprIntoOneUnit(*this, IE, IE->getType(), CurPtr,
                               AggValueSlot::DoesNotOverlap);
-      CurPtr = Address(Builder.CreateInBoundsGEP(CurPtr.getElementType(),
-                                                 CurPtr.emitRawPointer(*this),
-                                                 Builder.getSize(1),
-                                                 "array.exp.next"),
+      CurPtr = Address(Builder.CreateInBoundsGEP(
+                           CurPtr.getElementType(), CurPtr.getPointer(),
+                           Builder.getSize(1), "array.exp.next"),
                        CurPtr.getElementType(),
                        StartAlign.alignmentAtOffset((++i) * ElementSize));
     }
@@ -1188,7 +1187,7 @@ void CodeGenFunction::EmitNewArrayInitializer(
     // FIXME: Share this cleanup with the constructor call emission rather than
     // having it create a cleanup of its own.
     if (EndOfInit.isValid())
-      Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit);
+      Builder.CreateStore(CurPtr.getPointer(), EndOfInit);
 
     // Emit a constructor call loop to initialize the remaining elements.
     if (InitListElements)
@@ -1251,15 +1250,15 @@ void CodeGenFunction::EmitNewArrayInitializer(
   llvm::BasicBlock *ContBB = createBasicBlock("new.loop.end");
 
   // Find the end of the array, hoisted out of the loop.
-  llvm::Value *EndPtr = Builder.CreateInBoundsGEP(
-      BeginPtr.getElementType(), BeginPtr.emitRawPointer(*this), NumElements,
-      "array.end");
+  llvm::Value *EndPtr =
+    Builder.CreateInBoundsGEP(BeginPtr.getElementType(), BeginPtr.getPointer(),
+                              NumElements, "array.end");
 
   // If the number of elements isn't constant, we have to now check if there is
   // anything left to initialize.
   if (!ConstNum) {
-    llvm::Value *IsEmpty = Builder.CreateICmpEQ(CurPtr.emitRawPointer(*this),
-                                                EndPtr, "array.isempty");
+    llvm::Value *IsEmpty =
+      Builder.CreateICmpEQ(CurPtr.getPointer(), EndPtr, "array.isempty");
     Builder.CreateCondBr(IsEmpty, ContBB, LoopBB);
   }
 
@@ -1269,20 +1268,19 @@ void CodeGenFunction::EmitNewArrayInitializer(
   // Set up the current-element phi.
   llvm::PHINode *CurPtrPhi =
       Builder.CreatePHI(CurPtr.getType(), 2, "array.cur");
-  CurPtrPhi->addIncoming(CurPtr.emitRawPointer(*this), EntryBB);
+  CurPtrPhi->addIncoming(CurPtr.getPointer(), EntryBB);
 
   CurPtr = Address(CurPtrPhi, CurPtr.getElementType(), ElementAlign);
 
   // Store the new Cleanup position for irregular Cleanups.
   if (EndOfInit.isValid())
-    Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit);
+    Builder.CreateStore(CurPtr.getPointer(), EndOfInit);
 
   // Enter a partial-destruction Cleanup if necessary.
   if (!CleanupDominator && needsEHCleanup(DtorKind)) {
-    llvm::Value *BeginPtrRaw = BeginPtr.emitRawPointer(*this);
-    llvm::Value *CurPtrRaw = CurPtr.emitRawPointer(*this);
-    pushRegularPartialArrayCleanup(BeginPtrRaw, CurPtrRaw, ElementType,
-                                   ElementAlign, getDestroyer(DtorKind));
+    pushRegularPartialArrayCleanup(BeginPtr.getPointer(), CurPtr.getPointer(),
+                                   ElementType, ElementAlign,
+                                   getDestroyer(DtorKind));
     Cleanup = EHStack.stable_begin();
     CleanupDominator = Builder.CreateUnreachable();
   }
@@ -1298,8 +1296,9 @@ void CodeGenFunction::EmitNewArrayInitializer(
   }
 
   // Advance to the next element by adjusting the pointer type as necessary.
-  llvm::Value *NextPtr = Builder.CreateConstInBoundsGEP1_32(
-      ElementTy, CurPtr.emitRawPointer(*this), 1, "array.next");
+  llvm::Value *NextPtr =
+    Builder.CreateConstInBoundsGEP1_32(ElementTy, CurPtr.getPointer(), 1,
+                                       "array.next");
 
   // Check whether we've gotten to the end of the array and, if so,
   // exit the loop.
@@ -1525,9 +1524,14 @@ static void EnterNewDeleteCleanup(CodeGenFunction &CGF,
 
     typedef CallDeleteDuringNew<DirectCleanupTraits> DirectCleanup;
 
-    DirectCleanup *Cleanup = CGF.EHStack.pushCleanupWithExtra<DirectCleanup>(
-        EHCleanup, E->getNumPlacementArgs(), E->getOperatorDelete(),
-        NewPtr.emitRawPointer(CGF), AllocSize, E->passAlignment(), AllocAlign);
+    DirectCleanup *Cleanup = CGF.EHStack
+      .pushCleanupWithExtra<DirectCleanup>(EHCleanup,
+                                           E->getNumPlacementArgs(),
+                                           E->getOperatorDelete(),
+                                           NewPtr.getPointer(),
+                                           AllocSize,
+                                           E->passAlignment(),
+                                           AllocAlign);
     for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I) {
       auto &Arg = NewArgs[I + NumNonPlacementArgs];
       Cleanup->setPlacementArg(I, Arg.getRValue(CGF), Arg.Ty);
@@ -1538,7 +1542,7 @@ static void EnterNewDeleteCleanup(CodeGenFunction &CGF,
 
   // Otherwise, we need to save all this stuff.
   DominatingValue<RValue>::saved_type SavedNewPtr =
-      DominatingValue<RValue>::save(CGF, RValue::get(NewPtr, CGF));
+    DominatingValue<RValue>::save(CGF, RValue::get(NewPtr.getPointer()));
   DominatingValue<RValue>::saved_type SavedAllocSize =
     DominatingValue<RValue>::save(CGF, RValue::get(AllocSize));
 
@@ -1616,14 +1620,14 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
     // In these cases, discard the computed alignment and use the
     // formal alignment of the allocated type.
     if (BaseInfo.getAlignmentSource() != AlignmentSource::Decl)
-      allocation.setAlignment(allocAlign);
+      allocation = allocation.withAlignment(allocAlign);
 
     // Set up allocatorArgs for the call to operator delete if it's not
     // the reserved global operator.
     if (E->getOperatorDelete() &&
         !E->getOperatorDelete()->isReservedGlobalPlacementOperator()) {
       allocatorArgs.add(RValue::get(allocSize), getContext().getSizeType());
-      allocatorArgs.add(RValue::get(allocation, *this), arg->getType());
+      allocatorArgs.add(RValue::get(allocation.getPointer()), arg->getType());
     }
 
   } else {
@@ -1711,7 +1715,8 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
     llvm::BasicBlock *notNullBB = createBasicBlock("new.notnull");
     contBB = createBasicBlock("new.cont");
 
-    llvm::Value *isNull = Builder.CreateIsNull(allocation, "new.isnull");
+    llvm::Value *isNull =
+      Builder.CreateIsNull(allocation.getPointer(), "new.isnull");
     Builder.CreateCondBr(isNull, contBB, notNullBB);
     EmitBlock(notNullBB);
   }
@@ -1757,12 +1762,12 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
   SkippedChecks.set(SanitizerKind::Null, nullCheck);
   EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall,
                 E->getAllocatedTypeSourceInfo()->getTypeLoc().getBeginLoc(),
-                result, allocType, result.getAlignment(), SkippedChecks,
-                numElements);
+                result.getPointer(), allocType, result.getAlignment(),
+                SkippedChecks, numElements);
 
   EmitNewInitializer(*this, E, allocType, elementTy, result, numElements,
                      allocSizeWithoutCookie);
-  llvm::Value *resultPtr = result.emitRawPointer(*this);
+  llvm::Value *resultPtr = result.getPointer();
   if (E->isArray()) {
     // NewPtr is a pointer to the base element type.  If we're
     // allocating an array of arrays, we'll need to cast back to the
@@ -1906,8 +1911,7 @@ static void EmitDestroyingObjectDelete(CodeGenFunction &CGF,
     CGF.CGM.getCXXABI().emitVirtualObjectDelete(CGF, DE, Ptr, ElementType,
                                                 Dtor);
   else
-    CGF.EmitDeleteCall(DE->getOperatorDelete(), Ptr.emitRawPointer(CGF),
-                       ElementType);
+    CGF.EmitDeleteCall(DE->getOperatorDelete(), Ptr.getPointer(), ElementType);
 }
 
 /// Emit the code for deleting a single object.
@@ -1923,7 +1927,8 @@ static bool EmitObjectDelete(CodeGenFunction &CGF,
   //   dynamic type, the static type shall be a base class of the dynamic type
   //   of the object to be deleted and the static type shall have a virtual
   //   destructor or the behavior is undefined.
-  CGF.EmitTypeCheck(CodeGenFunction::TCK_MemberCall, DE->getExprLoc(), Ptr,
+  CGF.EmitTypeCheck(CodeGenFunction::TCK_MemberCall,
+                    DE->getExprLoc(), Ptr.getPointer(),
                     ElementType);
 
   const FunctionDecl *OperatorDelete = DE->getOperatorDelete();
@@ -1972,8 +1977,9 @@ static bool EmitObjectDelete(CodeGenFunction &CGF,
   // Make sure that we call delete even if the dtor throws.
   // This doesn't have to a conditional cleanup because we're going
   // to pop it off in a second.
-  CGF.EHStack.pushCleanup<CallObjectDelete>(
-      NormalAndEHCleanup, Ptr.emitRawPointer(CGF), OperatorDelete, ElementType);
+  CGF.EHStack.pushCleanup<CallObjectDelete>(NormalAndEHCleanup,
+                                            Ptr.getPointer(),
+                                            OperatorDelete, ElementType);
 
   if (Dtor)
     CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
@@ -2060,7 +2066,7 @@ static void EmitArrayDelete(CodeGenFunction &CGF,
     CharUnits elementAlign =
       deletedPtr.getAlignment().alignmentOfArrayElement(elementSize);
 
-    llvm::Value *arrayBegin = deletedPtr.emitRawPointer(CGF);
+    llvm::Value *arrayBegin = deletedPtr.getPointer();
     llvm::Value *arrayEnd = CGF.Builder.CreateInBoundsGEP(
       deletedPtr.getElementType(), arrayBegin, numElements, "delete.end");
 
@@ -2091,7 +2097,7 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) {
   llvm::BasicBlock *DeleteNotNull = createBasicBlock("delete.notnull");
   llvm::BasicBlock *DeleteEnd = createBasicBlock("delete.end");
 
-  llvm::Value *IsNull = Builder.CreateIsNull(Ptr, "isnull");
+  llvm::Value *IsNull = Builder.CreateIsNull(Ptr.getPointer(), "isnull");
 
   Builder.CreateCondBr(IsNull, DeleteEnd, DeleteNotNull);
   EmitBlock(DeleteNotNull);
@@ -2126,8 +2132,10 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) {
       GEP.push_back(Zero);
     }
 
-    Ptr = Builder.CreateInBoundsGEP(Ptr, GEP, ConvertTypeForMem(DeleteTy),
-                                    Ptr.getAlignment(), "del.first");
+    Ptr = Address(Builder.CreateInBoundsGEP(Ptr.getElementType(),
+                                            Ptr.getPointer(), GEP, "del.first"),
+                  ConvertTypeForMem(DeleteTy), Ptr.getAlignment(),
+                  Ptr.isKnownNonNull());
   }
 
   assert(ConvertTypeForMem(DeleteTy) == Ptr.getElementType());
@@ -2185,7 +2193,7 @@ static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E,
   //   destruction and the static type of the operand is neither the constructor
   //   or destructor’s class nor one of its bases, the behavior is undefined.
   CGF.EmitTypeCheck(CodeGenFunction::TCK_DynamicOperation, E->getExprLoc(),
-                    ThisPtr, SrcRecordTy);
+                    ThisPtr.getPointer(), SrcRecordTy);
 
   // C++ [expr.typeid]p2:
   //   If the glvalue expression is obtained by applying the unary * operator to
@@ -2201,7 +2209,7 @@ static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E,
         CGF.createBasicBlock("typeid.bad_typeid");
     llvm::BasicBlock *EndBlock = CGF.createBasicBlock("typeid.end");
 
-    llvm::Value *IsNull = CGF.Builder.CreateIsNull(ThisPtr);
+    llvm::Value *IsNull = CGF.Builder.CreateIsNull(ThisPtr.getPointer());
     CGF.Builder.CreateCondBr(IsNull, BadTypeidBlock, EndBlock);
 
     CGF.EmitBlock(BadTypeidBlock);
@@ -2287,7 +2295,8 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr,
   //   construction or destruction and the static type of the operand is not a
   //   pointer to or object of the constructor or destructor’s own class or one
   //   of its bases, the dynamic_cast results in undefined behavior.
-  EmitTypeCheck(TCK_DynamicOperation, DCE->getExprLoc(), ThisAddr, SrcRecordTy);
+  EmitTypeCheck(TCK_DynamicOperation, DCE->getExprLoc(), ThisAddr.getPointer(),
+                SrcRecordTy);
 
   if (DCE->isAlwaysNull()) {
     if (llvm::Value *T = EmitDynamicCastToNull(*this, DestTy)) {
@@ -2322,7 +2331,7 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr,
     CastNull = createBasicBlock("dynamic_cast.null");
     CastNotNull = createBasicBlock("dynamic_cast.notnull");
 
-    llvm::Value *IsNull = Builder.CreateIsNull(ThisAddr);
+    llvm::Value *IsNull = Builder.CreateIsNull(ThisAddr.getPointer());
     Builder.CreateCondBr(IsNull, CastNull, CastNotNull);
     EmitBlock(CastNotNull);
   }
diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index dbefd6d02e7b..75286dceb13a 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -800,8 +800,8 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD,
     // Add a vtable pointer, if we need one and it hasn't already been added.
     if (Layout.hasOwnVFPtr()) {
       llvm::Constant *VTableAddressPoint =
-          CGM.getCXXABI().getVTableAddressPoint(BaseSubobject(CD, Offset),
-                                                VTableClass);
+          CGM.getCXXABI().getVTableAddressPointForConstExpr(
+              BaseSubobject(CD, Offset), VTableClass);
       if (!AppendBytes(Offset, VTableAddressPoint))
         return false;
     }
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 83247aa48f86..8536570087ad 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -2250,7 +2250,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
     // performed and the object is not of the derived type.
     if (CGF.sanitizePerformTypeCheck())
       CGF.EmitTypeCheck(CodeGenFunction::TCK_DowncastPointer, CE->getExprLoc(),
-                        Derived, DestTy->getPointeeType());
+                        Derived.getPointer(), DestTy->getPointeeType());
 
     if (CGF.SanOpts.has(SanitizerKind::CFIDerivedCast))
       CGF.EmitVTablePtrCheckForCast(DestTy->getPointeeType(), Derived,
@@ -2258,14 +2258,13 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
                                     CodeGenFunction::CFITCK_DerivedCast,
                                     CE->getBeginLoc());
 
-    return CGF.getAsNaturalPointerTo(Derived, CE->getType()->getPointeeType());
+    return Derived.getPointer();
   }
   case CK_UncheckedDerivedToBase:
   case CK_DerivedToBase: {
     // The EmitPointerWithAlignment path does this fine; just discard
     // the alignment.
-    return CGF.getAsNaturalPointerTo(CGF.EmitPointerWithAlignment(CE),
-                                     CE->getType()->getPointeeType());
+    return CGF.EmitPointerWithAlignment(CE).getPointer();
   }
 
   case CK_Dynamic: {
@@ -2275,8 +2274,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
   }
 
   case CK_ArrayToPointerDecay:
-    return CGF.getAsNaturalPointerTo(CGF.EmitArrayToPointerDecay(E),
-                                     CE->getType()->getPointeeType());
+    return CGF.EmitArrayToPointerDecay(E).getPointer();
   case CK_FunctionToPointerDecay:
     return EmitLValue(E).getPointer(CGF);
 
@@ -5590,16 +5588,3 @@ CodeGenFunction::EmitCheckedInBoundsGEP(llvm::Type *ElemTy, Value *Ptr,
 
   return GEPVal;
 }
-
-Address CodeGenFunction::EmitCheckedInBoundsGEP(
-    Address Addr, ArrayRef<Value *> IdxList, llvm::Type *elementType,
-    bool SignedIndices, bool IsSubtraction, SourceLocation Loc, CharUnits Align,
-    const Twine &Name) {
-  if (!SanOpts.has(SanitizerKind::PointerOverflow))
-    return Builder.CreateInBoundsGEP(Addr, IdxList, elementType, Align, Name);
-
-  return RawAddress(
-      EmitCheckedInBoundsGEP(Addr.getElementType(), Addr.emitRawPointer(*this),
-                             IdxList, SignedIndices, IsSubtraction, Loc, Name),
-      elementType, Align);
-}
diff --git a/clang/lib/CodeGen/CGNonTrivialStruct.cpp b/clang/lib/CodeGen/CGNonTrivialStruct.cpp
index 8fade0fac21e..75c1d7fbea84 100644
--- a/clang/lib/CodeGen/CGNonTrivialStruct.cpp
+++ b/clang/lib/CodeGen/CGNonTrivialStruct.cpp
@@ -366,7 +366,7 @@ template <class Derived> struct GenFuncBase {
     llvm::Value *SizeInBytes =
         CGF.Builder.CreateNUWMul(BaseEltSizeVal, NumElts);
     llvm::Value *DstArrayEnd = CGF.Builder.CreateInBoundsGEP(
-        CGF.Int8Ty, DstAddr.emitRawPointer(CGF), SizeInBytes);
+        CGF.Int8Ty, DstAddr.getPointer(), SizeInBytes);
     llvm::BasicBlock *PreheaderBB = CGF.Builder.GetInsertBlock();
 
     // Create the header block and insert the phi instructions.
@@ -376,7 +376,7 @@ template <class Derived> struct GenFuncBase {
 
     for (unsigned I = 0; I < N; ++I) {
       PHIs[I] = CGF.Builder.CreatePHI(CGF.CGM.Int8PtrPtrTy, 2, "addr.cur");
-      PHIs[I]->addIncoming(StartAddrs[I].emitRawPointer(CGF), PreheaderBB);
+      PHIs[I]->addIncoming(StartAddrs[I].getPointer(), PreheaderBB);
     }
 
     // Create the exit and loop body blocks.
@@ -410,7 +410,7 @@ template <class Derived> struct GenFuncBase {
       // Instrs to update the destination and source addresses.
       // Update phi instructions.
       NewAddrs[I] = getAddrWithOffset(NewAddrs[I], EltSize);
-      PHIs[I]->addIncoming(NewAddrs[I].emitRawPointer(CGF), LoopBB);
+      PHIs[I]->addIncoming(NewAddrs[I].getPointer(), LoopBB);
     }
 
     // Insert an unconditional branch to the header block.
@@ -488,7 +488,7 @@ template <class Derived> struct GenFuncBase {
 
     for (unsigned I = 0; I < N; ++I) {
       Alignments[I] = Addrs[I].getAlignment();
-      Ptrs[I] = Addrs[I].emitRawPointer(CallerCGF);
+      Ptrs[I] = Addrs[I].getPointer();
     }
 
     if (llvm::Function *F =
diff --git a/clang/lib/CodeGen/CGObjC.cpp b/clang/lib/CodeGen/CGObjC.cpp
index c7f497a7c845..f3a948cf13f9 100644
--- a/clang/lib/CodeGen/CGObjC.cpp
+++ b/clang/lib/CodeGen/CGObjC.cpp
@@ -94,8 +94,8 @@ CodeGenFunction::EmitObjCBoxedExpr(const ObjCBoxedExpr *E) {
     // and cast value to correct type
     Address Temporary = CreateMemTemp(SubExpr->getType());
     EmitAnyExprToMem(SubExpr, Temporary, Qualifiers(), /*isInit*/ true);
-    llvm::Value *BitCast = Builder.CreateBitCast(
-        Temporary.emitRawPointer(*this), ConvertType(ArgQT));
+    llvm::Value *BitCast =
+        Builder.CreateBitCast(Temporary.getPointer(), ConvertType(ArgQT));
     Args.add(RValue::get(BitCast), ArgQT);
 
     // Create char array to store type encoding
@@ -204,11 +204,11 @@ llvm::Value *CodeGenFunction::EmitObjCCollectionLiteral(const Expr *E,
   ObjCMethodDecl::param_const_iterator PI = MethodWithObjects->param_begin();
   const ParmVarDecl *argDecl = *PI++;
   QualType ArgQT = argDecl->getType().getUnqualifiedType();
-  Args.add(RValue::get(Objects, *this), ArgQT);
+  Args.add(RValue::get(Objects.getPointer()), ArgQT);
   if (DLE) {
     argDecl = *PI++;
     ArgQT = argDecl->getType().getUnqualifiedType();
-    Args.add(RValue::get(Keys, *this), ArgQT);
+    Args.add(RValue::get(Keys.getPointer()), ArgQT);
   }
   argDecl = *PI;
   ArgQT = argDecl->getType().getUnqualifiedType();
@@ -827,7 +827,7 @@ static void emitStructGetterCall(CodeGenFunction &CGF, ObjCIvarDecl *ivar,
   //                  sizeof (Type of Ivar), isAtomic, false);
   CallArgList args;
 
-  llvm::Value *dest = CGF.ReturnValue.emitRawPointer(CGF);
+  llvm::Value *dest = CGF.ReturnValue.getPointer();
   args.add(RValue::get(dest), Context.VoidPtrTy);
   args.add(RValue::get(src), Context.VoidPtrTy);
 
@@ -1147,8 +1147,8 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
       callCStructCopyConstructor(Dst, Src);
     } else {
       ObjCIvarDecl *ivar = propImpl->getPropertyIvarDecl();
-      emitCPPObjectAtomicGetterCall(*this, ReturnValue.emitRawPointer(*this),
-                                    ivar, AtomicHelperFn);
+      emitCPPObjectAtomicGetterCall(*this, ReturnValue.getPointer(), ivar,
+                                    AtomicHelperFn);
     }
     return;
   }
@@ -1163,7 +1163,7 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
     }
     else {
       ObjCIvarDecl *ivar = propImpl->getPropertyIvarDecl();
-      emitCPPObjectAtomicGetterCall(*this, ReturnValue.emitRawPointer(*this),
+      emitCPPObjectAtomicGetterCall(*this, ReturnValue.getPointer(),
                                     ivar, AtomicHelperFn);
     }
     return;
@@ -1287,7 +1287,7 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
     case TEK_Scalar: {
       llvm::Value *value;
       if (propType->isReferenceType()) {
-        value = LV.getAddress(*this).emitRawPointer(*this);
+        value = LV.getAddress(*this).getPointer();
       } else {
         // We want to load and autoreleaseReturnValue ARC __weak ivars.
         if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) {
@@ -1821,14 +1821,16 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
   CallArgList Args;
 
   // The first argument is a temporary of the enumeration-state type.
-  Args.add(RValue::get(StatePtr, *this), getContext().getPointerType(StateTy));
+  Args.add(RValue::get(StatePtr.getPointer()),
+           getContext().getPointerType(StateTy));
 
   // The second argument is a temporary array with space for NumItems
   // pointers.  We'll actually be loading elements from the array
   // pointer written into the control state; this buffer is so that
   // collections that *aren't* backed by arrays can still queue up
   // batches of elements.
-  Args.add(RValue::get(ItemsPtr, *this), getContext().getPointerType(ItemsTy));
+  Args.add(RValue::get(ItemsPtr.getPointer()),
+           getContext().getPointerType(ItemsTy));
 
   // The third argument is the capacity of that temporary array.
   llvm::Type *NSUIntegerTy = ConvertType(getContext().getNSUIntegerType());
@@ -2196,7 +2198,7 @@ static llvm::Value *emitARCLoadOperation(CodeGenFunction &CGF, Address addr,
   if (!fn)
     fn = getARCIntrinsic(IntID, CGF.CGM);
 
-  return CGF.EmitNounwindRuntimeCall(fn, addr.emitRawPointer(CGF));
+  return CGF.EmitNounwindRuntimeCall(fn, addr.getPointer());
 }
 
 /// Perform an operation having the following signature:
@@ -2214,8 +2216,9 @@ static llvm::Value *emitARCStoreOperation(CodeGenFunction &CGF, Address addr,
   llvm::Type *origType = value->getType();
 
   llvm::Value *args[] = {
-      CGF.Builder.CreateBitCast(addr.emitRawPointer(CGF), CGF.Int8PtrPtrTy),
-      CGF.Builder.CreateBitCast(value, CGF.Int8PtrTy)};
+    CGF.Builder.CreateBitCast(addr.getPointer(), CGF.Int8PtrPtrTy),
+    CGF.Builder.CreateBitCast(value, CGF.Int8PtrTy)
+  };
   llvm::CallInst *result = CGF.EmitNounwindRuntimeCall(fn, args);
 
   if (ignored) return nullptr;
@@ -2234,8 +2237,9 @@ static void emitARCCopyOperation(CodeGenFunction &CGF, Address dst, Address src,
     fn = getARCIntrinsic(IntID, CGF.CGM);
 
   llvm::Value *args[] = {
-      CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF), CGF.Int8PtrPtrTy),
-      CGF.Builder.CreateBitCast(src.emitRawPointer(CGF), CGF.Int8PtrPtrTy)};
+    CGF.Builder.CreateBitCast(dst.getPointer(), CGF.Int8PtrPtrTy),
+    CGF.Builder.CreateBitCast(src.getPointer(), CGF.Int8PtrPtrTy)
+  };
   CGF.EmitNounwindRuntimeCall(fn, args);
 }
 
@@ -2486,8 +2490,9 @@ llvm::Value *CodeGenFunction::EmitARCStoreStrongCall(Address addr,
     fn = getARCIntrinsic(llvm::Intrinsic::objc_storeStrong, CGM);
 
   llvm::Value *args[] = {
-      Builder.CreateBitCast(addr.emitRawPointer(*this), Int8PtrPtrTy),
-      Builder.CreateBitCast(value, Int8PtrTy)};
+    Builder.CreateBitCast(addr.getPointer(), Int8PtrPtrTy),
+    Builder.CreateBitCast(value, Int8PtrTy)
+  };
   EmitNounwindRuntimeCall(fn, args);
 
   if (ignored) return nullptr;
@@ -2638,7 +2643,7 @@ void CodeGenFunction::EmitARCDestroyWeak(Address addr) {
   if (!fn)
     fn = getARCIntrinsic(llvm::Intrinsic::objc_destroyWeak, CGM);
 
-  EmitNounwindRuntimeCall(fn, addr.emitRawPointer(*this));
+  EmitNounwindRuntimeCall(fn, addr.getPointer());
 }
 
 /// void \@objc_moveWeak(i8** %dest, i8** %src)
diff --git a/clang/lib/CodeGen/CGObjCGNU.cpp b/clang/lib/CodeGen/CGObjCGNU.cpp
index 4e7f777ba1d9..a36b0cdddaf0 100644
--- a/clang/lib/CodeGen/CGObjCGNU.cpp
+++ b/clang/lib/CodeGen/CGObjCGNU.cpp
@@ -706,8 +706,7 @@ protected:
                               llvm::Value *cmd, MessageSendInfo &MSI) override {
     CGBuilderTy &Builder = CGF.Builder;
     llvm::Value *lookupArgs[] = {
-        EnforceType(Builder, ObjCSuper.emitRawPointer(CGF), PtrToObjCSuperTy),
-        cmd};
+        EnforceType(Builder, ObjCSuper.getPointer(), PtrToObjCSuperTy), cmd};
     return CGF.EmitNounwindRuntimeCall(MsgLookupSuperFn, lookupArgs);
   }
 
@@ -762,8 +761,8 @@ class CGObjCGNUstep : public CGObjCGNU {
       llvm::FunctionCallee LookupFn = SlotLookupFn;
 
       // Store the receiver on the stack so that we can reload it later
-      RawAddress ReceiverPtr =
-          CGF.CreateTempAlloca(Receiver->getType(), CGF.getPointerAlign());
+      Address ReceiverPtr =
+        CGF.CreateTempAlloca(Receiver->getType(), CGF.getPointerAlign());
       Builder.CreateStore(Receiver, ReceiverPtr);
 
       llvm::Value *self;
@@ -779,9 +778,9 @@ class CGObjCGNUstep : public CGObjCGNU {
         LookupFn2->addParamAttr(0, llvm::Attribute::NoCapture);
 
       llvm::Value *args[] = {
-          EnforceType(Builder, ReceiverPtr.getPointer(), PtrToIdTy),
-          EnforceType(Builder, cmd, SelectorTy),
-          EnforceType(Builder, self, IdTy)};
+              EnforceType(Builder, ReceiverPtr.getPointer(), PtrToIdTy),
+              EnforceType(Builder, cmd, SelectorTy),
+              EnforceType(Builder, self, IdTy) };
       llvm::CallBase *slot = CGF.EmitRuntimeCallOrInvoke(LookupFn, args);
       slot->setOnlyReadsMemory();
       slot->setMetadata(msgSendMDKind, node);
@@ -801,7 +800,7 @@ class CGObjCGNUstep : public CGObjCGNU {
                                 llvm::Value *cmd,
                                 MessageSendInfo &MSI) override {
       CGBuilderTy &Builder = CGF.Builder;
-      llvm::Value *lookupArgs[] = {ObjCSuper.emitRawPointer(CGF), cmd};
+      llvm::Value *lookupArgs[] = {ObjCSuper.getPointer(), cmd};
 
       llvm::CallInst *slot =
         CGF.EmitNounwindRuntimeCall(SlotLookupSuperFn, lookupArgs);
@@ -1222,10 +1221,10 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
                               llvm::Value *cmd, MessageSendInfo &MSI) override {
     // Don't access the slot unless we're trying to cache the result.
     CGBuilderTy &Builder = CGF.Builder;
-    llvm::Value *lookupArgs[] = {
-        CGObjCGNU::EnforceType(Builder, ObjCSuper.emitRawPointer(CGF),
-                               PtrToObjCSuperTy),
-        cmd};
+    llvm::Value *lookupArgs[] = {CGObjCGNU::EnforceType(Builder,
+                                                        ObjCSuper.getPointer(),
+                                                        PtrToObjCSuperTy),
+                                 cmd};
     return CGF.EmitNounwindRuntimeCall(MsgLookupSuperFn, lookupArgs);
   }
 
@@ -2187,8 +2186,7 @@ protected:
                               llvm::Value *cmd, MessageSendInfo &MSI) override {
     CGBuilderTy &Builder = CGF.Builder;
     llvm::Value *lookupArgs[] = {
-        EnforceType(Builder, ObjCSuper.emitRawPointer(CGF), PtrToObjCSuperTy),
-        cmd,
+        EnforceType(Builder, ObjCSuper.getPointer(), PtrToObjCSuperTy), cmd,
     };
 
     if (CGM.ReturnTypeUsesSRet(MSI.CallInfo))
@@ -4203,15 +4201,15 @@ void CGObjCGNU::EmitThrowStmt(CodeGenFunction &CGF,
 llvm::Value * CGObjCGNU::EmitObjCWeakRead(CodeGenFunction &CGF,
                                           Address AddrWeakObj) {
   CGBuilderTy &B = CGF.Builder;
-  return B.CreateCall(
-      WeakReadFn, EnforceType(B, AddrWeakObj.emitRawPointer(CGF), PtrToIdTy));
+  return B.CreateCall(WeakReadFn,
+                      EnforceType(B, AddrWeakObj.getPointer(), PtrToIdTy));
 }
 
 void CGObjCGNU::EmitObjCWeakAssign(CodeGenFunction &CGF,
                                    llvm::Value *src, Address dst) {
   CGBuilderTy &B = CGF.Builder;
   src = EnforceType(B, src, IdTy);
-  llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), PtrToIdTy);
+  llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy);
   B.CreateCall(WeakAssignFn, {src, dstVal});
 }
 
@@ -4220,7 +4218,7 @@ void CGObjCGNU::EmitObjCGlobalAssign(CodeGenFunction &CGF,
                                      bool threadlocal) {
   CGBuilderTy &B = CGF.Builder;
   src = EnforceType(B, src, IdTy);
-  llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), PtrToIdTy);
+  llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy);
   // FIXME. Add threadloca assign API
   assert(!threadlocal && "EmitObjCGlobalAssign - Threal Local API NYI");
   B.CreateCall(GlobalAssignFn, {src, dstVal});
@@ -4231,7 +4229,7 @@ void CGObjCGNU::EmitObjCIvarAssign(CodeGenFunction &CGF,
                                    llvm::Value *ivarOffset) {
   CGBuilderTy &B = CGF.Builder;
   src = EnforceType(B, src, IdTy);
-  llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), IdTy);
+  llvm::Value *dstVal = EnforceType(B, dst.getPointer(), IdTy);
   B.CreateCall(IvarAssignFn, {src, dstVal, ivarOffset});
 }
 
@@ -4239,7 +4237,7 @@ void CGObjCGNU::EmitObjCStrongCastAssign(CodeGenFunction &CGF,
                                          llvm::Value *src, Address dst) {
   CGBuilderTy &B = CGF.Builder;
   src = EnforceType(B, src, IdTy);
-  llvm::Value *dstVal = EnforceType(B, dst.emitRawPointer(CGF), PtrToIdTy);
+  llvm::Value *dstVal = EnforceType(B, dst.getPointer(), PtrToIdTy);
   B.CreateCall(StrongCastAssignFn, {src, dstVal});
 }
 
@@ -4248,8 +4246,8 @@ void CGObjCGNU::EmitGCMemmoveCollectable(CodeGenFunction &CGF,
                                          Address SrcPtr,
                                          llvm::Value *Size) {
   CGBuilderTy &B = CGF.Builder;
-  llvm::Value *DestPtrVal = EnforceType(B, DestPtr.emitRawPointer(CGF), PtrTy);
-  llvm::Value *SrcPtrVal = EnforceType(B, SrcPtr.emitRawPointer(CGF), PtrTy);
+  llvm::Value *DestPtrVal = EnforceType(B, DestPtr.getPointer(), PtrTy);
+  llvm::Value *SrcPtrVal = EnforceType(B, SrcPtr.getPointer(), PtrTy);
 
   B.CreateCall(MemMoveFn, {DestPtrVal, SrcPtrVal, Size});
 }
diff --git a/clang/lib/CodeGen/CGObjCMac.cpp b/clang/lib/CodeGen/CGObjCMac.cpp
index 5d0e27b4c15e..e815e097e1fb 100644
--- a/clang/lib/CodeGen/CGObjCMac.cpp
+++ b/clang/lib/CodeGen/CGObjCMac.cpp
@@ -1310,7 +1310,7 @@ private:
   /// EmitSelector - Return a Value*, of type ObjCTypes.SelectorPtrTy,
   /// for the given selector.
   llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel);
-  ConstantAddress EmitSelectorAddr(Selector Sel);
+  Address EmitSelectorAddr(Selector Sel);
 
 public:
   CGObjCMac(CodeGen::CodeGenModule &cgm);
@@ -1538,7 +1538,7 @@ private:
   /// EmitSelector - Return a Value*, of type ObjCTypes.SelectorPtrTy,
   /// for the given selector.
   llvm::Value *EmitSelector(CodeGenFunction &CGF, Selector Sel);
-  ConstantAddress EmitSelectorAddr(Selector Sel);
+  Address EmitSelectorAddr(Selector Sel);
 
   /// GetInterfaceEHType - Get the cached ehtype for the given Objective-C
   /// interface. The return value has type EHTypePtrTy.
@@ -2064,8 +2064,9 @@ CGObjCMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF,
                                     const ObjCMethodDecl *Method) {
   // Create and init a super structure; this is a (receiver, class)
   // pair we will pass to objc_msgSendSuper.
-  RawAddress ObjCSuper = CGF.CreateTempAlloca(
-      ObjCTypes.SuperTy, CGF.getPointerAlign(), "objc_super");
+  Address ObjCSuper =
+    CGF.CreateTempAlloca(ObjCTypes.SuperTy, CGF.getPointerAlign(),
+                         "objc_super");
   llvm::Value *ReceiverAsObject =
     CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy);
   CGF.Builder.CreateStore(ReceiverAsObject,
@@ -4258,7 +4259,7 @@ namespace {
 
       CGF.EmitBlock(FinallyCallExit);
       CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryExitFn(),
-                                  ExceptionData.emitRawPointer(CGF));
+                                  ExceptionData.getPointer());
 
       CGF.EmitBlock(FinallyNoCallExit);
 
@@ -4424,9 +4425,7 @@ void FragileHazards::emitHazardsInNewBlocks() {
 }
 
 static void addIfPresent(llvm::DenseSet<llvm::Value*> &S, Address V) {
-  if (V.isValid())
-    if (llvm::Value *Ptr = V.getBasePointer())
-      S.insert(Ptr);
+  if (V.isValid()) S.insert(V.getPointer());
 }
 
 void FragileHazards::collectLocals() {
@@ -4629,13 +4628,13 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
   //  - Call objc_exception_try_enter to push ExceptionData on top of
   //    the EH stack.
   CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(),
-                              ExceptionData.emitRawPointer(CGF));
+                              ExceptionData.getPointer());
 
   //  - Call setjmp on the exception data buffer.
   llvm::Constant *Zero = llvm::ConstantInt::get(CGF.Builder.getInt32Ty(), 0);
   llvm::Value *GEPIndexes[] = { Zero, Zero, Zero };
   llvm::Value *SetJmpBuffer = CGF.Builder.CreateGEP(
-      ObjCTypes.ExceptionDataTy, ExceptionData.emitRawPointer(CGF), GEPIndexes,
+      ObjCTypes.ExceptionDataTy, ExceptionData.getPointer(), GEPIndexes,
       "setjmp_buffer");
   llvm::CallInst *SetJmpResult = CGF.EmitNounwindRuntimeCall(
       ObjCTypes.getSetJmpFn(), SetJmpBuffer, "setjmp_result");
@@ -4674,9 +4673,9 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
   } else {
     // Retrieve the exception object.  We may emit multiple blocks but
     // nothing can cross this so the value is already in SSA form.
-    llvm::CallInst *Caught = CGF.EmitNounwindRuntimeCall(
-        ObjCTypes.getExceptionExtractFn(), ExceptionData.emitRawPointer(CGF),
-        "caught");
+    llvm::CallInst *Caught =
+      CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(),
+                                  ExceptionData.getPointer(), "caught");
 
     // Push the exception to rethrow onto the EH value stack for the
     // benefit of any @throws in the handlers.
@@ -4699,7 +4698,7 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
       // Enter a new exception try block (in case a @catch block
       // throws an exception).
       CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionTryEnterFn(),
-                                  ExceptionData.emitRawPointer(CGF));
+                                  ExceptionData.getPointer());
 
       llvm::CallInst *SetJmpResult =
         CGF.EmitNounwindRuntimeCall(ObjCTypes.getSetJmpFn(),
@@ -4830,9 +4829,9 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
       // Extract the new exception and save it to the
       // propagating-exception slot.
       assert(PropagatingExnVar.isValid());
-      llvm::CallInst *NewCaught = CGF.EmitNounwindRuntimeCall(
-          ObjCTypes.getExceptionExtractFn(), ExceptionData.emitRawPointer(CGF),
-          "caught");
+      llvm::CallInst *NewCaught =
+        CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(),
+                                    ExceptionData.getPointer(), "caught");
       CGF.Builder.CreateStore(NewCaught, PropagatingExnVar);
 
       // Don't pop the catch handler; the throw already did.
@@ -4862,8 +4861,9 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
 
     // Otherwise, just look in the buffer for the exception to throw.
     } else {
-      llvm::CallInst *Caught = CGF.EmitNounwindRuntimeCall(
-          ObjCTypes.getExceptionExtractFn(), ExceptionData.emitRawPointer(CGF));
+      llvm::CallInst *Caught =
+        CGF.EmitNounwindRuntimeCall(ObjCTypes.getExceptionExtractFn(),
+                                    ExceptionData.getPointer());
       PropagatingExn = Caught;
     }
 
@@ -4906,7 +4906,7 @@ llvm::Value * CGObjCMac::EmitObjCWeakRead(CodeGen::CodeGenFunction &CGF,
                                           Address AddrWeakObj) {
   llvm::Type* DestTy = AddrWeakObj.getElementType();
   llvm::Value *AddrWeakObjVal = CGF.Builder.CreateBitCast(
-      AddrWeakObj.emitRawPointer(CGF), ObjCTypes.PtrObjectPtrTy);
+      AddrWeakObj.getPointer(), ObjCTypes.PtrObjectPtrTy);
   llvm::Value *read_weak =
     CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcReadWeakFn(),
                                 AddrWeakObjVal, "weakread");
@@ -4928,8 +4928,8 @@ void CGObjCMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
     src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
   }
   src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
-  llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF),
-                                                  ObjCTypes.PtrObjectPtrTy);
+  llvm::Value *dstVal =
+      CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
   llvm::Value *args[] = { src, dstVal };
   CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignWeakFn(),
                               args, "weakassign");
@@ -4950,8 +4950,8 @@ void CGObjCMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
     src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
   }
   src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
-  llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF),
-                                                  ObjCTypes.PtrObjectPtrTy);
+  llvm::Value *dstVal =
+      CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
   llvm::Value *args[] = {src, dstVal};
   if (!threadlocal)
     CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignGlobalFn(),
@@ -4977,8 +4977,8 @@ void CGObjCMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
     src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
   }
   src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
-  llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF),
-                                                  ObjCTypes.PtrObjectPtrTy);
+  llvm::Value *dstVal =
+      CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
   llvm::Value *args[] = {src, dstVal, ivarOffset};
   CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignIvarFn(), args);
 }
@@ -4997,8 +4997,8 @@ void CGObjCMac::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF,
     src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
   }
   src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
-  llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF),
-                                                  ObjCTypes.PtrObjectPtrTy);
+  llvm::Value *dstVal =
+      CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
   llvm::Value *args[] = {src, dstVal};
   CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignStrongCastFn(),
                               args, "strongassign");
@@ -5007,8 +5007,7 @@ void CGObjCMac::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF,
 void CGObjCMac::EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF,
                                          Address DestPtr, Address SrcPtr,
                                          llvm::Value *size) {
-  llvm::Value *args[] = {DestPtr.emitRawPointer(CGF),
-                         SrcPtr.emitRawPointer(CGF), size};
+  llvm::Value *args[] = { DestPtr.getPointer(), SrcPtr.getPointer(), size };
   CGF.EmitNounwindRuntimeCall(ObjCTypes.GcMemmoveCollectableFn(), args);
 }
 
@@ -5244,7 +5243,7 @@ llvm::Value *CGObjCMac::EmitSelector(CodeGenFunction &CGF, Selector Sel) {
   return CGF.Builder.CreateLoad(EmitSelectorAddr(Sel));
 }
 
-ConstantAddress CGObjCMac::EmitSelectorAddr(Selector Sel) {
+Address CGObjCMac::EmitSelectorAddr(Selector Sel) {
   CharUnits Align = CGM.getPointerAlign();
 
   llvm::GlobalVariable *&Entry = SelectorReferences[Sel];
@@ -5255,7 +5254,7 @@ ConstantAddress CGObjCMac::EmitSelectorAddr(Selector Sel) {
     Entry->setExternallyInitialized(true);
   }
 
-  return ConstantAddress(Entry, ObjCTypes.SelectorPtrTy, Align);
+  return Address(Entry, ObjCTypes.SelectorPtrTy, Align);
 }
 
 llvm::Constant *CGObjCCommonMac::GetClassName(StringRef RuntimeName) {
@@ -7324,7 +7323,7 @@ CGObjCNonFragileABIMac::EmitVTableMessageSend(CodeGenFunction &CGF,
             ObjCTypes.MessageRefTy, CGF.getPointerAlign());
 
   // Update the message ref argument.
-  args[1].setRValue(RValue::get(mref, CGF));
+  args[1].setRValue(RValue::get(mref.getPointer()));
 
   // Load the function to call from the message ref table.
   Address calleeAddr = CGF.Builder.CreateStructGEP(mref, 0);
@@ -7553,8 +7552,9 @@ CGObjCNonFragileABIMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF,
   // ...
   // Create and init a super structure; this is a (receiver, class)
   // pair we will pass to objc_msgSendSuper.
-  RawAddress ObjCSuper = CGF.CreateTempAlloca(
-      ObjCTypes.SuperTy, CGF.getPointerAlign(), "objc_super");
+  Address ObjCSuper =
+    CGF.CreateTempAlloca(ObjCTypes.SuperTy, CGF.getPointerAlign(),
+                         "objc_super");
 
   llvm::Value *ReceiverAsObject =
     CGF.Builder.CreateBitCast(Receiver, ObjCTypes.ObjectPtrTy);
@@ -7594,7 +7594,7 @@ llvm::Value *CGObjCNonFragileABIMac::EmitSelector(CodeGenFunction &CGF,
   return LI;
 }
 
-ConstantAddress CGObjCNonFragileABIMac::EmitSelectorAddr(Selector Sel) {
+Address CGObjCNonFragileABIMac::EmitSelectorAddr(Selector Sel) {
   llvm::GlobalVariable *&Entry = SelectorReferences[Sel];
   CharUnits Align = CGM.getPointerAlign();
   if (!Entry) {
@@ -7610,7 +7610,7 @@ ConstantAddress CGObjCNonFragileABIMac::EmitSelectorAddr(Selector Sel) {
     CGM.addCompilerUsedGlobal(Entry);
   }
 
-  return ConstantAddress(Entry, ObjCTypes.SelectorPtrTy, Align);
+  return Address(Entry, ObjCTypes.SelectorPtrTy, Align);
 }
 
 /// EmitObjCIvarAssign - Code gen for assigning to a __strong object.
@@ -7629,8 +7629,8 @@ void CGObjCNonFragileABIMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
     src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
   }
   src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
-  llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF),
-                                                  ObjCTypes.PtrObjectPtrTy);
+  llvm::Value *dstVal =
+      CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
   llvm::Value *args[] = {src, dstVal, ivarOffset};
   CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignIvarFn(), args);
 }
@@ -7650,8 +7650,8 @@ void CGObjCNonFragileABIMac::EmitObjCStrongCastAssign(
     src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
   }
   src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
-  llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF),
-                                                  ObjCTypes.PtrObjectPtrTy);
+  llvm::Value *dstVal =
+      CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
   llvm::Value *args[] = {src, dstVal};
   CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignStrongCastFn(),
                               args, "weakassign");
@@ -7660,8 +7660,7 @@ void CGObjCNonFragileABIMac::EmitObjCStrongCastAssign(
 void CGObjCNonFragileABIMac::EmitGCMemmoveCollectable(
     CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr,
     llvm::Value *Size) {
-  llvm::Value *args[] = {DestPtr.emitRawPointer(CGF),
-                         SrcPtr.emitRawPointer(CGF), Size};
+  llvm::Value *args[] = { DestPtr.getPointer(), SrcPtr.getPointer(), Size };
   CGF.EmitNounwindRuntimeCall(ObjCTypes.GcMemmoveCollectableFn(), args);
 }
 
@@ -7673,7 +7672,7 @@ llvm::Value * CGObjCNonFragileABIMac::EmitObjCWeakRead(
   Address AddrWeakObj) {
   llvm::Type *DestTy = AddrWeakObj.getElementType();
   llvm::Value *AddrWeakObjVal = CGF.Builder.CreateBitCast(
-      AddrWeakObj.emitRawPointer(CGF), ObjCTypes.PtrObjectPtrTy);
+      AddrWeakObj.getPointer(), ObjCTypes.PtrObjectPtrTy);
   llvm::Value *read_weak =
     CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcReadWeakFn(),
                                 AddrWeakObjVal, "weakread");
@@ -7695,8 +7694,8 @@ void CGObjCNonFragileABIMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
     src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
   }
   src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
-  llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF),
-                                                  ObjCTypes.PtrObjectPtrTy);
+  llvm::Value *dstVal =
+      CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
   llvm::Value *args[] = {src, dstVal};
   CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignWeakFn(),
                               args, "weakassign");
@@ -7717,8 +7716,8 @@ void CGObjCNonFragileABIMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
     src = CGF.Builder.CreateIntToPtr(src, ObjCTypes.Int8PtrTy);
   }
   src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
-  llvm::Value *dstVal = CGF.Builder.CreateBitCast(dst.emitRawPointer(CGF),
-                                                  ObjCTypes.PtrObjectPtrTy);
+  llvm::Value *dstVal =
+      CGF.Builder.CreateBitCast(dst.getPointer(), ObjCTypes.PtrObjectPtrTy);
   llvm::Value *args[] = {src, dstVal};
   if (!threadlocal)
     CGF.EmitNounwindRuntimeCall(ObjCTypes.getGcAssignGlobalFn(),
diff --git a/clang/lib/CodeGen/CGObjCRuntime.cpp b/clang/lib/CodeGen/CGObjCRuntime.cpp
index 01d0f35da196..424564f97599 100644
--- a/clang/lib/CodeGen/CGObjCRuntime.cpp
+++ b/clang/lib/CodeGen/CGObjCRuntime.cpp
@@ -67,7 +67,7 @@ LValue CGObjCRuntime::EmitValueForIvarAtOffset(CodeGen::CodeGenFunction &CGF,
   V = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, V, Offset, "add.ptr");
 
   if (!Ivar->isBitField()) {
-    LValue LV = CGF.MakeNaturalAlignRawAddrLValue(V, IvarTy);
+    LValue LV = CGF.MakeNaturalAlignAddrLValue(V, IvarTy);
     return LV;
   }
 
@@ -233,7 +233,7 @@ void CGObjCRuntime::EmitTryCatchStmt(CodeGenFunction &CGF,
       llvm::Instruction *CPICandidate = Handler.Block->getFirstNonPHI();
       if (auto *CPI = dyn_cast_or_null<llvm::CatchPadInst>(CPICandidate)) {
         CGF.CurrentFuncletPad = CPI;
-        CPI->setOperand(2, CGF.getExceptionSlot().emitRawPointer(CGF));
+        CPI->setOperand(2, CGF.getExceptionSlot().getPointer());
         CGF.EHStack.pushCleanup<CatchRetScope>(NormalCleanup, CPI);
       }
     }
@@ -405,7 +405,7 @@ bool CGObjCRuntime::canMessageReceiverBeNull(CodeGenFunction &CGF,
     auto self = curMethod->getSelfDecl();
     if (self->getType().isConstQualified()) {
       if (auto LI = dyn_cast<llvm::LoadInst>(receiver->stripPointerCasts())) {
-        llvm::Value *selfAddr = CGF.GetAddrOfLocalVar(self).emitRawPointer(CGF);
+        llvm::Value *selfAddr = CGF.GetAddrOfLocalVar(self).getPointer();
         if (selfAddr == LI->getPointerOperand()) {
           return false;
         }
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 934d1e722d0d..e8a68dbcc687 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -622,7 +622,7 @@ static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
     auto *GV = new llvm::GlobalVariable(
         CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
         llvm::GlobalValue::PrivateLinkage, Init, Name);
-    LValue LV = CGF.MakeNaturalAlignRawAddrLValue(GV, Ty);
+    LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
     RValue InitRVal;
     switch (CGF.getEvaluationKind(Ty)) {
     case TEK_Scalar:
@@ -668,8 +668,8 @@ static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
 
   llvm::Value *SrcBegin = nullptr;
   if (DRD)
-    SrcBegin = SrcAddr.emitRawPointer(CGF);
-  llvm::Value *DestBegin = DestAddr.emitRawPointer(CGF);
+    SrcBegin = SrcAddr.getPointer();
+  llvm::Value *DestBegin = DestAddr.getPointer();
   // Cast from pointer to array type to pointer to single element.
   llvm::Value *DestEnd =
       CGF.Builder.CreateGEP(DestAddr.getElementType(), DestBegin, NumElements);
@@ -912,7 +912,7 @@ static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
 
 static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
                           Address OriginalBaseAddress, llvm::Value *Addr) {
-  RawAddress Tmp = RawAddress::invalid();
+  Address Tmp = Address::invalid();
   Address TopTmp = Address::invalid();
   Address MostTopTmp = Address::invalid();
   BaseTy = BaseTy.getNonReferenceType();
@@ -971,10 +971,10 @@ Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
     Address SharedAddr = SharedAddresses[N].first.getAddress(CGF);
     llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
         SharedAddr.getElementType(), BaseLValue.getPointer(CGF),
-        SharedAddr.emitRawPointer(CGF));
+        SharedAddr.getPointer());
     llvm::Value *PrivatePointer =
         CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
-            PrivateAddr.emitRawPointer(CGF), SharedAddr.getType());
+            PrivateAddr.getPointer(), SharedAddr.getType());
     llvm::Value *Ptr = CGF.Builder.CreateGEP(
         SharedAddr.getElementType(), PrivatePointer, Adjustment);
     return castToBase(CGF, OrigVD->getType(),
@@ -1557,7 +1557,7 @@ static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc(
   return OMPBuilder.getTargetEntryUniqueInfo(FileInfoCallBack, ParentName);
 }
 
-ConstantAddress CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
+Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
   auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(VD); };
 
   auto LinkageForVariable = [&VD, this]() {
@@ -1579,8 +1579,8 @@ ConstantAddress CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
       LinkageForVariable);
 
   if (!addr)
-    return ConstantAddress::invalid();
-  return ConstantAddress(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(VD));
+    return Address::invalid();
+  return Address(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(VD));
 }
 
 llvm::Constant *
@@ -1604,7 +1604,7 @@ Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
   llvm::Type *VarTy = VDAddr.getElementType();
   llvm::Value *Args[] = {
       emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
-      CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy),
+      CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy),
       CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
       getOrCreateThreadPrivateCache(VD)};
   return Address(
@@ -1627,8 +1627,7 @@ void CGOpenMPRuntime::emitThreadPrivateVarInit(
   // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
   // to register constructor/destructor for variable.
   llvm::Value *Args[] = {
-      OMPLoc,
-      CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.VoidPtrTy),
+      OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy),
       Ctor, CopyCtor, Dtor};
   CGF.EmitRuntimeCall(
       OMPBuilder.getOrCreateRuntimeFunction(
@@ -1901,13 +1900,13 @@ void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
 
     // OutlinedFn(&GTid, &zero_bound, CapturedStruct);
     Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
-    RawAddress ZeroAddrBound =
+    Address ZeroAddrBound =
         CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
                                          /*Name=*/".bound.zero.addr");
     CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddrBound);
     llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
     // ThreadId for serialized parallels is 0.
-    OutlinedFnArgs.push_back(ThreadIDAddr.emitRawPointer(CGF));
+    OutlinedFnArgs.push_back(ThreadIDAddr.getPointer());
     OutlinedFnArgs.push_back(ZeroAddrBound.getPointer());
     OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
 
@@ -2273,7 +2272,7 @@ void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
         emitUpdateLocation(CGF, Loc), // ident_t *<loc>
         getThreadID(CGF, Loc),        // i32 <gtid>
         BufSize,                      // size_t <buf_size>
-        CL.emitRawPointer(CGF),       // void *<copyprivate list>
+        CL.getPointer(),              // void *<copyprivate list>
         CpyFn,                        // void (*) (void *, void *) <copy_func>
         DidItVal                      // i32 did_it
     };
@@ -2592,10 +2591,10 @@ static void emitForStaticInitCall(
       ThreadId,
       CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1,
                                                   M2)), // Schedule type
-      Values.IL.emitRawPointer(CGF),                    // &isLastIter
-      Values.LB.emitRawPointer(CGF),                    // &LB
-      Values.UB.emitRawPointer(CGF),                    // &UB
-      Values.ST.emitRawPointer(CGF),                    // &Stride
+      Values.IL.getPointer(),                           // &isLastIter
+      Values.LB.getPointer(),                           // &LB
+      Values.UB.getPointer(),                           // &UB
+      Values.ST.getPointer(),                           // &Stride
       CGF.Builder.getIntN(Values.IVSize, 1),            // Incr
       Chunk                                             // Chunk
   };
@@ -2698,11 +2697,12 @@ llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
   //          kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
   //          kmp_int[32|64] *p_stride);
   llvm::Value *Args[] = {
-      emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
-      IL.emitRawPointer(CGF), // &isLastIter
-      LB.emitRawPointer(CGF), // &Lower
-      UB.emitRawPointer(CGF), // &Upper
-      ST.emitRawPointer(CGF)  // &Stride
+      emitUpdateLocation(CGF, Loc),
+      getThreadID(CGF, Loc),
+      IL.getPointer(), // &isLastIter
+      LB.getPointer(), // &Lower
+      UB.getPointer(), // &Upper
+      ST.getPointer()  // &Stride
   };
   llvm::Value *Call = CGF.EmitRuntimeCall(
       OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), Args);
@@ -3047,7 +3047,7 @@ emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
       CGF.Builder
           .CreatePointerBitCastOrAddrSpaceCast(TDBase.getAddress(CGF),
                                                CGF.VoidPtrTy, CGF.Int8Ty)
-          .emitRawPointer(CGF)};
+          .getPointer()};
   SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs),
                                           std::end(CommonArgs));
   if (isOpenMPTaskLoopDirective(Kind)) {
@@ -3574,8 +3574,7 @@ getPointerAndSize(CodeGenFunction &CGF, const Expr *E) {
         CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false);
     Address UpAddrAddress = UpAddrLVal.getAddress(CGF);
     llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32(
-        UpAddrAddress.getElementType(), UpAddrAddress.emitRawPointer(CGF),
-        /*Idx0=*/1);
+        UpAddrAddress.getElementType(), UpAddrAddress.getPointer(), /*Idx0=*/1);
     llvm::Value *LowIntPtr = CGF.Builder.CreatePtrToInt(Addr, CGF.SizeTy);
     llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGF.SizeTy);
     SizeVal = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
@@ -3889,9 +3888,8 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
         llvm::Value *Size;
         std::tie(Addr, Size) = getPointerAndSize(CGF, E);
         llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
-        LValue Base =
-            CGF.MakeAddrLValue(CGF.Builder.CreateGEP(CGF, AffinitiesArray, Idx),
-                               KmpTaskAffinityInfoTy);
+        LValue Base = CGF.MakeAddrLValue(
+            CGF.Builder.CreateGEP(AffinitiesArray, Idx), KmpTaskAffinityInfoTy);
         // affs[i].base_addr = &<Affinities[i].second>;
         LValue BaseAddrLVal = CGF.EmitLValueForField(
             Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));
@@ -3912,7 +3910,7 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
     llvm::Value *LocRef = emitUpdateLocation(CGF, Loc);
     llvm::Value *GTid = getThreadID(CGF, Loc);
     llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
-        AffinitiesArray.emitRawPointer(CGF), CGM.VoidPtrTy);
+        AffinitiesArray.getPointer(), CGM.VoidPtrTy);
     // FIXME: Emit the function and ignore its result for now unless the
     // runtime function is properly implemented.
     (void)CGF.EmitRuntimeCall(
@@ -3923,8 +3921,8 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
   llvm::Value *NewTaskNewTaskTTy =
       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
           NewTask, KmpTaskTWithPrivatesPtrTy);
-  LValue Base = CGF.MakeNaturalAlignRawAddrLValue(NewTaskNewTaskTTy,
-                                                  KmpTaskTWithPrivatesQTy);
+  LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
+                                               KmpTaskTWithPrivatesQTy);
   LValue TDBase =
       CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
   // Fill the data in the resulting kmp_task_t record.
@@ -4049,7 +4047,7 @@ CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal,
           CGF.ConvertTypeForMem(KmpDependInfoPtrTy)),
       KmpDependInfoPtrTy->castAs<PointerType>());
   Address DepObjAddr = CGF.Builder.CreateGEP(
-      CGF, Base.getAddress(CGF),
+      Base.getAddress(CGF),
       llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
   LValue NumDepsBase = CGF.MakeAddrLValue(
       DepObjAddr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo());
@@ -4099,7 +4097,7 @@ static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
       LValue &PosLVal = *Pos.get<LValue *>();
       llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
       Base = CGF.MakeAddrLValue(
-          CGF.Builder.CreateGEP(CGF, DependenciesArray, Idx), KmpDependInfoTy);
+          CGF.Builder.CreateGEP(DependenciesArray, Idx), KmpDependInfoTy);
     }
     // deps[i].base_addr = &<Dependencies[i].second>;
     LValue BaseAddrLVal = CGF.EmitLValueForField(
@@ -4197,7 +4195,7 @@ void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF,
           ElSize,
           CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false));
       llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc());
-      Address DepAddr = CGF.Builder.CreateGEP(CGF, DependenciesArray, Pos);
+      Address DepAddr = CGF.Builder.CreateGEP(DependenciesArray, Pos);
       CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(CGF), Size);
 
       // Increase pos.
@@ -4432,7 +4430,7 @@ void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
       Base.getAddress(CGF), CGF.ConvertTypeForMem(KmpDependInfoPtrTy),
       CGF.ConvertTypeForMem(KmpDependInfoTy));
   llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
-      Addr.getElementType(), Addr.emitRawPointer(CGF),
+      Addr.getElementType(), Addr.getPointer(),
       llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true));
   DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr,
                                                                CGF.VoidPtrTy);
@@ -4462,8 +4460,8 @@ void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
 
   Address Begin = Base.getAddress(CGF);
   // Cast from pointer to array type to pointer to single element.
-  llvm::Value *End = CGF.Builder.CreateGEP(Begin.getElementType(),
-                                           Begin.emitRawPointer(CGF), NumDeps);
+  llvm::Value *End = CGF.Builder.CreateGEP(
+      Begin.getElementType(), Begin.getPointer(), NumDeps);
   // The basic structure here is a while-do loop.
   llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body");
   llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done");
@@ -4471,7 +4469,7 @@ void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
   CGF.EmitBlock(BodyBB);
   llvm::PHINode *ElementPHI =
       CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast");
-  ElementPHI->addIncoming(Begin.emitRawPointer(CGF), EntryBB);
+  ElementPHI->addIncoming(Begin.getPointer(), EntryBB);
   Begin = Begin.withPointer(ElementPHI, KnownNonNull);
   Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(),
                             Base.getTBAAInfo());
@@ -4485,12 +4483,12 @@ void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
       FlagsLVal);
 
   // Shift the address forward by one element.
-  llvm::Value *ElementNext =
-      CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext")
-          .emitRawPointer(CGF);
-  ElementPHI->addIncoming(ElementNext, CGF.Builder.GetInsertBlock());
+  Address ElementNext =
+      CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext");
+  ElementPHI->addIncoming(ElementNext.getPointer(),
+                          CGF.Builder.GetInsertBlock());
   llvm::Value *IsEmpty =
-      CGF.Builder.CreateICmpEQ(ElementNext, End, "omp.isempty");
+      CGF.Builder.CreateICmpEQ(ElementNext.getPointer(), End, "omp.isempty");
   CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
   // Done.
   CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
@@ -4533,7 +4531,7 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
     DepTaskArgs[1] = ThreadID;
     DepTaskArgs[2] = NewTask;
     DepTaskArgs[3] = NumOfElements;
-    DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF);
+    DepTaskArgs[4] = DependenciesArray.getPointer();
     DepTaskArgs[5] = CGF.Builder.getInt32(0);
     DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
   }
@@ -4565,7 +4563,7 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
     DepWaitTaskArgs[0] = UpLoc;
     DepWaitTaskArgs[1] = ThreadID;
     DepWaitTaskArgs[2] = NumOfElements;
-    DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
+    DepWaitTaskArgs[3] = DependenciesArray.getPointer();
     DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
     DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
     DepWaitTaskArgs[6] =
@@ -4727,8 +4725,8 @@ static void EmitOMPAggregateReduction(
   const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
   llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr);
 
-  llvm::Value *RHSBegin = RHSAddr.emitRawPointer(CGF);
-  llvm::Value *LHSBegin = LHSAddr.emitRawPointer(CGF);
+  llvm::Value *RHSBegin = RHSAddr.getPointer();
+  llvm::Value *LHSBegin = LHSAddr.getPointer();
   // Cast from pointer to array type to pointer to single element.
   llvm::Value *LHSEnd =
       CGF.Builder.CreateGEP(LHSAddr.getElementType(), LHSBegin, NumElements);
@@ -4992,7 +4990,7 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
   QualType ReductionArrayTy = C.getConstantArrayType(
       C.VoidPtrTy, ArraySize, nullptr, ArraySizeModifier::Normal,
       /*IndexTypeQuals=*/0);
-  RawAddress ReductionList =
+  Address ReductionList =
       CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
   const auto *IPriv = Privates.begin();
   unsigned Idx = 0;
@@ -5464,7 +5462,7 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
       C.getConstantArrayType(RDType, ArraySize, nullptr,
                              ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
   // kmp_task_red_input_t .rd_input.[Size];
-  RawAddress TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input.");
+  Address TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input.");
   ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionOrigs,
                        Data.ReductionCopies, Data.ReductionOps);
   for (unsigned Cnt = 0; Cnt < Size; ++Cnt) {
@@ -5475,7 +5473,7 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
         TaskRedInput.getElementType(), TaskRedInput.getPointer(), Idxs,
         /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc,
         ".rd_input.gep.");
-    LValue ElemLVal = CGF.MakeNaturalAlignRawAddrLValue(GEP, RDType);
+    LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType);
     // ElemLVal.reduce_shar = &Shareds[Cnt];
     LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD);
     RCG.emitSharedOrigLValue(CGF, Cnt);
@@ -5631,7 +5629,7 @@ void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
       DepWaitTaskArgs[0] = UpLoc;
       DepWaitTaskArgs[1] = ThreadID;
       DepWaitTaskArgs[2] = NumOfElements;
-      DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
+      DepWaitTaskArgs[3] = DependenciesArray.getPointer();
       DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
       DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
       DepWaitTaskArgs[6] =
@@ -5854,7 +5852,7 @@ void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF,
   AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, CGF.getContext().VoidPtrTy,
                                            AllocatorTraitsLVal.getBaseInfo(),
                                            AllocatorTraitsLVal.getTBAAInfo());
-  llvm::Value *Traits = Addr.emitRawPointer(CGF);
+  llvm::Value *Traits = Addr.getPointer();
 
   llvm::Value *AllocatorVal =
       CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
@@ -7314,19 +7312,17 @@ private:
                       CGF.EmitOMPSharedLValue(MC.getAssociatedExpression())
                           .getAddress(CGF);
                 }
-                llvm::Value *ComponentLBPtr = ComponentLB.emitRawPointer(CGF);
-                llvm::Value *LBPtr = LB.emitRawPointer(CGF);
-                Size = CGF.Builder.CreatePtrDiff(CGF.Int8Ty, ComponentLBPtr,
-                                                 LBPtr);
+                Size = CGF.Builder.CreatePtrDiff(
+                    CGF.Int8Ty, ComponentLB.getPointer(), LB.getPointer());
                 break;
               }
             }
             assert(Size && "Failed to determine structure size");
             CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
-            CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF));
+            CombinedInfo.BasePointers.push_back(BP.getPointer());
             CombinedInfo.DevicePtrDecls.push_back(nullptr);
             CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
-            CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));
+            CombinedInfo.Pointers.push_back(LB.getPointer());
             CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
                 Size, CGF.Int64Ty, /*isSigned=*/true));
             CombinedInfo.Types.push_back(Flags);
@@ -7336,14 +7332,13 @@ private:
             LB = CGF.Builder.CreateConstGEP(ComponentLB, 1);
           }
           CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
-          CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF));
+          CombinedInfo.BasePointers.push_back(BP.getPointer());
           CombinedInfo.DevicePtrDecls.push_back(nullptr);
           CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
-          CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));
-          llvm::Value *LBPtr = LB.emitRawPointer(CGF);
+          CombinedInfo.Pointers.push_back(LB.getPointer());
           Size = CGF.Builder.CreatePtrDiff(
-              CGF.Int8Ty, CGF.Builder.CreateConstGEP(HB, 1).emitRawPointer(CGF),
-              LBPtr);
+              CGF.Int8Ty, CGF.Builder.CreateConstGEP(HB, 1).getPointer(),
+              LB.getPointer());
           CombinedInfo.Sizes.push_back(
               CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true));
           CombinedInfo.Types.push_back(Flags);
@@ -7361,21 +7356,20 @@ private:
             (Next == CE && MapType != OMPC_MAP_unknown)) {
           if (!IsMappingWholeStruct) {
             CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
-            CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF));
+            CombinedInfo.BasePointers.push_back(BP.getPointer());
             CombinedInfo.DevicePtrDecls.push_back(nullptr);
             CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
-            CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));
+            CombinedInfo.Pointers.push_back(LB.getPointer());
             CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
                 Size, CGF.Int64Ty, /*isSigned=*/true));
             CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize
                                                                       : 1);
           } else {
             StructBaseCombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);
-            StructBaseCombinedInfo.BasePointers.push_back(
-                BP.emitRawPointer(CGF));
+            StructBaseCombinedInfo.BasePointers.push_back(BP.getPointer());
             StructBaseCombinedInfo.DevicePtrDecls.push_back(nullptr);
             StructBaseCombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
-            StructBaseCombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF));
+            StructBaseCombinedInfo.Pointers.push_back(LB.getPointer());
             StructBaseCombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast(
                 Size, CGF.Int64Ty, /*isSigned=*/true));
             StructBaseCombinedInfo.NonContigInfo.Dims.push_back(
@@ -8217,11 +8211,11 @@ public:
     }
     CombinedInfo.Exprs.push_back(VD);
     // Base is the base of the struct
-    CombinedInfo.BasePointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
+    CombinedInfo.BasePointers.push_back(PartialStruct.Base.getPointer());
     CombinedInfo.DevicePtrDecls.push_back(nullptr);
     CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);
     // Pointer is the address of the lowest element
-    llvm::Value *LB = LBAddr.emitRawPointer(CGF);
+    llvm::Value *LB = LBAddr.getPointer();
     const CXXMethodDecl *MD =
         CGF.CurFuncDecl ? dyn_cast<CXXMethodDecl>(CGF.CurFuncDecl) : nullptr;
     const CXXRecordDecl *RD = MD ? MD->getParent() : nullptr;
@@ -8235,7 +8229,7 @@ public:
       // if the this[:1] expression had appeared in a map clause with a map-type
       // of tofrom.
       // Emit this[:1]
-      CombinedInfo.Pointers.push_back(PartialStruct.Base.emitRawPointer(CGF));
+      CombinedInfo.Pointers.push_back(PartialStruct.Base.getPointer());
       QualType Ty = MD->getFunctionObjectParameterType();
       llvm::Value *Size =
           CGF.Builder.CreateIntCast(CGF.getTypeSize(Ty), CGF.Int64Ty,
@@ -8244,7 +8238,7 @@ public:
     } else {
       CombinedInfo.Pointers.push_back(LB);
       // Size is (addr of {highest+1} element) - (addr of lowest element)
-      llvm::Value *HB = HBAddr.emitRawPointer(CGF);
+      llvm::Value *HB = HBAddr.getPointer();
       llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32(
           HBAddr.getElementType(), HB, /*Idx0=*/1);
       llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy);
@@ -8753,7 +8747,7 @@ public:
         Address PtrAddr = CGF.EmitLoadOfReference(CGF.MakeAddrLValue(
             CV, ElementType, CGF.getContext().getDeclAlign(VD),
             AlignmentSource::Decl));
-        CombinedInfo.Pointers.push_back(PtrAddr.emitRawPointer(CGF));
+        CombinedInfo.Pointers.push_back(PtrAddr.getPointer());
       } else {
         CombinedInfo.Pointers.push_back(CV);
       }
@@ -9564,11 +9558,10 @@ static void emitTargetCallKernelLaunch(
     bool HasNoWait = D.hasClausesOfKind<OMPNowaitClause>();
     unsigned NumTargetItems = InputInfo.NumberOfTargetItems;
 
-    llvm::Value *BasePointersArray =
-        InputInfo.BasePointersArray.emitRawPointer(CGF);
-    llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF);
-    llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF);
-    llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF);
+    llvm::Value *BasePointersArray = InputInfo.BasePointersArray.getPointer();
+    llvm::Value *PointersArray = InputInfo.PointersArray.getPointer();
+    llvm::Value *SizesArray = InputInfo.SizesArray.getPointer();
+    llvm::Value *MappersArray = InputInfo.MappersArray.getPointer();
 
     auto &&EmitTargetCallFallbackCB =
         [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
@@ -10316,16 +10309,15 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
     // Source location for the ident struct
     llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc());
 
-    llvm::Value *OffloadingArgs[] = {
-        RTLoc,
-        DeviceID,
-        PointerNum,
-        InputInfo.BasePointersArray.emitRawPointer(CGF),
-        InputInfo.PointersArray.emitRawPointer(CGF),
-        InputInfo.SizesArray.emitRawPointer(CGF),
-        MapTypesArray,
-        MapNamesArray,
-        InputInfo.MappersArray.emitRawPointer(CGF)};
+    llvm::Value *OffloadingArgs[] = {RTLoc,
+                                     DeviceID,
+                                     PointerNum,
+                                     InputInfo.BasePointersArray.getPointer(),
+                                     InputInfo.PointersArray.getPointer(),
+                                     InputInfo.SizesArray.getPointer(),
+                                     MapTypesArray,
+                                     MapNamesArray,
+                                     InputInfo.MappersArray.getPointer()};
 
     // Select the right runtime function call for each standalone
     // directive.
@@ -11136,7 +11128,7 @@ void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF,
       getThreadID(CGF, D.getBeginLoc()),
       llvm::ConstantInt::getSigned(CGM.Int32Ty, NumIterations.size()),
       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
-          CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).emitRawPointer(CGF),
+          CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).getPointer(),
           CGM.VoidPtrTy)};
 
   llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction(
@@ -11170,8 +11162,7 @@ static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM,
                           /*Volatile=*/false, Int64Ty);
   }
   llvm::Value *Args[] = {
-      ULoc, ThreadID,
-      CGF.Builder.CreateConstArrayGEP(CntAddr, 0).emitRawPointer(CGF)};
+      ULoc, ThreadID, CGF.Builder.CreateConstArrayGEP(CntAddr, 0).getPointer()};
   llvm::FunctionCallee RTLFn;
   llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
   OMPDoacrossKind<T> ODK;
@@ -11341,7 +11332,7 @@ Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,
         Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID(
             CGF, SourceLocation::getFromRawEncoding(LocEncoding));
         Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
-            Addr.emitRawPointer(CGF), CGF.VoidPtrTy);
+            Addr.getPointer(), CGF.VoidPtrTy);
         llvm::Value *AllocVal = getAllocatorVal(CGF, AllocExpr);
         Args[2] = AllocVal;
         CGF.EmitRuntimeCall(RTLFn, Args);
@@ -11699,17 +11690,15 @@ void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF,
       LLIVTy, getName({UniqueDeclName, "iv"}));
   cast<llvm::GlobalVariable>(LastIV)->setAlignment(
       IVLVal.getAlignment().getAsAlign());
-  LValue LastIVLVal =
-      CGF.MakeNaturalAlignRawAddrLValue(LastIV, IVLVal.getType());
+  LValue LastIVLVal = CGF.MakeNaturalAlignAddrLValue(LastIV, IVLVal.getType());
 
   // Last value of the lastprivate conditional.
   // decltype(priv_a) last_a;
   llvm::GlobalVariable *Last = OMPBuilder.getOrCreateInternalVariable(
       CGF.ConvertTypeForMem(LVal.getType()), UniqueDeclName);
-  cast<llvm::GlobalVariable>(Last)->setAlignment(
-      LVal.getAlignment().getAsAlign());
-  LValue LastLVal =
-      CGF.MakeRawAddrLValue(Last, LVal.getType(), LVal.getAlignment());
+  Last->setAlignment(LVal.getAlignment().getAsAlign());
+  LValue LastLVal = CGF.MakeAddrLValue(
+      Address(Last, Last->getValueType(), LVal.getAlignment()), LVal.getType());
 
   // Global loop counter. Required to handle inner parallel-for regions.
   // iv
@@ -11882,8 +11871,9 @@ void CGOpenMPRuntime::emitLastprivateConditionalFinalUpdate(
   // The variable was not updated in the region - exit.
   if (!GV)
     return;
-  LValue LPLVal = CGF.MakeRawAddrLValue(
-      GV, PrivLVal.getType().getNonReferenceType(), PrivLVal.getAlignment());
+  LValue LPLVal = CGF.MakeAddrLValue(
+      Address(GV, GV->getValueType(), PrivLVal.getAlignment()),
+      PrivLVal.getType().getNonReferenceType());
   llvm::Value *Res = CGF.EmitLoadOfScalar(LPLVal, Loc);
   CGF.EmitStoreOfScalar(Res, PrivLVal);
 }
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index 522ae3d35d22..c3206427b143 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -1068,12 +1068,13 @@ public:
   /// \param Loc Location of the reference to threadprivate var.
   /// \return Address of the threadprivate variable for the current thread.
   virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
-                                         const VarDecl *VD, Address VDAddr,
+                                         const VarDecl *VD,
+                                         Address VDAddr,
                                          SourceLocation Loc);
 
   /// Returns the address of the variable marked as declare target with link
   /// clause OR as declare target with to clause and unified memory.
-  virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD);
+  virtual Address getAddrOfDeclareTargetVar(const VarDecl *VD);
 
   /// Emit a code for initialization of threadprivate variable. It emits
   /// a call to runtime library which adds initial value to the newly created
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index 5baac8f0e3e2..299ee1460b3d 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -1096,8 +1096,7 @@ void CGOpenMPRuntimeGPU::emitGenericVarsProlog(CodeGenFunction &CGF,
     llvm::PointerType *VarPtrTy = CGF.ConvertTypeForMem(VarTy)->getPointerTo();
     llvm::Value *CastedVoidPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
         VoidPtr, VarPtrTy, VD->getName() + "_on_stack");
-    LValue VarAddr =
-        CGF.MakeNaturalAlignPointeeRawAddrLValue(CastedVoidPtr, VarTy);
+    LValue VarAddr = CGF.MakeNaturalAlignAddrLValue(CastedVoidPtr, VarTy);
     Rec.second.PrivateAddr = VarAddr.getAddress(CGF);
     Rec.second.GlobalizedVal = VoidPtr;
 
@@ -1207,8 +1206,8 @@ void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF,
 
   bool IsBareKernel = D.getSingleClause<OMPXBareClause>();
 
-  RawAddress ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
-                                                         /*Name=*/".zero.addr");
+  Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
+                                                      /*Name=*/".zero.addr");
   CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddr);
   llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
   // We don't emit any thread id function call in bare kernel, but because the
@@ -1216,7 +1215,7 @@ void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF,
   if (IsBareKernel)
     OutlinedFnArgs.push_back(llvm::ConstantPointerNull::get(CGM.VoidPtrTy));
   else
-    OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).emitRawPointer(CGF));
+    OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).getPointer());
   OutlinedFnArgs.push_back(ZeroAddr.getPointer());
   OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
   emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
@@ -1290,7 +1289,7 @@ void CGOpenMPRuntimeGPU::emitParallelCall(CodeGenFunction &CGF,
         llvm::ConstantInt::get(CGF.Int32Ty, -1),
         FnPtr,
         ID,
-        Bld.CreateBitOrPointerCast(CapturedVarsAddrs.emitRawPointer(CGF),
+        Bld.CreateBitOrPointerCast(CapturedVarsAddrs.getPointer(),
                                    CGF.VoidPtrPtrTy),
         llvm::ConstantInt::get(CGM.SizeTy, CapturedVars.size())};
     CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
@@ -1504,18 +1503,17 @@ static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr,
       CGF.EmitBlock(PreCondBB);
       llvm::PHINode *PhiSrc =
           Bld.CreatePHI(Ptr.getType(), /*NumReservedValues=*/2);
-      PhiSrc->addIncoming(Ptr.emitRawPointer(CGF), CurrentBB);
+      PhiSrc->addIncoming(Ptr.getPointer(), CurrentBB);
       llvm::PHINode *PhiDest =
           Bld.CreatePHI(ElemPtr.getType(), /*NumReservedValues=*/2);
-      PhiDest->addIncoming(ElemPtr.emitRawPointer(CGF), CurrentBB);
+      PhiDest->addIncoming(ElemPtr.getPointer(), CurrentBB);
       Ptr = Address(PhiSrc, Ptr.getElementType(), Ptr.getAlignment());
       ElemPtr =
           Address(PhiDest, ElemPtr.getElementType(), ElemPtr.getAlignment());
-      llvm::Value *PtrEndRaw = PtrEnd.emitRawPointer(CGF);
-      llvm::Value *PtrRaw = Ptr.emitRawPointer(CGF);
       llvm::Value *PtrDiff = Bld.CreatePtrDiff(
-          CGF.Int8Ty, PtrEndRaw,
-          Bld.CreatePointerBitCastOrAddrSpaceCast(PtrRaw, CGF.VoidPtrTy));
+          CGF.Int8Ty, PtrEnd.getPointer(),
+          Bld.CreatePointerBitCastOrAddrSpaceCast(Ptr.getPointer(),
+                                                  CGF.VoidPtrTy));
       Bld.CreateCondBr(Bld.CreateICmpSGT(PtrDiff, Bld.getInt64(IntSize - 1)),
                        ThenBB, ExitBB);
       CGF.EmitBlock(ThenBB);
@@ -1530,8 +1528,8 @@ static void shuffleAndStore(CodeGenFunction &CGF, Address SrcAddr,
                             TBAAAccessInfo());
       Address LocalPtr = Bld.CreateConstGEP(Ptr, 1);
       Address LocalElemPtr = Bld.CreateConstGEP(ElemPtr, 1);
-      PhiSrc->addIncoming(LocalPtr.emitRawPointer(CGF), ThenBB);
-      PhiDest->addIncoming(LocalElemPtr.emitRawPointer(CGF), ThenBB);
+      PhiSrc->addIncoming(LocalPtr.getPointer(), ThenBB);
+      PhiDest->addIncoming(LocalElemPtr.getPointer(), ThenBB);
       CGF.EmitBranch(PreCondBB);
       CGF.EmitBlock(ExitBB);
     } else {
@@ -1678,10 +1676,10 @@ static void emitReductionListCopy(
     // scope and that of functions it invokes (i.e., reduce_function).
     // RemoteReduceData[i] = (void*)&RemoteElem
     if (UpdateDestListPtr) {
-      CGF.EmitStoreOfScalar(
-          Bld.CreatePointerBitCastOrAddrSpaceCast(
-              DestElementAddr.emitRawPointer(CGF), CGF.VoidPtrTy),
-          DestElementPtrAddr, /*Volatile=*/false, C.VoidPtrTy);
+      CGF.EmitStoreOfScalar(Bld.CreatePointerBitCastOrAddrSpaceCast(
+                                DestElementAddr.getPointer(), CGF.VoidPtrTy),
+                            DestElementPtrAddr, /*Volatile=*/false,
+                            C.VoidPtrTy);
     }
 
     ++Idx;
@@ -1832,7 +1830,7 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM,
       // elemptr = ((CopyType*)(elemptrptr)) + I
       Address ElemPtr(ElemPtrPtr, CopyType, Align);
       if (NumIters > 1)
-        ElemPtr = Bld.CreateGEP(CGF, ElemPtr, Cnt);
+        ElemPtr = Bld.CreateGEP(ElemPtr, Cnt);
 
       // Get pointer to location in transfer medium.
       // MediumPtr = &medium[warp_id]
@@ -1896,7 +1894,7 @@ static llvm::Value *emitInterWarpCopyFunction(CodeGenModule &CGM,
           TargetElemPtrPtr, /*Volatile=*/false, C.VoidPtrTy, Loc);
       Address TargetElemPtr(TargetElemPtrVal, CopyType, Align);
       if (NumIters > 1)
-        TargetElemPtr = Bld.CreateGEP(CGF, TargetElemPtr, Cnt);
+        TargetElemPtr = Bld.CreateGEP(TargetElemPtr, Cnt);
 
       // *TargetElemPtr = SrcMediumVal;
       llvm::Value *SrcMediumValue =
@@ -2107,9 +2105,9 @@ static llvm::Function *emitShuffleAndReduceFunction(
   CGF.EmitBlock(ThenBB);
   // reduce_function(LocalReduceList, RemoteReduceList)
   llvm::Value *LocalReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
-      LocalReduceList.emitRawPointer(CGF), CGF.VoidPtrTy);
+      LocalReduceList.getPointer(), CGF.VoidPtrTy);
   llvm::Value *RemoteReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
-      RemoteReduceList.emitRawPointer(CGF), CGF.VoidPtrTy);
+      RemoteReduceList.getPointer(), CGF.VoidPtrTy);
   CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
       CGF, Loc, ReduceFn, {LocalReduceListPtr, RemoteReduceListPtr});
   Bld.CreateBr(MergeBB);
@@ -2220,9 +2218,9 @@ static llvm::Value *emitListToGlobalCopyFunction(
     llvm::Value *BufferPtr =
         Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs);
     LValue GlobLVal = CGF.EmitLValueForField(
-        CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD);
+        CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD);
     Address GlobAddr = GlobLVal.getAddress(CGF);
-    GlobLVal.setAddress(Address(GlobAddr.emitRawPointer(CGF),
+    GlobLVal.setAddress(Address(GlobAddr.getPointer(),
                                 CGF.ConvertTypeForMem(Private->getType()),
                                 GlobAddr.getAlignment()));
     switch (CGF.getEvaluationKind(Private->getType())) {
@@ -2306,7 +2304,7 @@ static llvm::Value *emitListToGlobalReduceFunction(
 
   // 1. Build a list of reduction variables.
   // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
-  RawAddress ReductionList =
+  Address ReductionList =
       CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
   auto IPriv = Privates.begin();
   llvm::Value *Idxs[] = {CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(&IdxArg),
@@ -2321,10 +2319,10 @@ static llvm::Value *emitListToGlobalReduceFunction(
     llvm::Value *BufferPtr =
         Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs);
     LValue GlobLVal = CGF.EmitLValueForField(
-        CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD);
+        CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD);
     Address GlobAddr = GlobLVal.getAddress(CGF);
-    CGF.EmitStoreOfScalar(GlobAddr.emitRawPointer(CGF), Elem,
-                          /*Volatile=*/false, C.VoidPtrTy);
+    CGF.EmitStoreOfScalar(GlobAddr.getPointer(), Elem, /*Volatile=*/false,
+                          C.VoidPtrTy);
     if ((*IPriv)->getType()->isVariablyModifiedType()) {
       // Store array size.
       ++Idx;
@@ -2427,9 +2425,9 @@ static llvm::Value *emitGlobalToListCopyFunction(
     llvm::Value *BufferPtr =
         Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs);
     LValue GlobLVal = CGF.EmitLValueForField(
-        CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD);
+        CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD);
     Address GlobAddr = GlobLVal.getAddress(CGF);
-    GlobLVal.setAddress(Address(GlobAddr.emitRawPointer(CGF),
+    GlobLVal.setAddress(Address(GlobAddr.getPointer(),
                                 CGF.ConvertTypeForMem(Private->getType()),
                                 GlobAddr.getAlignment()));
     switch (CGF.getEvaluationKind(Private->getType())) {
@@ -2528,10 +2526,10 @@ static llvm::Value *emitGlobalToListReduceFunction(
     llvm::Value *BufferPtr =
         Bld.CreateInBoundsGEP(LLVMReductionsBufferTy, BufferArrPtr, Idxs);
     LValue GlobLVal = CGF.EmitLValueForField(
-        CGF.MakeNaturalAlignRawAddrLValue(BufferPtr, StaticTy), FD);
+        CGF.MakeNaturalAlignAddrLValue(BufferPtr, StaticTy), FD);
     Address GlobAddr = GlobLVal.getAddress(CGF);
-    CGF.EmitStoreOfScalar(GlobAddr.emitRawPointer(CGF), Elem,
-                          /*Volatile=*/false, C.VoidPtrTy);
+    CGF.EmitStoreOfScalar(GlobAddr.getPointer(), Elem, /*Volatile=*/false,
+                          C.VoidPtrTy);
     if ((*IPriv)->getType()->isVariablyModifiedType()) {
       // Store array size.
       ++Idx;
@@ -2547,7 +2545,7 @@ static llvm::Value *emitGlobalToListReduceFunction(
   }
 
   // Call reduce_function(ReduceList, GlobalReduceList)
-  llvm::Value *GlobalReduceList = ReductionList.emitRawPointer(CGF);
+  llvm::Value *GlobalReduceList = ReductionList.getPointer();
   Address AddrReduceListArg = CGF.GetAddrOfLocalVar(&ReduceListArg);
   llvm::Value *ReducedPtr = CGF.EmitLoadOfScalar(
       AddrReduceListArg, /*Volatile=*/false, C.VoidPtrTy, Loc);
@@ -2878,7 +2876,7 @@ void CGOpenMPRuntimeGPU::emitReduction(
   }
 
   llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
-      ReductionList.emitRawPointer(CGF), CGF.VoidPtrTy);
+      ReductionList.getPointer(), CGF.VoidPtrTy);
   llvm::Function *ReductionFn = emitReductionFunction(
       CGF.CurFn->getName(), Loc, CGF.ConvertTypeForMem(ReductionArrayTy),
       Privates, LHSExprs, RHSExprs, ReductionOps);
@@ -3108,15 +3106,15 @@ llvm::Function *CGOpenMPRuntimeGPU::createParallelDataSharingWrapper(
   // Get the array of arguments.
   SmallVector<llvm::Value *, 8> Args;
 
-  Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).emitRawPointer(CGF));
-  Args.emplace_back(ZeroAddr.emitRawPointer(CGF));
+  Args.emplace_back(CGF.GetAddrOfLocalVar(&WrapperArg).getPointer());
+  Args.emplace_back(ZeroAddr.getPointer());
 
   CGBuilderTy &Bld = CGF.Builder;
   auto CI = CS.capture_begin();
 
   // Use global memory for data sharing.
   // Handle passing of global args to workers.
-  RawAddress GlobalArgs =
+  Address GlobalArgs =
       CGF.CreateDefaultAlignTempAlloca(CGF.VoidPtrPtrTy, "global_args");
   llvm::Value *GlobalArgsPtr = GlobalArgs.getPointer();
   llvm::Value *DataSharingArgs[] = {GlobalArgsPtr};
@@ -3402,7 +3400,7 @@ void CGOpenMPRuntimeGPU::adjustTargetSpecificDataForLambdas(
         VDAddr = CGF.EmitLoadOfReferenceLValue(VDAddr,
                                                VD->getType().getCanonicalType())
                      .getAddress(CGF);
-      CGF.EmitStoreOfScalar(VDAddr.emitRawPointer(CGF), VarLVal);
+      CGF.EmitStoreOfScalar(VDAddr.getPointer(), VarLVal);
     }
   }
 }
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 576fe2f7a2d4..cb5a004e4f4a 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -2294,7 +2294,7 @@ std::pair<llvm::Value*, llvm::Type *> CodeGenFunction::EmitAsmInputLValue(
 
   Address Addr = InputValue.getAddress(*this);
   ConstraintStr += '*';
-  return {InputValue.getPointer(*this), Addr.getElementType()};
+  return {Addr.getPointer(), Addr.getElementType()};
 }
 
 std::pair<llvm::Value *, llvm::Type *>
@@ -2701,7 +2701,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
 
       ArgTypes.push_back(DestAddr.getType());
       ArgElemTypes.push_back(DestAddr.getElementType());
-      Args.push_back(DestAddr.emitRawPointer(*this));
+      Args.push_back(DestAddr.getPointer());
       Constraints += "=*";
       Constraints += OutputConstraint;
       ReadOnly = ReadNone = false;
@@ -3076,8 +3076,8 @@ CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) {
   CapturedStmtInfo->setContextValue(Builder.CreateLoad(DeclPtr));
 
   // Initialize variable-length arrays.
-  LValue Base = MakeNaturalAlignRawAddrLValue(
-      CapturedStmtInfo->getContextValue(), Ctx.getTagDeclType(RD));
+  LValue Base = MakeNaturalAlignAddrLValue(CapturedStmtInfo->getContextValue(),
+                                           Ctx.getTagDeclType(RD));
   for (auto *FD : RD->fields()) {
     if (FD->hasCapturedVLAType()) {
       auto *ExprArg =
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index e6d504bcdeca..f37ac549d10a 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -350,8 +350,7 @@ void CodeGenFunction::GenerateOpenMPCapturedVars(
         LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
 
         llvm::Value *SrcAddrVal = EmitScalarConversion(
-            DstAddr.emitRawPointer(*this),
-            Ctx.getPointerType(Ctx.getUIntPtrType()),
+            DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
             Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
         LValue SrcLV =
             MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
@@ -365,8 +364,7 @@ void CodeGenFunction::GenerateOpenMPCapturedVars(
       CapturedVars.push_back(CV);
     } else {
       assert(CurCap->capturesVariable() && "Expected capture by reference.");
-      CapturedVars.push_back(
-          EmitLValue(*I).getAddress(*this).emitRawPointer(*this));
+      CapturedVars.push_back(EmitLValue(*I).getAddress(*this).getPointer());
     }
   }
 }
@@ -377,9 +375,8 @@ static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc,
   ASTContext &Ctx = CGF.getContext();
 
   llvm::Value *CastedPtr = CGF.EmitScalarConversion(
-      AddrLV.getAddress(CGF).emitRawPointer(CGF), Ctx.getUIntPtrType(),
+      AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(),
       Ctx.getPointerType(DstType), Loc);
-  // FIXME: should the pointee type (DstType) be passed?
   Address TmpAddr =
       CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress(CGF);
   return TmpAddr;
@@ -705,8 +702,8 @@ void CodeGenFunction::EmitOMPAggregateAssign(
   llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
   SrcAddr = SrcAddr.withElementType(DestAddr.getElementType());
 
-  llvm::Value *SrcBegin = SrcAddr.emitRawPointer(*this);
-  llvm::Value *DestBegin = DestAddr.emitRawPointer(*this);
+  llvm::Value *SrcBegin = SrcAddr.getPointer();
+  llvm::Value *DestBegin = DestAddr.getPointer();
   // Cast from pointer to array type to pointer to single element.
   llvm::Value *DestEnd = Builder.CreateInBoundsGEP(DestAddr.getElementType(),
                                                    DestBegin, NumElements);
@@ -1010,10 +1007,10 @@ bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) {
           CopyBegin = createBasicBlock("copyin.not.master");
           CopyEnd = createBasicBlock("copyin.not.master.end");
           // TODO: Avoid ptrtoint conversion.
-          auto *MasterAddrInt = Builder.CreatePtrToInt(
-              MasterAddr.emitRawPointer(*this), CGM.IntPtrTy);
-          auto *PrivateAddrInt = Builder.CreatePtrToInt(
-              PrivateAddr.emitRawPointer(*this), CGM.IntPtrTy);
+          auto *MasterAddrInt =
+              Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy);
+          auto *PrivateAddrInt =
+              Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy);
           Builder.CreateCondBr(
               Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,
               CopyEnd);
@@ -1669,7 +1666,7 @@ Address CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
 
   llvm::Type *VarTy = VDAddr.getElementType();
   llvm::Value *Data =
-      CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy);
+      CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy);
   llvm::ConstantInt *Size = CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy));
   std::string Suffix = getNameWithSeparators({"cache", ""});
   llvm::Twine CacheName = Twine(CGM.getMangledName(VD)).concat(Suffix);
@@ -2048,7 +2045,7 @@ void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) {
                            ->getParam(0)
                            ->getType()
                            .getNonReferenceType();
-  RawAddress CountAddr = CreateMemTemp(LogicalTy, ".count.addr");
+  Address CountAddr = CreateMemTemp(LogicalTy, ".count.addr");
   emitCapturedStmtCall(*this, DistanceClosure, {CountAddr.getPointer()});
   llvm::Value *DistVal = Builder.CreateLoad(CountAddr, ".count");
 
@@ -2064,7 +2061,7 @@ void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) {
     LValue LCVal = EmitLValue(LoopVarRef);
     Address LoopVarAddress = LCVal.getAddress(*this);
     emitCapturedStmtCall(*this, LoopVarClosure,
-                         {LoopVarAddress.emitRawPointer(*this), IndVar});
+                         {LoopVarAddress.getPointer(), IndVar});
 
     RunCleanupsScope BodyScope(*this);
     EmitStmt(BodyStmt);
@@ -4798,7 +4795,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
       ParamTypes.push_back(PrivatesPtr->getType());
       for (const Expr *E : Data.PrivateVars) {
         const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
-        RawAddress PrivatePtr = CGF.CreateMemTemp(
+        Address PrivatePtr = CGF.CreateMemTemp(
             CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
         PrivatePtrs.emplace_back(VD, PrivatePtr);
         CallArgs.push_back(PrivatePtr.getPointer());
@@ -4806,7 +4803,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
       }
       for (const Expr *E : Data.FirstprivateVars) {
         const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
-        RawAddress PrivatePtr =
+        Address PrivatePtr =
             CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
                               ".firstpriv.ptr.addr");
         PrivatePtrs.emplace_back(VD, PrivatePtr);
@@ -4816,7 +4813,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
       }
       for (const Expr *E : Data.LastprivateVars) {
         const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
-        RawAddress PrivatePtr =
+        Address PrivatePtr =
             CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
                               ".lastpriv.ptr.addr");
         PrivatePtrs.emplace_back(VD, PrivatePtr);
@@ -4829,7 +4826,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
           Ty = CGF.getContext().getPointerType(Ty);
         if (isAllocatableDecl(VD))
           Ty = CGF.getContext().getPointerType(Ty);
-        RawAddress PrivatePtr = CGF.CreateMemTemp(
+        Address PrivatePtr = CGF.CreateMemTemp(
             CGF.getContext().getPointerType(Ty), ".local.ptr.addr");
         auto Result = UntiedLocalVars.insert(
             std::make_pair(VD, std::make_pair(PrivatePtr, Address::invalid())));
@@ -4862,7 +4859,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
         if (auto *DI = CGF.getDebugInfo())
           if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
             (void)DI->EmitDeclareOfAutoVariable(
-                Pair.first, Pair.second.getBasePointer(), CGF.Builder,
+                Pair.first, Pair.second.getPointer(), CGF.Builder,
                 /*UsePointerValue*/ true);
       }
       // Adjust mapping for internal locals by mapping actual memory instead of
@@ -4915,14 +4912,14 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
                                                            RedCG, Cnt);
         Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
             CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
-        Replacement = Address(
-            CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),
-                                     CGF.getContext().VoidPtrTy,
-                                     CGF.getContext().getPointerType(
-                                         Data.ReductionCopies[Cnt]->getType()),
-                                     Data.ReductionCopies[Cnt]->getExprLoc()),
-            CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
-            Replacement.getAlignment());
+        Replacement =
+            Address(CGF.EmitScalarConversion(
+                        Replacement.getPointer(), CGF.getContext().VoidPtrTy,
+                        CGF.getContext().getPointerType(
+                            Data.ReductionCopies[Cnt]->getType()),
+                        Data.ReductionCopies[Cnt]->getExprLoc()),
+                    CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
+                    Replacement.getAlignment());
         Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
         Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
       }
@@ -4973,7 +4970,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
             CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
         Replacement = Address(
             CGF.EmitScalarConversion(
-                Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,
+                Replacement.getPointer(), CGF.getContext().VoidPtrTy,
                 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
                 InRedPrivs[Cnt]->getExprLoc()),
             CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
@@ -5092,7 +5089,7 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
     // If there is no user-defined mapper, the mapper array will be nullptr. In
     // this case, we don't need to privatize it.
     if (!isa_and_nonnull<llvm::ConstantPointerNull>(
-            InputInfo.MappersArray.emitRawPointer(*this))) {
+            InputInfo.MappersArray.getPointer())) {
       MVD = createImplicitFirstprivateForType(
           getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
       TargetScope.addPrivate(MVD, InputInfo.MappersArray);
@@ -5118,7 +5115,7 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
       ParamTypes.push_back(PrivatesPtr->getType());
       for (const Expr *E : Data.FirstprivateVars) {
         const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
-        RawAddress PrivatePtr =
+        Address PrivatePtr =
             CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
                               ".firstpriv.ptr.addr");
         PrivatePtrs.emplace_back(VD, PrivatePtr);
@@ -5197,14 +5194,14 @@ void CodeGenFunction::processInReduction(const OMPExecutableDirective &S,
                                                          RedCG, Cnt);
       Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
           CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
-      Replacement = Address(
-          CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),
-                                   CGF.getContext().VoidPtrTy,
-                                   CGF.getContext().getPointerType(
-                                       Data.ReductionCopies[Cnt]->getType()),
-                                   Data.ReductionCopies[Cnt]->getExprLoc()),
-          CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
-          Replacement.getAlignment());
+      Replacement =
+          Address(CGF.EmitScalarConversion(
+                      Replacement.getPointer(), CGF.getContext().VoidPtrTy,
+                      CGF.getContext().getPointerType(
+                          Data.ReductionCopies[Cnt]->getType()),
+                      Data.ReductionCopies[Cnt]->getExprLoc()),
+                  CGF.ConvertTypeForMem(Data.ReductionCopies[Cnt]->getType()),
+                  Replacement.getAlignment());
       Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
       Scope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);
     }
@@ -5250,7 +5247,7 @@ void CodeGenFunction::processInReduction(const OMPExecutableDirective &S,
           CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
       Replacement = Address(
           CGF.EmitScalarConversion(
-              Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,
+              Replacement.getPointer(), CGF.getContext().VoidPtrTy,
               CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
               InRedPrivs[Cnt]->getExprLoc()),
           CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),
@@ -5397,7 +5394,7 @@ void CodeGenFunction::EmitOMPDepobjDirective(const OMPDepobjDirective &S) {
     Dependencies.DepExprs.append(DC->varlist_begin(), DC->varlist_end());
     Address DepAddr = CGM.getOpenMPRuntime().emitDepobjDependClause(
         *this, Dependencies, DC->getBeginLoc());
-    EmitStoreOfScalar(DepAddr.emitRawPointer(*this), DOLVal);
+    EmitStoreOfScalar(DepAddr.getPointer(), DOLVal);
     return;
   }
   if (const auto *DC = S.getSingleClause<OMPDestroyClause>()) {
@@ -6474,21 +6471,21 @@ static void emitOMPAtomicCompareExpr(
           D->getType()->hasSignedIntegerRepresentation());
 
   llvm::OpenMPIRBuilder::AtomicOpValue XOpVal{
-      XAddr.emitRawPointer(CGF), XAddr.getElementType(),
+      XAddr.getPointer(), XAddr.getElementType(),
       X->getType()->hasSignedIntegerRepresentation(),
       X->getType().isVolatileQualified()};
   llvm::OpenMPIRBuilder::AtomicOpValue VOpVal, ROpVal;
   if (V) {
     LValue LV = CGF.EmitLValue(V);
     Address Addr = LV.getAddress(CGF);
-    VOpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(),
+    VOpVal = {Addr.getPointer(), Addr.getElementType(),
               V->getType()->hasSignedIntegerRepresentation(),
               V->getType().isVolatileQualified()};
   }
   if (R) {
     LValue LV = CGF.EmitLValue(R);
     Address Addr = LV.getAddress(CGF);
-    ROpVal = {Addr.emitRawPointer(CGF), Addr.getElementType(),
+    ROpVal = {Addr.getPointer(), Addr.getElementType(),
               R->getType()->hasSignedIntegerRepresentation(),
               R->getType().isVolatileQualified()};
   }
@@ -7032,7 +7029,7 @@ void CodeGenFunction::EmitOMPInteropDirective(const OMPInteropDirective &S) {
     std::tie(NumDependences, DependenciesArray) =
         CGM.getOpenMPRuntime().emitDependClause(*this, Data.Dependences,
                                                 S.getBeginLoc());
-    DependenceList = DependenciesArray.emitRawPointer(*this);
+    DependenceList = DependenciesArray.getPointer();
   }
   Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();
 
diff --git a/clang/lib/CodeGen/CGVTables.cpp b/clang/lib/CodeGen/CGVTables.cpp
index 862369ae009f..8dee3f74b44b 100644
--- a/clang/lib/CodeGen/CGVTables.cpp
+++ b/clang/lib/CodeGen/CGVTables.cpp
@@ -201,13 +201,14 @@ CodeGenFunction::GenerateVarArgsThunk(llvm::Function *Fn,
 
   // Find the first store of "this", which will be to the alloca associated
   // with "this".
-  Address ThisPtr = makeNaturalAddressForPointer(
-      &*AI, MD->getFunctionObjectParameterType(),
-      CGM.getClassPointerAlignment(MD->getParent()));
+  Address ThisPtr =
+      Address(&*AI, ConvertTypeForMem(MD->getFunctionObjectParameterType()),
+              CGM.getClassPointerAlignment(MD->getParent()));
   llvm::BasicBlock *EntryBB = &Fn->front();
   llvm::BasicBlock::iterator ThisStore =
       llvm::find_if(*EntryBB, [&](llvm::Instruction &I) {
-        return isa<llvm::StoreInst>(I) && I.getOperand(0) == &*AI;
+        return isa<llvm::StoreInst>(I) &&
+               I.getOperand(0) == ThisPtr.getPointer();
       });
   assert(ThisStore != EntryBB->end() &&
          "Store of this should be in entry block?");
diff --git a/clang/lib/CodeGen/CGValue.h b/clang/lib/CodeGen/CGValue.h
index 9265bcbb438e..1e6f67250583 100644
--- a/clang/lib/CodeGen/CGValue.h
+++ b/clang/lib/CodeGen/CGValue.h
@@ -14,13 +14,12 @@
 #ifndef LLVM_CLANG_LIB_CODEGEN_CGVALUE_H
 #define LLVM_CLANG_LIB_CODEGEN_CGVALUE_H
 
-#include "Address.h"
-#include "CodeGenTBAA.h"
-#include "EHScopeStack.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Type.h"
-#include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
+#include "llvm/IR/Type.h"
+#include "Address.h"
+#include "CodeGenTBAA.h"
 
 namespace llvm {
   class Constant;
@@ -29,64 +28,57 @@ namespace llvm {
 
 namespace clang {
 namespace CodeGen {
-class AggValueSlot;
-class CGBuilderTy;
-class CodeGenFunction;
-struct CGBitFieldInfo;
+  class AggValueSlot;
+  class CodeGenFunction;
+  struct CGBitFieldInfo;
 
 /// RValue - This trivial value class is used to represent the result of an
 /// expression that is evaluated.  It can be one of three things: either a
 /// simple LLVM SSA value, a pair of SSA values for complex numbers, or the
 /// address of an aggregate value in memory.
 class RValue {
-  friend struct DominatingValue<RValue>;
+  enum Flavor { Scalar, Complex, Aggregate };
 
-  enum FlavorEnum { Scalar, Complex, Aggregate };
+  // The shift to make to an aggregate's alignment to make it look
+  // like a pointer.
+  enum { AggAlignShift = 4 };
 
-  union {
-    // Stores first and second value.
-    struct {
-      llvm::Value *first;
-      llvm::Value *second;
-    } Vals;
-
-    // Stores aggregate address.
-    Address AggregateAddr;
-  };
-
-  unsigned IsVolatile : 1;
-  unsigned Flavor : 2;
+  // Stores first value and flavor.
+  llvm::PointerIntPair<llvm::Value *, 2, Flavor> V1;
+  // Stores second value and volatility.
+  llvm::PointerIntPair<llvm::Value *, 1, bool> V2;
+  // Stores element type for aggregate values.
+  llvm::Type *ElementType;
 
 public:
-  RValue() : Vals{nullptr, nullptr}, Flavor(Scalar) {}
-
-  bool isScalar() const { return Flavor == Scalar; }
-  bool isComplex() const { return Flavor == Complex; }
-  bool isAggregate() const { return Flavor == Aggregate; }
+  bool isScalar() const { return V1.getInt() == Scalar; }
+  bool isComplex() const { return V1.getInt() == Complex; }
+  bool isAggregate() const { return V1.getInt() == Aggregate; }
 
-  bool isVolatileQualified() const { return IsVolatile; }
+  bool isVolatileQualified() const { return V2.getInt(); }
 
   /// getScalarVal() - Return the Value* of this scalar value.
   llvm::Value *getScalarVal() const {
     assert(isScalar() && "Not a scalar!");
-    return Vals.first;
+    return V1.getPointer();
   }
 
   /// getComplexVal - Return the real/imag components of this complex value.
   ///
   std::pair<llvm::Value *, llvm::Value *> getComplexVal() const {
-    return std::make_pair(Vals.first, Vals.second);
+    return std::make_pair(V1.getPointer(), V2.getPointer());
   }
 
   /// getAggregateAddr() - Return the Value* of the address of the aggregate.
   Address getAggregateAddress() const {
     assert(isAggregate() && "Not an aggregate!");
-    return AggregateAddr;
+    auto align = reinterpret_cast<uintptr_t>(V2.getPointer()) >> AggAlignShift;
+    return Address(
+        V1.getPointer(), ElementType, CharUnits::fromQuantity(align));
   }
-
-  llvm::Value *getAggregatePointer(QualType PointeeType,
-                                   CodeGenFunction &CGF) const {
-    return getAggregateAddress().getBasePointer();
+  llvm::Value *getAggregatePointer() const {
+    assert(isAggregate() && "Not an aggregate!");
+    return V1.getPointer();
   }
 
   static RValue getIgnored() {
@@ -96,19 +88,17 @@ public:
 
   static RValue get(llvm::Value *V) {
     RValue ER;
-    ER.Vals.first = V;
-    ER.Flavor = Scalar;
-    ER.IsVolatile = false;
+    ER.V1.setPointer(V);
+    ER.V1.setInt(Scalar);
+    ER.V2.setInt(false);
     return ER;
   }
-  static RValue get(Address Addr, CodeGenFunction &CGF) {
-    return RValue::get(Addr.emitRawPointer(CGF));
-  }
   static RValue getComplex(llvm::Value *V1, llvm::Value *V2) {
     RValue ER;
-    ER.Vals = {V1, V2};
-    ER.Flavor = Complex;
-    ER.IsVolatile = false;
+    ER.V1.setPointer(V1);
+    ER.V2.setPointer(V2);
+    ER.V1.setInt(Complex);
+    ER.V2.setInt(false);
     return ER;
   }
   static RValue getComplex(const std::pair<llvm::Value *, llvm::Value *> &C) {
@@ -117,15 +107,15 @@ public:
   // FIXME: Aggregate rvalues need to retain information about whether they are
   // volatile or not.  Remove default to find all places that probably get this
   // wrong.
-
-  /// Convert an Address to an RValue. If the Address is not
-  /// signed, create an RValue using the unsigned address. Otherwise, resign the
-  /// address using the provided type.
   static RValue getAggregate(Address addr, bool isVolatile = false) {
     RValue ER;
-    ER.AggregateAddr = addr;
-    ER.Flavor = Aggregate;
-    ER.IsVolatile = isVolatile;
+    ER.V1.setPointer(addr.getPointer());
+    ER.V1.setInt(Aggregate);
+    ER.ElementType = addr.getElementType();
+
+    auto align = static_cast<uintptr_t>(addr.getAlignment().getQuantity());
+    ER.V2.setPointer(reinterpret_cast<llvm::Value*>(align << AggAlignShift));
+    ER.V2.setInt(isVolatile);
     return ER;
   }
 };
@@ -188,10 +178,8 @@ class LValue {
     MatrixElt     // This is a matrix element, use getVector*
   } LVType;
 
-  union {
-    Address Addr = Address::invalid();
-    llvm::Value *V;
-  };
+  llvm::Value *V;
+  llvm::Type *ElementType;
 
   union {
     // Index into a vector subscript: V[i]
@@ -209,6 +197,10 @@ class LValue {
   // 'const' is unused here
   Qualifiers Quals;
 
+  // The alignment to use when accessing this lvalue.  (For vector elements,
+  // this is the alignment of the whole vector.)
+  unsigned Alignment;
+
   // objective-c's ivar
   bool Ivar:1;
 
@@ -242,19 +234,23 @@ class LValue {
   Expr *BaseIvarExp;
 
 private:
-  void Initialize(QualType Type, Qualifiers Quals, Address Addr,
+  void Initialize(QualType Type, Qualifiers Quals, CharUnits Alignment,
                   LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) {
+    assert((!Alignment.isZero() || Type->isIncompleteType()) &&
+           "initializing l-value with zero alignment!");
+    if (isGlobalReg())
+      assert(ElementType == nullptr && "Global reg does not store elem type");
+    else
+      assert(ElementType != nullptr && "Must have elem type");
+
     this->Type = Type;
     this->Quals = Quals;
     const unsigned MaxAlign = 1U << 31;
-    CharUnits Alignment = Addr.getAlignment();
-    assert((isGlobalReg() || !Alignment.isZero() || Type->isIncompleteType()) &&
-           "initializing l-value with zero alignment!");
-    if (Alignment.getQuantity() > MaxAlign) {
-      assert(false && "Alignment exceeds allowed max!");
-      Alignment = CharUnits::fromQuantity(MaxAlign);
-    }
-    this->Addr = Addr;
+    this->Alignment = Alignment.getQuantity() <= MaxAlign
+                          ? Alignment.getQuantity()
+                          : MaxAlign;
+    assert(this->Alignment == Alignment.getQuantity() &&
+           "Alignment exceeds allowed max!");
     this->BaseInfo = BaseInfo;
     this->TBAAInfo = TBAAInfo;
 
@@ -266,16 +262,6 @@ private:
     this->BaseIvarExp = nullptr;
   }
 
-  void initializeSimpleLValue(Address Addr, QualType Type,
-                              LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo,
-                              ASTContext &Context) {
-    Qualifiers QS = Type.getQualifiers();
-    QS.setObjCGCAttr(Context.getObjCGCAttrKind(Type));
-    LVType = Simple;
-    Initialize(Type, QS, Addr, BaseInfo, TBAAInfo);
-    assert(Addr.getBasePointer()->getType()->isPointerTy());
-  }
-
 public:
   bool isSimple() const { return LVType == Simple; }
   bool isVectorElt() const { return LVType == VectorElt; }
@@ -342,8 +328,8 @@ public:
 
   LangAS getAddressSpace() const { return Quals.getAddressSpace(); }
 
-  CharUnits getAlignment() const { return Addr.getAlignment(); }
-  void setAlignment(CharUnits A) { Addr.setAlignment(A); }
+  CharUnits getAlignment() const { return CharUnits::fromQuantity(Alignment); }
+  void setAlignment(CharUnits A) { Alignment = A.getQuantity(); }
 
   LValueBaseInfo getBaseInfo() const { return BaseInfo; }
   void setBaseInfo(LValueBaseInfo Info) { BaseInfo = Info; }
@@ -359,32 +345,28 @@ public:
   // simple lvalue
   llvm::Value *getPointer(CodeGenFunction &CGF) const {
     assert(isSimple());
-    return Addr.getBasePointer();
+    return V;
   }
-  llvm::Value *emitRawPointer(CodeGenFunction &CGF) const {
-    assert(isSimple());
-    return Addr.isValid() ? Addr.emitRawPointer(CGF) : nullptr;
-  }
-
   Address getAddress(CodeGenFunction &CGF) const {
-    // FIXME: remove parameter.
-    return Addr;
+    return Address(getPointer(CGF), ElementType, getAlignment(),
+                   isKnownNonNull());
+  }
+  void setAddress(Address address) {
+    assert(isSimple());
+    V = address.getPointer();
+    ElementType = address.getElementType();
+    Alignment = address.getAlignment().getQuantity();
+    IsKnownNonNull = address.isKnownNonNull();
   }
-
-  void setAddress(Address address) { Addr = address; }
 
   // vector elt lvalue
   Address getVectorAddress() const {
-    assert(isVectorElt());
-    return Addr;
-  }
-  llvm::Value *getRawVectorPointer(CodeGenFunction &CGF) const {
-    assert(isVectorElt());
-    return Addr.emitRawPointer(CGF);
+    return Address(getVectorPointer(), ElementType, getAlignment(),
+                   (KnownNonNull_t)isKnownNonNull());
   }
   llvm::Value *getVectorPointer() const {
     assert(isVectorElt());
-    return Addr.getBasePointer();
+    return V;
   }
   llvm::Value *getVectorIdx() const {
     assert(isVectorElt());
@@ -392,12 +374,12 @@ public:
   }
 
   Address getMatrixAddress() const {
-    assert(isMatrixElt());
-    return Addr;
+    return Address(getMatrixPointer(), ElementType, getAlignment(),
+                   (KnownNonNull_t)isKnownNonNull());
   }
   llvm::Value *getMatrixPointer() const {
     assert(isMatrixElt());
-    return Addr.getBasePointer();
+    return V;
   }
   llvm::Value *getMatrixIdx() const {
     assert(isMatrixElt());
@@ -406,12 +388,12 @@ public:
 
   // extended vector elements.
   Address getExtVectorAddress() const {
-    assert(isExtVectorElt());
-    return Addr;
+    return Address(getExtVectorPointer(), ElementType, getAlignment(),
+                   (KnownNonNull_t)isKnownNonNull());
   }
-  llvm::Value *getRawExtVectorPointer(CodeGenFunction &CGF) const {
+  llvm::Value *getExtVectorPointer() const {
     assert(isExtVectorElt());
-    return Addr.emitRawPointer(CGF);
+    return V;
   }
   llvm::Constant *getExtVectorElts() const {
     assert(isExtVectorElt());
@@ -420,14 +402,10 @@ public:
 
   // bitfield lvalue
   Address getBitFieldAddress() const {
-    assert(isBitField());
-    return Addr;
-  }
-  llvm::Value *getRawBitFieldPointer(CodeGenFunction &CGF) const {
-    assert(isBitField());
-    return Addr.emitRawPointer(CGF);
+    return Address(getBitFieldPointer(), ElementType, getAlignment(),
+                   (KnownNonNull_t)isKnownNonNull());
   }
-
+  llvm::Value *getBitFieldPointer() const { assert(isBitField()); return V; }
   const CGBitFieldInfo &getBitFieldInfo() const {
     assert(isBitField());
     return *BitFieldInfo;
@@ -436,13 +414,18 @@ public:
   // global register lvalue
   llvm::Value *getGlobalReg() const { assert(isGlobalReg()); return V; }
 
-  static LValue MakeAddr(Address Addr, QualType type, ASTContext &Context,
+  static LValue MakeAddr(Address address, QualType type, ASTContext &Context,
                          LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) {
+    Qualifiers qs = type.getQualifiers();
+    qs.setObjCGCAttr(Context.getObjCGCAttrKind(type));
+
     LValue R;
     R.LVType = Simple;
-    R.initializeSimpleLValue(Addr, type, BaseInfo, TBAAInfo, Context);
-    R.Addr = Addr;
-    assert(Addr.getType()->isPointerTy());
+    assert(address.getPointer()->getType()->isPointerTy());
+    R.V = address.getPointer();
+    R.ElementType = address.getElementType();
+    R.IsKnownNonNull = address.isKnownNonNull();
+    R.Initialize(type, qs, address.getAlignment(), BaseInfo, TBAAInfo);
     return R;
   }
 
@@ -451,18 +434,26 @@ public:
                               TBAAAccessInfo TBAAInfo) {
     LValue R;
     R.LVType = VectorElt;
+    R.V = vecAddress.getPointer();
+    R.ElementType = vecAddress.getElementType();
     R.VectorIdx = Idx;
-    R.Initialize(type, type.getQualifiers(), vecAddress, BaseInfo, TBAAInfo);
+    R.IsKnownNonNull = vecAddress.isKnownNonNull();
+    R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(),
+                 BaseInfo, TBAAInfo);
     return R;
   }
 
-  static LValue MakeExtVectorElt(Address Addr, llvm::Constant *Elts,
+  static LValue MakeExtVectorElt(Address vecAddress, llvm::Constant *Elts,
                                  QualType type, LValueBaseInfo BaseInfo,
                                  TBAAAccessInfo TBAAInfo) {
     LValue R;
     R.LVType = ExtVectorElt;
+    R.V = vecAddress.getPointer();
+    R.ElementType = vecAddress.getElementType();
     R.VectorElts = Elts;
-    R.Initialize(type, type.getQualifiers(), Addr, BaseInfo, TBAAInfo);
+    R.IsKnownNonNull = vecAddress.isKnownNonNull();
+    R.Initialize(type, type.getQualifiers(), vecAddress.getAlignment(),
+                 BaseInfo, TBAAInfo);
     return R;
   }
 
@@ -477,8 +468,12 @@ public:
                              TBAAAccessInfo TBAAInfo) {
     LValue R;
     R.LVType = BitField;
+    R.V = Addr.getPointer();
+    R.ElementType = Addr.getElementType();
     R.BitFieldInfo = &Info;
-    R.Initialize(type, type.getQualifiers(), Addr, BaseInfo, TBAAInfo);
+    R.IsKnownNonNull = Addr.isKnownNonNull();
+    R.Initialize(type, type.getQualifiers(), Addr.getAlignment(), BaseInfo,
+                 TBAAInfo);
     return R;
   }
 
@@ -486,9 +481,11 @@ public:
                               QualType type) {
     LValue R;
     R.LVType = GlobalReg;
-    R.Initialize(type, type.getQualifiers(), Address::invalid(),
-                 LValueBaseInfo(AlignmentSource::Decl), TBAAAccessInfo());
     R.V = V;
+    R.ElementType = nullptr;
+    R.IsKnownNonNull = true;
+    R.Initialize(type, type.getQualifiers(), alignment,
+                 LValueBaseInfo(AlignmentSource::Decl), TBAAAccessInfo());
     return R;
   }
 
@@ -497,8 +494,12 @@ public:
                               TBAAAccessInfo TBAAInfo) {
     LValue R;
     R.LVType = MatrixElt;
+    R.V = matAddress.getPointer();
+    R.ElementType = matAddress.getElementType();
     R.VectorIdx = Idx;
-    R.Initialize(type, type.getQualifiers(), matAddress, BaseInfo, TBAAInfo);
+    R.IsKnownNonNull = matAddress.isKnownNonNull();
+    R.Initialize(type, type.getQualifiers(), matAddress.getAlignment(),
+                 BaseInfo, TBAAInfo);
     return R;
   }
 
@@ -642,17 +643,17 @@ public:
     return NeedsGCBarriers_t(ObjCGCFlag);
   }
 
-  llvm::Value *getPointer(QualType PointeeTy, CodeGenFunction &CGF) const;
-
-  llvm::Value *emitRawPointer(CodeGenFunction &CGF) const {
-    return Addr.isValid() ? Addr.emitRawPointer(CGF) : nullptr;
+  llvm::Value *getPointer() const {
+    return Addr.getPointer();
   }
 
   Address getAddress() const {
     return Addr;
   }
 
-  bool isIgnored() const { return !Addr.isValid(); }
+  bool isIgnored() const {
+    return !Addr.isValid();
+  }
 
   CharUnits getAlignment() const {
     return Addr.getAlignment();
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 0ceac7db89df..fad26c43da3d 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -193,35 +193,26 @@ CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() {
   CGF.Builder.setDefaultConstrainedRounding(OldRounding);
 }
 
-static LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,
-                                         bool ForPointeeType,
-                                         CodeGenFunction &CGF) {
+LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) {
   LValueBaseInfo BaseInfo;
   TBAAAccessInfo TBAAInfo;
-  CharUnits Alignment =
-      CGF.CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo, ForPointeeType);
-  Address Addr = Address(V, CGF.ConvertTypeForMem(T), Alignment);
-  return CGF.MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
-}
-
-LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) {
-  return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false, *this);
+  CharUnits Alignment = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo);
+  Address Addr(V, ConvertTypeForMem(T), Alignment);
+  return LValue::MakeAddr(Addr, T, getContext(), BaseInfo, TBAAInfo);
 }
 
+/// Given a value of type T* that may not be to a complete object,
+/// construct an l-value with the natural pointee alignment of T.
 LValue
 CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {
-  return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true, *this);
-}
-
-LValue CodeGenFunction::MakeNaturalAlignRawAddrLValue(llvm::Value *V,
-                                                      QualType T) {
-  return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false, *this);
+  LValueBaseInfo BaseInfo;
+  TBAAAccessInfo TBAAInfo;
+  CharUnits Align = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo,
+                                                /* forPointeeType= */ true);
+  Address Addr(V, ConvertTypeForMem(T), Align);
+  return MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
 }
 
-LValue CodeGenFunction::MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V,
-                                                             QualType T) {
-  return ::MakeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true, *this);
-}
 
 llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {
   return CGM.getTypes().ConvertTypeForMem(T);
@@ -534,8 +525,7 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
     ReturnBlock.getBlock()->eraseFromParent();
   }
   if (ReturnValue.isValid()) {
-    auto *RetAlloca =
-        dyn_cast<llvm::AllocaInst>(ReturnValue.emitRawPointer(*this));
+    auto *RetAlloca = dyn_cast<llvm::AllocaInst>(ReturnValue.getPointer());
     if (RetAlloca && RetAlloca->use_empty()) {
       RetAlloca->eraseFromParent();
       ReturnValue = Address::invalid();
@@ -1132,14 +1122,13 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
     auto AI = CurFn->arg_begin();
     if (CurFnInfo->getReturnInfo().isSRetAfterThis())
       ++AI;
-    ReturnValue = makeNaturalAddressForPointer(
-        &*AI, RetTy, CurFnInfo->getReturnInfo().getIndirectAlign(), false,
-        nullptr, nullptr, KnownNonNull);
+    ReturnValue =
+        Address(&*AI, ConvertType(RetTy),
+                CurFnInfo->getReturnInfo().getIndirectAlign(), KnownNonNull);
     if (!CurFnInfo->getReturnInfo().getIndirectByVal()) {
-      ReturnValuePointer =
-          CreateDefaultAlignTempAlloca(ReturnValue.getType(), "result.ptr");
-      Builder.CreateStore(ReturnValue.emitRawPointer(*this),
-                          ReturnValuePointer);
+      ReturnValuePointer = CreateDefaultAlignTempAlloca(
+          ReturnValue.getPointer()->getType(), "result.ptr");
+      Builder.CreateStore(ReturnValue.getPointer(), ReturnValuePointer);
     }
   } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca &&
              !hasScalarEvaluationKind(CurFnInfo->getReturnType())) {
@@ -1200,9 +1189,8 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
         // or contains the address of the enclosing object).
         LValue ThisFieldLValue = EmitLValueForLambdaField(LambdaThisCaptureField);
         if (!LambdaThisCaptureField->getType()->isPointerType()) {
-          // If the enclosing object was captured by value, just use its
-          // address. Sign this pointer.
-          CXXThisValue = ThisFieldLValue.getPointer(*this);
+          // If the enclosing object was captured by value, just use its address.
+          CXXThisValue = ThisFieldLValue.getAddress(*this).getPointer();
         } else {
           // Load the lvalue pointed to by the field, since '*this' was captured
           // by reference.
@@ -2024,9 +2012,8 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
     = llvm::ConstantInt::get(CGF.IntPtrTy, baseSize.getQuantity());
 
   Address begin = dest.withElementType(CGF.Int8Ty);
-  llvm::Value *end = Builder.CreateInBoundsGEP(begin.getElementType(),
-                                               begin.emitRawPointer(CGF),
-                                               sizeInChars, "vla.end");
+  llvm::Value *end = Builder.CreateInBoundsGEP(
+      begin.getElementType(), begin.getPointer(), sizeInChars, "vla.end");
 
   llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock();
   llvm::BasicBlock *loopBB = CGF.createBasicBlock("vla-init.loop");
@@ -2037,7 +2024,7 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
   CGF.EmitBlock(loopBB);
 
   llvm::PHINode *cur = Builder.CreatePHI(begin.getType(), 2, "vla.cur");
-  cur->addIncoming(begin.emitRawPointer(CGF), originBB);
+  cur->addIncoming(begin.getPointer(), originBB);
 
   CharUnits curAlign =
     dest.getAlignment().alignmentOfArrayElement(baseSize);
@@ -2231,10 +2218,10 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
     addr = addr.withElementType(baseType);
   } else {
     // Create the actual GEP.
-    addr = Address(Builder.CreateInBoundsGEP(addr.getElementType(),
-                                             addr.emitRawPointer(*this),
-                                             gepIndices, "array.begin"),
-                   ConvertTypeForMem(eltType), addr.getAlignment());
+    addr = Address(Builder.CreateInBoundsGEP(
+        addr.getElementType(), addr.getPointer(), gepIndices, "array.begin"),
+        ConvertTypeForMem(eltType),
+        addr.getAlignment());
   }
 
   baseType = eltType;
@@ -2575,7 +2562,7 @@ void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) {
 Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
                                               Address Addr) {
   assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
-  llvm::Value *V = Addr.emitRawPointer(*this);
+  llvm::Value *V = Addr.getPointer();
   llvm::Type *VTy = V->getType();
   auto *PTy = dyn_cast<llvm::PointerType>(VTy);
   unsigned AS = PTy ? PTy->getAddressSpace() : 0;
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 8dd6da5f85f1..e8f8aa601ed0 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -151,9 +151,6 @@ struct DominatingLLVMValue {
 
   /// Answer whether the given value needs extra work to be saved.
   static bool needsSaving(llvm::Value *value) {
-    if (!value)
-      return false;
-
     // If it's not an instruction, we don't need to save.
     if (!isa<llvm::Instruction>(value)) return false;
 
@@ -180,28 +177,21 @@ template <> struct DominatingValue<Address> {
   typedef Address type;
 
   struct saved_type {
-    DominatingLLVMValue::saved_type BasePtr;
+    DominatingLLVMValue::saved_type SavedValue;
     llvm::Type *ElementType;
     CharUnits Alignment;
-    DominatingLLVMValue::saved_type Offset;
-    llvm::PointerType *EffectiveType;
   };
 
   static bool needsSaving(type value) {
-    if (DominatingLLVMValue::needsSaving(value.getBasePointer()) ||
-        DominatingLLVMValue::needsSaving(value.getOffset()))
-      return true;
-    return false;
+    return DominatingLLVMValue::needsSaving(value.getPointer());
   }
   static saved_type save(CodeGenFunction &CGF, type value) {
-    return {DominatingLLVMValue::save(CGF, value.getBasePointer()),
-            value.getElementType(), value.getAlignment(),
-            DominatingLLVMValue::save(CGF, value.getOffset()), value.getType()};
+    return { DominatingLLVMValue::save(CGF, value.getPointer()),
+             value.getElementType(), value.getAlignment() };
   }
   static type restore(CodeGenFunction &CGF, saved_type value) {
-    return Address(DominatingLLVMValue::restore(CGF, value.BasePtr),
-                   value.ElementType, value.Alignment,
-                   DominatingLLVMValue::restore(CGF, value.Offset));
+    return Address(DominatingLLVMValue::restore(CGF, value.SavedValue),
+                   value.ElementType, value.Alignment);
   }
 };
 
@@ -211,26 +201,14 @@ template <> struct DominatingValue<RValue> {
   class saved_type {
     enum Kind { ScalarLiteral, ScalarAddress, AggregateLiteral,
                 AggregateAddress, ComplexAddress };
-    union {
-      struct {
-        DominatingLLVMValue::saved_type first, second;
-      } Vals;
-      DominatingValue<Address>::saved_type AggregateAddr;
-    };
+
+    llvm::Value *Value;
+    llvm::Type *ElementType;
     LLVM_PREFERRED_TYPE(Kind)
     unsigned K : 3;
-    unsigned IsVolatile : 1;
-
-    saved_type(DominatingLLVMValue::saved_type Val1, unsigned K)
-        : Vals{Val1, DominatingLLVMValue::saved_type()}, K(K) {}
-
-    saved_type(DominatingLLVMValue::saved_type Val1,
-               DominatingLLVMValue::saved_type Val2)
-        : Vals{Val1, Val2}, K(ComplexAddress) {}
-
-    saved_type(DominatingValue<Address>::saved_type AggregateAddr,
-               bool IsVolatile, unsigned K)
-        : AggregateAddr(AggregateAddr), K(K) {}
+    unsigned Align : 29;
+    saved_type(llvm::Value *v, llvm::Type *e, Kind k, unsigned a = 0)
+      : Value(v), ElementType(e), K(k), Align(a) {}
 
   public:
     static bool needsSaving(RValue value);
@@ -681,7 +659,7 @@ public:
     llvm::Value *Size;
 
   public:
-    CallLifetimeEnd(RawAddress addr, llvm::Value *size)
+    CallLifetimeEnd(Address addr, llvm::Value *size)
         : Addr(addr.getPointer()), Size(size) {}
 
     void Emit(CodeGenFunction &CGF, Flags flags) override {
@@ -706,7 +684,7 @@ public:
   };
 
   /// i32s containing the indexes of the cleanup destinations.
-  RawAddress NormalCleanupDest = RawAddress::invalid();
+  Address NormalCleanupDest = Address::invalid();
 
   unsigned NextCleanupDestIndex = 1;
 
@@ -841,10 +819,10 @@ public:
   template <class T, class... As>
   void pushCleanupAfterFullExpr(CleanupKind Kind, As... A) {
     if (!isInConditionalBranch())
-      return pushCleanupAfterFullExprWithActiveFlag<T>(
-          Kind, RawAddress::invalid(), A...);
+      return pushCleanupAfterFullExprWithActiveFlag<T>(Kind, Address::invalid(),
+                                                       A...);
 
-    RawAddress ActiveFlag = createCleanupActiveFlag();
+    Address ActiveFlag = createCleanupActiveFlag();
     assert(!DominatingValue<Address>::needsSaving(ActiveFlag) &&
            "cleanup active flag should never need saving");
 
@@ -857,7 +835,7 @@ public:
 
   template <class T, class... As>
   void pushCleanupAfterFullExprWithActiveFlag(CleanupKind Kind,
-                                              RawAddress ActiveFlag, As... A) {
+                                              Address ActiveFlag, As... A) {
     LifetimeExtendedCleanupHeader Header = {sizeof(T), Kind,
                                             ActiveFlag.isValid()};
 
@@ -872,7 +850,7 @@ public:
     new (Buffer) LifetimeExtendedCleanupHeader(Header);
     new (Buffer + sizeof(Header)) T(A...);
     if (Header.IsConditional)
-      new (Buffer + sizeof(Header) + sizeof(T)) RawAddress(ActiveFlag);
+      new (Buffer + sizeof(Header) + sizeof(T)) Address(ActiveFlag);
   }
 
   /// Set up the last cleanup that was pushed as a conditional
@@ -881,8 +859,8 @@ public:
     initFullExprCleanupWithFlag(createCleanupActiveFlag());
   }
 
-  void initFullExprCleanupWithFlag(RawAddress ActiveFlag);
-  RawAddress createCleanupActiveFlag();
+  void initFullExprCleanupWithFlag(Address ActiveFlag);
+  Address createCleanupActiveFlag();
 
   /// PushDestructorCleanup - Push a cleanup to call the
   /// complete-object destructor of an object of the given type at the
@@ -1070,7 +1048,7 @@ public:
       QualType VarTy = LocalVD->getType();
       if (VarTy->isReferenceType()) {
         Address Temp = CGF.CreateMemTemp(VarTy);
-        CGF.Builder.CreateStore(TempAddr.emitRawPointer(CGF), Temp);
+        CGF.Builder.CreateStore(TempAddr.getPointer(), Temp);
         TempAddr = Temp;
       }
       SavedTempAddresses.try_emplace(LocalVD, TempAddr);
@@ -1265,12 +1243,10 @@ public:
   /// one branch or the other of a conditional expression.
   bool isInConditionalBranch() const { return OutermostConditional != nullptr; }
 
-  void setBeforeOutermostConditional(llvm::Value *value, Address addr,
-                                     CodeGenFunction &CGF) {
+  void setBeforeOutermostConditional(llvm::Value *value, Address addr) {
     assert(isInConditionalBranch());
     llvm::BasicBlock *block = OutermostConditional->getStartingBlock();
-    auto store =
-        new llvm::StoreInst(value, addr.emitRawPointer(CGF), &block->back());
+    auto store = new llvm::StoreInst(value, addr.getPointer(), &block->back());
     store->setAlignment(addr.getAlignment().getAsAlign());
   }
 
@@ -1625,7 +1601,7 @@ public:
   /// If \p StepV is null, the default increment is 1.
   void maybeUpdateMCDCTestVectorBitmap(const Expr *E) {
     if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) {
-      PGO.emitMCDCTestVectorBitmapUpdate(Builder, E, MCDCCondBitmapAddr, *this);
+      PGO.emitMCDCTestVectorBitmapUpdate(Builder, E, MCDCCondBitmapAddr);
       PGO.setCurrentStmt(E);
     }
   }
@@ -1633,7 +1609,7 @@ public:
   /// Update the MCDC temp value with the condition's evaluated result.
   void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val) {
     if (isMCDCCoverageEnabled()) {
-      PGO.emitMCDCCondBitmapUpdate(Builder, E, MCDCCondBitmapAddr, Val, *this);
+      PGO.emitMCDCCondBitmapUpdate(Builder, E, MCDCCondBitmapAddr, Val);
       PGO.setCurrentStmt(E);
     }
   }
@@ -1728,7 +1704,7 @@ public:
         : CGF(CGF), OldCXXThisValue(CGF.CXXThisValue),
           OldCXXThisAlignment(CGF.CXXThisAlignment),
           SourceLocScope(E, CGF.CurSourceLocExprScope) {
-      CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getBasePointer();
+      CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getPointer();
       CGF.CXXThisAlignment = CGF.CXXDefaultInitExprThis.getAlignment();
     }
     ~CXXDefaultInitExprScope() {
@@ -2114,7 +2090,7 @@ public:
   llvm::Value *getExceptionFromSlot();
   llvm::Value *getSelectorFromSlot();
 
-  RawAddress getNormalCleanupDestSlot();
+  Address getNormalCleanupDestSlot();
 
   llvm::BasicBlock *getUnreachableBlock() {
     if (!UnreachableBlock) {
@@ -2603,40 +2579,10 @@ public:
   //                                  Helpers
   //===--------------------------------------------------------------------===//
 
-  Address mergeAddressesInConditionalExpr(Address LHS, Address RHS,
-                                          llvm::BasicBlock *LHSBlock,
-                                          llvm::BasicBlock *RHSBlock,
-                                          llvm::BasicBlock *MergeBlock,
-                                          QualType MergedType) {
-    Builder.SetInsertPoint(MergeBlock);
-    llvm::PHINode *PtrPhi = Builder.CreatePHI(LHS.getType(), 2, "cond");
-    PtrPhi->addIncoming(LHS.getBasePointer(), LHSBlock);
-    PtrPhi->addIncoming(RHS.getBasePointer(), RHSBlock);
-    LHS.replaceBasePointer(PtrPhi);
-    LHS.setAlignment(std::min(LHS.getAlignment(), RHS.getAlignment()));
-    return LHS;
-  }
-
-  /// Construct an address with the natural alignment of T. If a pointer to T
-  /// is expected to be signed, the pointer passed to this function must have
-  /// been signed, and the returned Address will have the pointer authentication
-  /// information needed to authenticate the signed pointer.
-  Address makeNaturalAddressForPointer(
-      llvm::Value *Ptr, QualType T, CharUnits Alignment = CharUnits::Zero(),
-      bool ForPointeeType = false, LValueBaseInfo *BaseInfo = nullptr,
-      TBAAAccessInfo *TBAAInfo = nullptr,
-      KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {
-    if (Alignment.isZero())
-      Alignment =
-          CGM.getNaturalTypeAlignment(T, BaseInfo, TBAAInfo, ForPointeeType);
-    return Address(Ptr, ConvertTypeForMem(T), Alignment, nullptr,
-                   IsKnownNonNull);
-  }
-
   LValue MakeAddrLValue(Address Addr, QualType T,
                         AlignmentSource Source = AlignmentSource::Type) {
-    return MakeAddrLValue(Addr, T, LValueBaseInfo(Source),
-                          CGM.getTBAAAccessInfo(T));
+    return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source),
+                            CGM.getTBAAAccessInfo(T));
   }
 
   LValue MakeAddrLValue(Address Addr, QualType T, LValueBaseInfo BaseInfo,
@@ -2646,14 +2592,6 @@ public:
 
   LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
                         AlignmentSource Source = AlignmentSource::Type) {
-    return MakeAddrLValue(makeNaturalAddressForPointer(V, T, Alignment), T,
-                          LValueBaseInfo(Source), CGM.getTBAAAccessInfo(T));
-  }
-
-  /// Same as MakeAddrLValue above except that the pointer is known to be
-  /// unsigned.
-  LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,
-                           AlignmentSource Source = AlignmentSource::Type) {
     Address Addr(V, ConvertTypeForMem(T), Alignment);
     return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source),
                             CGM.getTBAAAccessInfo(T));
@@ -2666,18 +2604,9 @@ public:
                             TBAAAccessInfo());
   }
 
-  /// Given a value of type T* that may not be to a complete object, construct
-  /// an l-value with the natural pointee alignment of T.
   LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T);
-
   LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T);
 
-  /// Same as MakeNaturalAlignPointeeAddrLValue except that the pointer is known
-  /// to be unsigned.
-  LValue MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, QualType T);
-
-  LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T);
-
   Address EmitLoadOfReference(LValue RefLVal,
                               LValueBaseInfo *PointeeBaseInfo = nullptr,
                               TBAAAccessInfo *PointeeTBAAInfo = nullptr);
@@ -2726,13 +2655,13 @@ public:
   /// more efficient if the caller knows that the address will not be exposed.
   llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty, const Twine &Name = "tmp",
                                      llvm::Value *ArraySize = nullptr);
-  RawAddress CreateTempAlloca(llvm::Type *Ty, CharUnits align,
-                              const Twine &Name = "tmp",
-                              llvm::Value *ArraySize = nullptr,
-                              RawAddress *Alloca = nullptr);
-  RawAddress CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align,
-                                         const Twine &Name = "tmp",
-                                         llvm::Value *ArraySize = nullptr);
+  Address CreateTempAlloca(llvm::Type *Ty, CharUnits align,
+                           const Twine &Name = "tmp",
+                           llvm::Value *ArraySize = nullptr,
+                           Address *Alloca = nullptr);
+  Address CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align,
+                                      const Twine &Name = "tmp",
+                                      llvm::Value *ArraySize = nullptr);
 
   /// CreateDefaultAlignedTempAlloca - This creates an alloca with the
   /// default ABI alignment of the given LLVM type.
@@ -2744,8 +2673,8 @@ public:
   /// not hand this address off to arbitrary IRGen routines, and especially
   /// do not pass it as an argument to a function that might expect a
   /// properly ABI-aligned value.
-  RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty,
-                                          const Twine &Name = "tmp");
+  Address CreateDefaultAlignTempAlloca(llvm::Type *Ty,
+                                       const Twine &Name = "tmp");
 
   /// CreateIRTemp - Create a temporary IR object of the given type, with
   /// appropriate alignment. This routine should only be used when an temporary
@@ -2755,31 +2684,32 @@ public:
   ///
   /// That is, this is exactly equivalent to CreateMemTemp, but calling
   /// ConvertType instead of ConvertTypeForMem.
-  RawAddress CreateIRTemp(QualType T, const Twine &Name = "tmp");
+  Address CreateIRTemp(QualType T, const Twine &Name = "tmp");
 
   /// CreateMemTemp - Create a temporary memory object of the given type, with
   /// appropriate alignmen and cast it to the default address space. Returns
   /// the original alloca instruction by \p Alloca if it is not nullptr.
-  RawAddress CreateMemTemp(QualType T, const Twine &Name = "tmp",
-                           RawAddress *Alloca = nullptr);
-  RawAddress CreateMemTemp(QualType T, CharUnits Align,
-                           const Twine &Name = "tmp",
-                           RawAddress *Alloca = nullptr);
+  Address CreateMemTemp(QualType T, const Twine &Name = "tmp",
+                        Address *Alloca = nullptr);
+  Address CreateMemTemp(QualType T, CharUnits Align, const Twine &Name = "tmp",
+                        Address *Alloca = nullptr);
 
   /// CreateMemTemp - Create a temporary memory object of the given type, with
   /// appropriate alignmen without casting it to the default address space.
-  RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp");
-  RawAddress CreateMemTempWithoutCast(QualType T, CharUnits Align,
-                                      const Twine &Name = "tmp");
+  Address CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp");
+  Address CreateMemTempWithoutCast(QualType T, CharUnits Align,
+                                   const Twine &Name = "tmp");
 
   /// CreateAggTemp - Create a temporary memory object for the given
   /// aggregate type.
   AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp",
-                             RawAddress *Alloca = nullptr) {
-    return AggValueSlot::forAddr(
-        CreateMemTemp(T, Name, Alloca), T.getQualifiers(),
-        AggValueSlot::IsNotDestructed, AggValueSlot::DoesNotNeedGCBarriers,
-        AggValueSlot::IsNotAliased, AggValueSlot::DoesNotOverlap);
+                             Address *Alloca = nullptr) {
+    return AggValueSlot::forAddr(CreateMemTemp(T, Name, Alloca),
+                                 T.getQualifiers(),
+                                 AggValueSlot::IsNotDestructed,
+                                 AggValueSlot::DoesNotNeedGCBarriers,
+                                 AggValueSlot::IsNotAliased,
+                                 AggValueSlot::DoesNotOverlap);
   }
 
   /// EvaluateExprAsBool - Perform the usual unary conversions on the specified
@@ -3153,25 +3083,6 @@ public:
   /// calls to EmitTypeCheck can be skipped.
   bool sanitizePerformTypeCheck() const;
 
-  void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV,
-                     QualType Type, SanitizerSet SkippedChecks = SanitizerSet(),
-                     llvm::Value *ArraySize = nullptr) {
-    if (!sanitizePerformTypeCheck())
-      return;
-    EmitTypeCheck(TCK, Loc, LV.emitRawPointer(*this), Type, LV.getAlignment(),
-                  SkippedChecks, ArraySize);
-  }
-
-  void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, Address Addr,
-                     QualType Type, CharUnits Alignment = CharUnits::Zero(),
-                     SanitizerSet SkippedChecks = SanitizerSet(),
-                     llvm::Value *ArraySize = nullptr) {
-    if (!sanitizePerformTypeCheck())
-      return;
-    EmitTypeCheck(TCK, Loc, Addr.emitRawPointer(*this), Type, Alignment,
-                  SkippedChecks, ArraySize);
-  }
-
   /// Emit a check that \p V is the address of storage of the
   /// appropriate size and alignment for an object of type \p Type
   /// (or if ArraySize is provided, for an array of that bound).
@@ -3272,17 +3183,17 @@ public:
 
     /// Address with original alloca instruction. Invalid if the variable was
     /// emitted as a global constant.
-    RawAddress AllocaAddr;
+    Address AllocaAddr;
 
     struct Invalid {};
     AutoVarEmission(Invalid)
         : Variable(nullptr), Addr(Address::invalid()),
-          AllocaAddr(RawAddress::invalid()) {}
+          AllocaAddr(Address::invalid()) {}
 
     AutoVarEmission(const VarDecl &variable)
         : Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr),
           IsEscapingByRef(false), IsConstantAggregate(false),
-          SizeForLifetimeMarkers(nullptr), AllocaAddr(RawAddress::invalid()) {}
+          SizeForLifetimeMarkers(nullptr), AllocaAddr(Address::invalid()) {}
 
     bool wasEmittedAsGlobal() const { return !Addr.isValid(); }
 
@@ -3305,7 +3216,7 @@ public:
     }
 
     /// Returns the address for the original alloca instruction.
-    RawAddress getOriginalAllocatedAddress() const { return AllocaAddr; }
+    Address getOriginalAllocatedAddress() const { return AllocaAddr; }
 
     /// Returns the address of the object within this declaration.
     /// Note that this does not chase the forwarding pointer for
@@ -3335,32 +3246,23 @@ public:
                          llvm::GlobalValue::LinkageTypes Linkage);
 
   class ParamValue {
-    union {
-      Address Addr;
-      llvm::Value *Value;
-    };
-
-    bool IsIndirect;
-
-    ParamValue(llvm::Value *V) : Value(V), IsIndirect(false) {}
-    ParamValue(Address A) : Addr(A), IsIndirect(true) {}
-
+    llvm::Value *Value;
+    llvm::Type *ElementType;
+    unsigned Alignment;
+    ParamValue(llvm::Value *V, llvm::Type *T, unsigned A)
+        : Value(V), ElementType(T), Alignment(A) {}
   public:
     static ParamValue forDirect(llvm::Value *value) {
-      return ParamValue(value);
+      return ParamValue(value, nullptr, 0);
     }
     static ParamValue forIndirect(Address addr) {
       assert(!addr.getAlignment().isZero());
-      return ParamValue(addr);
+      return ParamValue(addr.getPointer(), addr.getElementType(),
+                        addr.getAlignment().getQuantity());
     }
 
-    bool isIndirect() const { return IsIndirect; }
-    llvm::Value *getAnyValue() const {
-      if (!isIndirect())
-        return Value;
-      assert(!Addr.hasOffset() && "unexpected offset");
-      return Addr.getBasePointer();
-    }
+    bool isIndirect() const { return Alignment != 0; }
+    llvm::Value *getAnyValue() const { return Value; }
 
     llvm::Value *getDirectValue() const {
       assert(!isIndirect());
@@ -3369,7 +3271,8 @@ public:
 
     Address getIndirectAddress() const {
       assert(isIndirect());
-      return Addr;
+      return Address(Value, ElementType, CharUnits::fromQuantity(Alignment),
+                     KnownNonNull);
     }
   };
 
@@ -4279,9 +4182,6 @@ public:
                                   const Twine &name = "");
   llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
                                           const Twine &name = "");
-  llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
-                                          ArrayRef<Address> args,
-                                          const Twine &name = "");
   llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
                                           ArrayRef<llvm::Value *> args,
                                           const Twine &name = "");
@@ -4308,12 +4208,6 @@ public:
                                                CXXDtorType Type,
                                                const CXXRecordDecl *RD);
 
-  llvm::Value *getAsNaturalPointerTo(Address Addr, QualType PointeeType) {
-    return Addr.getBasePointer();
-  }
-
-  bool isPointerKnownNonNull(const Expr *E);
-
   // Return the copy constructor name with the prefix "__copy_constructor_"
   // removed.
   static std::string getNonTrivialCopyConstructorStr(QualType QT,
@@ -4886,11 +4780,6 @@ public:
                                       SourceLocation Loc,
                                       const Twine &Name = "");
 
-  Address EmitCheckedInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList,
-                                 llvm::Type *elementType, bool SignedIndices,
-                                 bool IsSubtraction, SourceLocation Loc,
-                                 CharUnits Align, const Twine &Name = "");
-
   /// Specifies which type of sanitizer check to apply when handling a
   /// particular builtin.
   enum BuiltinCheckKind {
@@ -4953,10 +4842,6 @@ public:
   void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc,
                            AbstractCallee AC, unsigned ParmNum);
 
-  void EmitNonNullArgCheck(Address Addr, QualType ArgType,
-                           SourceLocation ArgLoc, AbstractCallee AC,
-                           unsigned ParmNum);
-
   /// EmitCallArg - Emit a single call argument.
   void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType);
 
@@ -5165,7 +5050,7 @@ DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) {
       CGF.CreateTempAlloca(value->getType(), align, "cond-cleanup.save");
   CGF.Builder.CreateStore(value, alloca);
 
-  return saved_type(alloca.emitRawPointer(CGF), true);
+  return saved_type(alloca.getPointer(), true);
 }
 
 inline llvm::Value *DominatingLLVMValue::restore(CodeGenFunction &CGF,
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 3997da17bb28..bc7d7ac56111 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -7267,7 +7267,7 @@ void CodeGenFunction::EmitDeclMetadata() {
 
   for (auto &I : LocalDeclMap) {
     const Decl *D = I.first;
-    llvm::Value *Addr = I.second.emitRawPointer(*this);
+    llvm::Value *Addr = I.second.getPointer();
     if (auto *Alloca = dyn_cast<llvm::AllocaInst>(Addr)) {
       llvm::Value *DAddr = GetPointerConstant(getLLVMContext(), D);
       Alloca->setMetadata(
diff --git a/clang/lib/CodeGen/CodeGenPGO.cpp b/clang/lib/CodeGen/CodeGenPGO.cpp
index 76704c4d7be4..2619edfeb7dc 100644
--- a/clang/lib/CodeGen/CodeGenPGO.cpp
+++ b/clang/lib/CodeGen/CodeGenPGO.cpp
@@ -1239,8 +1239,7 @@ void CodeGenPGO::emitMCDCParameters(CGBuilderTy &Builder) {
 
 void CodeGenPGO::emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder,
                                                 const Expr *S,
-                                                Address MCDCCondBitmapAddr,
-                                                CodeGenFunction &CGF) {
+                                                Address MCDCCondBitmapAddr) {
   if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState)
     return;
 
@@ -1263,7 +1262,7 @@ void CodeGenPGO::emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder,
                           Builder.getInt64(FunctionHash),
                           Builder.getInt32(RegionMCDCState->BitmapBytes),
                           Builder.getInt32(MCDCTestVectorBitmapOffset),
-                          MCDCCondBitmapAddr.emitRawPointer(CGF)};
+                          MCDCCondBitmapAddr.getPointer()};
   Builder.CreateCall(
       CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_tvbitmap_update), Args);
 }
@@ -1284,8 +1283,7 @@ void CodeGenPGO::emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S,
 
 void CodeGenPGO::emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S,
                                           Address MCDCCondBitmapAddr,
-                                          llvm::Value *Val,
-                                          CodeGenFunction &CGF) {
+                                          llvm::Value *Val) {
   if (!canEmitMCDCCoverage(Builder) || !RegionMCDCState)
     return;
 
@@ -1314,7 +1312,7 @@ void CodeGenPGO::emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S,
   llvm::Value *Args[5] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
                           Builder.getInt64(FunctionHash),
                           Builder.getInt32(Branch.ID),
-                          MCDCCondBitmapAddr.emitRawPointer(CGF), Val};
+                          MCDCCondBitmapAddr.getPointer(), Val};
   Builder.CreateCall(
       CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_condbitmap_update),
       Args);
diff --git a/clang/lib/CodeGen/CodeGenPGO.h b/clang/lib/CodeGen/CodeGenPGO.h
index 9d66ffad6f43..036fbf6815a4 100644
--- a/clang/lib/CodeGen/CodeGenPGO.h
+++ b/clang/lib/CodeGen/CodeGenPGO.h
@@ -113,14 +113,12 @@ public:
   void emitCounterSetOrIncrement(CGBuilderTy &Builder, const Stmt *S,
                                  llvm::Value *StepV);
   void emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, const Expr *S,
-                                      Address MCDCCondBitmapAddr,
-                                      CodeGenFunction &CGF);
+                                      Address MCDCCondBitmapAddr);
   void emitMCDCParameters(CGBuilderTy &Builder);
   void emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S,
                                Address MCDCCondBitmapAddr);
   void emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S,
-                                Address MCDCCondBitmapAddr, llvm::Value *Val,
-                                CodeGenFunction &CGF);
+                                Address MCDCCondBitmapAddr, llvm::Value *Val);
 
   /// Return the region count for the counter at the given index.
   uint64_t getRegionCount(const Stmt *S) {
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index fd71317572f0..bdd53a192f82 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -307,6 +307,10 @@ public:
       CodeGenFunction &CGF, const CXXRecordDecl *VTableClass,
       BaseSubobject Base, const CXXRecordDecl *NearestVBase);
 
+  llvm::Constant *
+  getVTableAddressPointForConstExpr(BaseSubobject Base,
+                                    const CXXRecordDecl *VTableClass) override;
+
   llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD,
                                         CharUnits VPtrOffset) override;
 
@@ -642,7 +646,7 @@ CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
 
   // Apply the adjustment and cast back to the original struct type
   // for consistency.
-  llvm::Value *This = ThisAddr.emitRawPointer(CGF);
+  llvm::Value *This = ThisAddr.getPointer();
   This = Builder.CreateInBoundsGEP(Builder.getInt8Ty(), This, Adj);
   ThisPtrForCall = This;
 
@@ -846,7 +850,7 @@ llvm::Value *ItaniumCXXABI::EmitMemberDataPointerAddress(
   CGBuilderTy &Builder = CGF.Builder;
 
   // Apply the offset, which we assume is non-null.
-  return Builder.CreateInBoundsGEP(CGF.Int8Ty, Base.emitRawPointer(CGF), MemPtr,
+  return Builder.CreateInBoundsGEP(CGF.Int8Ty, Base.getPointer(), MemPtr,
                                    "memptr.offset");
 }
 
@@ -1241,7 +1245,7 @@ void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF,
                                                         CGF.getPointerAlign());
 
     // Apply the offset.
-    llvm::Value *CompletePtr = Ptr.emitRawPointer(CGF);
+    llvm::Value *CompletePtr = Ptr.getPointer();
     CompletePtr =
         CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, CompletePtr, Offset);
 
@@ -1478,8 +1482,7 @@ llvm::Value *ItaniumCXXABI::emitDynamicCastCall(
       computeOffsetHint(CGF.getContext(), SrcDecl, DestDecl).getQuantity());
 
   // Emit the call to __dynamic_cast.
-  llvm::Value *Args[] = {ThisAddr.emitRawPointer(CGF), SrcRTTI, DestRTTI,
-                         OffsetHint};
+  llvm::Value *Args[] = {ThisAddr.getPointer(), SrcRTTI, DestRTTI, OffsetHint};
   llvm::Value *Value =
       CGF.EmitNounwindRuntimeCall(getItaniumDynamicCastFn(CGF), Args);
 
@@ -1568,7 +1571,7 @@ llvm::Value *ItaniumCXXABI::emitExactDynamicCast(
       VPtr, CGM.getTBAAVTablePtrAccessInfo(CGF.VoidPtrPtrTy));
   llvm::Value *Success = CGF.Builder.CreateICmpEQ(
       VPtr, getVTableAddressPoint(BaseSubobject(SrcDecl, *Offset), DestDecl));
-  llvm::Value *Result = ThisAddr.emitRawPointer(CGF);
+  llvm::Value *Result = ThisAddr.getPointer();
   if (!Offset->isZero())
     Result = CGF.Builder.CreateInBoundsGEP(
         CGF.CharTy, Result,
@@ -1608,7 +1611,7 @@ llvm::Value *ItaniumCXXABI::emitDynamicCastToVoid(CodeGenFunction &CGF,
         PtrDiffLTy, OffsetToTop, CGF.getPointerAlign(), "offset.to.top");
   }
   // Finally, add the offset to the pointer.
-  return CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ThisAddr.emitRawPointer(CGF),
+  return CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ThisAddr.getPointer(),
                                        OffsetToTop);
 }
 
@@ -1789,8 +1792,8 @@ void ItaniumCXXABI::EmitDestructorCall(CodeGenFunction &CGF,
   else
     Callee = CGCallee::forDirect(CGM.getAddrOfCXXStructor(GD), GD);
 
-  CGF.EmitCXXDestructorCall(GD, Callee, CGF.getAsNaturalPointerTo(This, ThisTy),
-                            ThisTy, VTT, VTTTy, nullptr);
+  CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, VTT, VTTTy,
+                            nullptr);
 }
 
 void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT,
@@ -1949,6 +1952,11 @@ llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructorWithVTT(
                                        CGF.getPointerAlign());
 }
 
+llvm::Constant *ItaniumCXXABI::getVTableAddressPointForConstExpr(
+    BaseSubobject Base, const CXXRecordDecl *VTableClass) {
+  return getVTableAddressPoint(Base, VTableClass);
+}
+
 llvm::GlobalVariable *ItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *RD,
                                                      CharUnits VPtrOffset) {
   assert(VPtrOffset.isZero() && "Itanium ABI only supports zero vptr offsets");
@@ -2080,8 +2088,8 @@ llvm::Value *ItaniumCXXABI::EmitVirtualDestructorCall(
     ThisTy = D->getDestroyedType();
   }
 
-  CGF.EmitCXXDestructorCall(GD, Callee, This.emitRawPointer(CGF), ThisTy,
-                            nullptr, QualType(), nullptr);
+  CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy, nullptr,
+                            QualType(), nullptr);
   return nullptr;
 }
 
@@ -2154,7 +2162,7 @@ static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF,
                                           int64_t VirtualAdjustment,
                                           bool IsReturnAdjustment) {
   if (!NonVirtualAdjustment && !VirtualAdjustment)
-    return InitialPtr.emitRawPointer(CGF);
+    return InitialPtr.getPointer();
 
   Address V = InitialPtr.withElementType(CGF.Int8Ty);
 
@@ -2187,10 +2195,10 @@ static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF,
                                              CGF.getPointerAlign());
     }
     // Adjust our pointer.
-    ResultPtr = CGF.Builder.CreateInBoundsGEP(V.getElementType(),
-                                              V.emitRawPointer(CGF), Offset);
+    ResultPtr = CGF.Builder.CreateInBoundsGEP(
+        V.getElementType(), V.getPointer(), Offset);
   } else {
-    ResultPtr = V.emitRawPointer(CGF);
+    ResultPtr = V.getPointer();
   }
 
   // In a derived-to-base conversion, the non-virtual adjustment is
@@ -2276,7 +2284,7 @@ Address ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
         llvm::FunctionType::get(CGM.VoidTy, NumElementsPtr.getType(), false);
     llvm::FunctionCallee F =
         CGM.CreateRuntimeFunction(FTy, "__asan_poison_cxx_array_cookie");
-    CGF.Builder.CreateCall(F, NumElementsPtr.emitRawPointer(CGF));
+    CGF.Builder.CreateCall(F, NumElementsPtr.getPointer());
   }
 
   // Finally, compute a pointer to the actual data buffer by skipping
@@ -2307,7 +2315,7 @@ llvm::Value *ItaniumCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
       llvm::FunctionType::get(CGF.SizeTy, CGF.UnqualPtrTy, false);
   llvm::FunctionCallee F =
       CGM.CreateRuntimeFunction(FTy, "__asan_load_cxx_array_cookie");
-  return CGF.Builder.CreateCall(F, numElementsPtr.emitRawPointer(CGF));
+  return CGF.Builder.CreateCall(F, numElementsPtr.getPointer());
 }
 
 CharUnits ARMCXXABI::getArrayCookieSizeImpl(QualType elementType) {
@@ -2619,7 +2627,7 @@ void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF,
 
     // Call __cxa_guard_release.  This cannot throw.
     CGF.EmitNounwindRuntimeCall(getGuardReleaseFn(CGM, guardPtrTy),
-                                guardAddr.emitRawPointer(CGF));
+                                guardAddr.getPointer());
   } else if (D.isLocalVarDecl()) {
     // For local variables, store 1 into the first byte of the guard variable
     // after the object initialization completes so that initialization is
@@ -3112,10 +3120,10 @@ LValue ItaniumCXXABI::EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF,
 
   LValue LV;
   if (VD->getType()->isReferenceType())
-    LV = CGF.MakeNaturalAlignRawAddrLValue(CallVal, LValType);
+    LV = CGF.MakeNaturalAlignAddrLValue(CallVal, LValType);
   else
-    LV = CGF.MakeRawAddrLValue(CallVal, LValType,
-                               CGF.getContext().getDeclAlign(VD));
+    LV = CGF.MakeAddrLValue(CallVal, LValType,
+                            CGF.getContext().getDeclAlign(VD));
   // FIXME: need setObjCGCLValueClass?
   return LV;
 }
@@ -4596,7 +4604,7 @@ static void InitCatchParam(CodeGenFunction &CGF,
         CGF.Builder.CreateStore(Casted, ExnPtrTmp);
 
         // Bind the reference to the temporary.
-        AdjustedExn = ExnPtrTmp.emitRawPointer(CGF);
+        AdjustedExn = ExnPtrTmp.getPointer();
       }
     }
 
diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
index d38a26940a3c..172c4c937b97 100644
--- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -327,6 +327,10 @@ public:
       CodeGenFunction &CGF, const CXXRecordDecl *VTableClass,
       BaseSubobject Base, const CXXRecordDecl *NearestVBase) override;
 
+  llvm::Constant *
+  getVTableAddressPointForConstExpr(BaseSubobject Base,
+                                    const CXXRecordDecl *VTableClass) override;
+
   llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD,
                                         CharUnits VPtrOffset) override;
 
@@ -933,7 +937,7 @@ void MicrosoftCXXABI::emitBeginCatch(CodeGenFunction &CGF,
   }
 
   CodeGenFunction::AutoVarEmission var = CGF.EmitAutoVarAlloca(*CatchParam);
-  CPI->setArgOperand(2, var.getObjectAddress(CGF).emitRawPointer(CGF));
+  CPI->setArgOperand(2, var.getObjectAddress(CGF).getPointer());
   CGF.EHStack.pushCleanup<CatchRetScope>(NormalCleanup, CPI);
   CGF.EmitAutoVarCleanups(var);
 }
@@ -970,7 +974,7 @@ MicrosoftCXXABI::performBaseAdjustment(CodeGenFunction &CGF, Address Value,
   llvm::Value *Offset =
     GetVirtualBaseClassOffset(CGF, Value, SrcDecl, PolymorphicBase);
   llvm::Value *Ptr = CGF.Builder.CreateInBoundsGEP(
-      Value.getElementType(), Value.emitRawPointer(CGF), Offset);
+      Value.getElementType(), Value.getPointer(), Offset);
   CharUnits VBaseAlign =
     CGF.CGM.getVBaseAlignment(Value.getAlignment(), SrcDecl, PolymorphicBase);
   return std::make_tuple(Address(Ptr, CGF.Int8Ty, VBaseAlign), Offset,
@@ -1007,7 +1011,7 @@ llvm::Value *MicrosoftCXXABI::EmitTypeid(CodeGenFunction &CGF,
                                          llvm::Type *StdTypeInfoPtrTy) {
   std::tie(ThisPtr, std::ignore, std::ignore) =
       performBaseAdjustment(CGF, ThisPtr, SrcRecordTy);
-  llvm::CallBase *Typeid = emitRTtypeidCall(CGF, ThisPtr.emitRawPointer(CGF));
+  llvm::CallBase *Typeid = emitRTtypeidCall(CGF, ThisPtr.getPointer());
   return CGF.Builder.CreateBitCast(Typeid, StdTypeInfoPtrTy);
 }
 
@@ -1029,7 +1033,7 @@ llvm::Value *MicrosoftCXXABI::emitDynamicCastCall(
   llvm::Value *Offset;
   std::tie(This, Offset, std::ignore) =
       performBaseAdjustment(CGF, This, SrcRecordTy);
-  llvm::Value *ThisPtr = This.emitRawPointer(CGF);
+  llvm::Value *ThisPtr = This.getPointer();
   Offset = CGF.Builder.CreateTrunc(Offset, CGF.Int32Ty);
 
   // PVOID __RTDynamicCast(
@@ -1061,7 +1065,7 @@ llvm::Value *MicrosoftCXXABI::emitDynamicCastToVoid(CodeGenFunction &CGF,
   llvm::FunctionCallee Function = CGF.CGM.CreateRuntimeFunction(
       llvm::FunctionType::get(CGF.Int8PtrTy, ArgTypes, false),
       "__RTCastToVoid");
-  llvm::Value *Args[] = {Value.emitRawPointer(CGF)};
+  llvm::Value *Args[] = {Value.getPointer()};
   return CGF.EmitRuntimeCall(Function, Args);
 }
 
@@ -1489,7 +1493,7 @@ Address MicrosoftCXXABI::adjustThisArgumentForVirtualFunctionCall(
     llvm::Value *VBaseOffset =
       GetVirtualBaseClassOffset(CGF, Result, Derived, VBase);
     llvm::Value *VBasePtr = CGF.Builder.CreateInBoundsGEP(
-        Result.getElementType(), Result.emitRawPointer(CGF), VBaseOffset);
+        Result.getElementType(), Result.getPointer(), VBaseOffset);
     CharUnits VBaseAlign =
       CGF.CGM.getVBaseAlignment(Result.getAlignment(), Derived, VBase);
     Result = Address(VBasePtr, CGF.Int8Ty, VBaseAlign);
@@ -1656,8 +1660,7 @@ void MicrosoftCXXABI::EmitDestructorCall(CodeGenFunction &CGF,
   llvm::Value *Implicit =
       getCXXDestructorImplicitParam(CGF, DD, Type, ForVirtualBase,
                                     Delegating); // = nullptr
-  CGF.EmitCXXDestructorCall(GD, Callee, CGF.getAsNaturalPointerTo(This, ThisTy),
-                            ThisTy,
+  CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy,
                             /*ImplicitParam=*/Implicit,
                             /*ImplicitParamTy=*/QualType(), nullptr);
   if (BaseDtorEndBB) {
@@ -1788,6 +1791,13 @@ MicrosoftCXXABI::getVTableAddressPoint(BaseSubobject Base,
   return VFTablesMap[ID];
 }
 
+llvm::Constant *MicrosoftCXXABI::getVTableAddressPointForConstExpr(
+    BaseSubobject Base, const CXXRecordDecl *VTableClass) {
+  llvm::Constant *VFTable = getVTableAddressPoint(Base, VTableClass);
+  assert(VFTable && "Couldn't find a vftable for the given base?");
+  return VFTable;
+}
+
 llvm::GlobalVariable *MicrosoftCXXABI::getAddrOfVTable(const CXXRecordDecl *RD,
                                                        CharUnits VPtrOffset) {
   // getAddrOfVTable may return 0 if asked to get an address of a vtable which
@@ -2003,9 +2013,8 @@ llvm::Value *MicrosoftCXXABI::EmitVirtualDestructorCall(
   }
 
   This = adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, true);
-  RValue RV =
-      CGF.EmitCXXDestructorCall(GD, Callee, This.emitRawPointer(CGF), ThisTy,
-                                ImplicitParam, Context.IntTy, CE);
+  RValue RV = CGF.EmitCXXDestructorCall(GD, Callee, This.getPointer(), ThisTy,
+                                        ImplicitParam, Context.IntTy, CE);
   return RV.getScalarVal();
 }
 
@@ -2203,13 +2212,13 @@ llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF,
                                                     Address This,
                                                     const ThisAdjustment &TA) {
   if (TA.isEmpty())
-    return This.emitRawPointer(CGF);
+    return This.getPointer();
 
   This = This.withElementType(CGF.Int8Ty);
 
   llvm::Value *V;
   if (TA.Virtual.isEmpty()) {
-    V = This.emitRawPointer(CGF);
+    V = This.getPointer();
   } else {
     assert(TA.Virtual.Microsoft.VtordispOffset < 0);
     // Adjust the this argument based on the vtordisp value.
@@ -2218,7 +2227,7 @@ llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF,
                  CharUnits::fromQuantity(TA.Virtual.Microsoft.VtordispOffset));
     VtorDispPtr = VtorDispPtr.withElementType(CGF.Int32Ty);
     llvm::Value *VtorDisp = CGF.Builder.CreateLoad(VtorDispPtr, "vtordisp");
-    V = CGF.Builder.CreateGEP(This.getElementType(), This.emitRawPointer(CGF),
+    V = CGF.Builder.CreateGEP(This.getElementType(), This.getPointer(),
                               CGF.Builder.CreateNeg(VtorDisp));
 
     // Unfortunately, having applied the vtordisp means that we no
@@ -2255,11 +2264,11 @@ llvm::Value *
 MicrosoftCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret,
                                          const ReturnAdjustment &RA) {
   if (RA.isEmpty())
-    return Ret.emitRawPointer(CGF);
+    return Ret.getPointer();
 
   Ret = Ret.withElementType(CGF.Int8Ty);
 
-  llvm::Value *V = Ret.emitRawPointer(CGF);
+  llvm::Value *V = Ret.getPointer();
   if (RA.Virtual.Microsoft.VBIndex) {
     assert(RA.Virtual.Microsoft.VBIndex > 0);
     int32_t IntSize = CGF.getIntSize().getQuantity();
@@ -2574,7 +2583,7 @@ struct ResetGuardBit final : EHScopeStack::Cleanup {
 
 struct CallInitThreadAbort final : EHScopeStack::Cleanup {
   llvm::Value *Guard;
-  CallInitThreadAbort(RawAddress Guard) : Guard(Guard.getPointer()) {}
+  CallInitThreadAbort(Address Guard) : Guard(Guard.getPointer()) {}
 
   void Emit(CodeGenFunction &CGF, Flags flags) override {
     // Calling _Init_thread_abort will reset the guard's state.
@@ -3114,8 +3123,8 @@ MicrosoftCXXABI::GetVBaseOffsetFromVBPtr(CodeGenFunction &CGF,
                                          llvm::Value **VBPtrOut) {
   CGBuilderTy &Builder = CGF.Builder;
   // Load the vbtable pointer from the vbptr in the instance.
-  llvm::Value *VBPtr = Builder.CreateInBoundsGEP(
-      CGM.Int8Ty, This.emitRawPointer(CGF), VBPtrOffset, "vbptr");
+  llvm::Value *VBPtr = Builder.CreateInBoundsGEP(CGM.Int8Ty, This.getPointer(),
+                                                 VBPtrOffset, "vbptr");
   if (VBPtrOut)
     *VBPtrOut = VBPtr;
 
@@ -3194,7 +3203,7 @@ llvm::Value *MicrosoftCXXABI::AdjustVirtualBase(
     Builder.CreateBr(SkipAdjustBB);
     CGF.EmitBlock(SkipAdjustBB);
     llvm::PHINode *Phi = Builder.CreatePHI(CGM.Int8PtrTy, 2, "memptr.base");
-    Phi->addIncoming(Base.emitRawPointer(CGF), OriginalBB);
+    Phi->addIncoming(Base.getPointer(), OriginalBB);
     Phi->addIncoming(AdjustedBase, VBaseAdjustBB);
     return Phi;
   }
@@ -3229,7 +3238,7 @@ llvm::Value *MicrosoftCXXABI::EmitMemberDataPointerAddress(
     Addr = AdjustVirtualBase(CGF, E, RD, Base, VirtualBaseAdjustmentOffset,
                              VBPtrOffset);
   } else {
-    Addr = Base.emitRawPointer(CGF);
+    Addr = Base.getPointer();
   }
 
   // Apply the offset, which we assume is non-null.
@@ -3517,7 +3526,7 @@ CGCallee MicrosoftCXXABI::EmitLoadOfMemberFunctionPointer(
     ThisPtrForCall = AdjustVirtualBase(CGF, E, RD, This,
                                    VirtualBaseAdjustmentOffset, VBPtrOffset);
   } else {
-    ThisPtrForCall = This.emitRawPointer(CGF);
+    ThisPtrForCall = This.getPointer();
   }
 
   if (NonVirtualBaseAdjustment)
@@ -4436,7 +4445,10 @@ void MicrosoftCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) {
   llvm::GlobalVariable *TI = getThrowInfo(ThrowType);
 
   // Call into the runtime to throw the exception.
-  llvm::Value *Args[] = {AI.emitRawPointer(CGF), TI};
+  llvm::Value *Args[] = {
+    AI.getPointer(),
+    TI
+  };
   CGF.EmitNoreturnRuntimeCallOrInvoke(getThrowFn(), Args);
 }
 
diff --git a/clang/lib/CodeGen/TargetInfo.h b/clang/lib/CodeGen/TargetInfo.h
index b1dfe5bf8f27..6893b50a3cfe 100644
--- a/clang/lib/CodeGen/TargetInfo.h
+++ b/clang/lib/CodeGen/TargetInfo.h
@@ -295,11 +295,6 @@ public:
   /// Get the AST address space for alloca.
   virtual LangAS getASTAllocaAddressSpace() const { return LangAS::Default; }
 
-  Address performAddrSpaceCast(CodeGen::CodeGenFunction &CGF, Address Addr,
-                               LangAS SrcAddr, LangAS DestAddr,
-                               llvm::Type *DestTy,
-                               bool IsNonNull = false) const;
-
   /// Perform address space cast of an expression of pointer type.
   /// \param V is the LLVM value to be casted to another address space.
   /// \param SrcAddr is the language address space of \p V.
diff --git a/clang/lib/CodeGen/Targets/NVPTX.cpp b/clang/lib/CodeGen/Targets/NVPTX.cpp
index 7dce5042c3dc..8718f1ecf3a7 100644
--- a/clang/lib/CodeGen/Targets/NVPTX.cpp
+++ b/clang/lib/CodeGen/Targets/NVPTX.cpp
@@ -85,7 +85,7 @@ private:
                                            LValue Src) {
     llvm::Value *Handle = nullptr;
     llvm::Constant *C =
-        llvm::dyn_cast<llvm::Constant>(Src.getAddress(CGF).emitRawPointer(CGF));
+        llvm::dyn_cast<llvm::Constant>(Src.getAddress(CGF).getPointer());
     // Lookup `addrspacecast` through the constant pointer if any.
     if (auto *ASC = llvm::dyn_cast_or_null<llvm::AddrSpaceCastOperator>(C))
       C = llvm::cast<llvm::Constant>(ASC->getPointerOperand());
diff --git a/clang/lib/CodeGen/Targets/PPC.cpp b/clang/lib/CodeGen/Targets/PPC.cpp
index 362add30b435..00b04723f17d 100644
--- a/clang/lib/CodeGen/Targets/PPC.cpp
+++ b/clang/lib/CodeGen/Targets/PPC.cpp
@@ -513,10 +513,9 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
     CharUnits RegSize = CharUnits::fromQuantity((isInt || IsSoftFloatABI) ? 4 : 8);
     llvm::Value *RegOffset =
         Builder.CreateMul(NumRegs, Builder.getInt8(RegSize.getQuantity()));
-    RegAddr = Address(Builder.CreateInBoundsGEP(
-                          CGF.Int8Ty, RegAddr.emitRawPointer(CGF), RegOffset),
-                      DirectTy,
-                      RegAddr.getAlignment().alignmentOfArrayElement(RegSize));
+    RegAddr = Address(
+        Builder.CreateInBoundsGEP(CGF.Int8Ty, RegAddr.getPointer(), RegOffset),
+        DirectTy, RegAddr.getAlignment().alignmentOfArrayElement(RegSize));
 
     // Increase the used-register count.
     NumRegs =
@@ -552,7 +551,7 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
     // Round up address of argument to alignment
     CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
     if (Align > OverflowAreaAlign) {
-      llvm::Value *Ptr = OverflowArea.emitRawPointer(CGF);
+      llvm::Value *Ptr = OverflowArea.getPointer();
       OverflowArea = Address(emitRoundPointerUpToAlignment(CGF, Ptr, Align),
                              OverflowArea.getElementType(), Align);
     }
@@ -561,7 +560,7 @@ Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
 
     // Increase the overflow area.
     OverflowArea = Builder.CreateConstInBoundsByteGEP(OverflowArea, Size);
-    Builder.CreateStore(OverflowArea.emitRawPointer(CGF), OverflowAreaAddr);
+    Builder.CreateStore(OverflowArea.getPointer(), OverflowAreaAddr);
     CGF.EmitBranch(Cont);
   }
 
diff --git a/clang/lib/CodeGen/Targets/Sparc.cpp b/clang/lib/CodeGen/Targets/Sparc.cpp
index 9025a633f328..a337a52a94ec 100644
--- a/clang/lib/CodeGen/Targets/Sparc.cpp
+++ b/clang/lib/CodeGen/Targets/Sparc.cpp
@@ -326,7 +326,7 @@ Address SparcV9ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
 
   // Update VAList.
   Address NextPtr = Builder.CreateConstInBoundsByteGEP(Addr, Stride, "ap.next");
-  Builder.CreateStore(NextPtr.emitRawPointer(CGF), VAListAddr);
+  Builder.CreateStore(NextPtr.getPointer(), VAListAddr);
 
   return ArgAddr.withElementType(ArgTy);
 }
diff --git a/clang/lib/CodeGen/Targets/SystemZ.cpp b/clang/lib/CodeGen/Targets/SystemZ.cpp
index deaafc85a315..6eb0c6ef2f7d 100644
--- a/clang/lib/CodeGen/Targets/SystemZ.cpp
+++ b/clang/lib/CodeGen/Targets/SystemZ.cpp
@@ -306,7 +306,7 @@ Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
 
     // Update overflow_arg_area_ptr pointer
     llvm::Value *NewOverflowArgArea = CGF.Builder.CreateGEP(
-        OverflowArgArea.getElementType(), OverflowArgArea.emitRawPointer(CGF),
+        OverflowArgArea.getElementType(), OverflowArgArea.getPointer(),
         PaddedSizeV, "overflow_arg_area");
     CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr);
 
@@ -382,9 +382,10 @@ Address SystemZABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
   Address MemAddr = RawMemAddr.withElementType(DirectTy);
 
   // Update overflow_arg_area_ptr pointer
-  llvm::Value *NewOverflowArgArea = CGF.Builder.CreateGEP(
-      OverflowArgArea.getElementType(), OverflowArgArea.emitRawPointer(CGF),
-      PaddedSizeV, "overflow_arg_area");
+  llvm::Value *NewOverflowArgArea =
+    CGF.Builder.CreateGEP(OverflowArgArea.getElementType(),
+                          OverflowArgArea.getPointer(), PaddedSizeV,
+                          "overflow_arg_area");
   CGF.Builder.CreateStore(NewOverflowArgArea, OverflowArgAreaPtr);
   CGF.EmitBranch(ContBlock);
 
diff --git a/clang/lib/CodeGen/Targets/XCore.cpp b/clang/lib/CodeGen/Targets/XCore.cpp
index 88edb781a947..aeb48f851e16 100644
--- a/clang/lib/CodeGen/Targets/XCore.cpp
+++ b/clang/lib/CodeGen/Targets/XCore.cpp
@@ -180,7 +180,7 @@ Address XCoreABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
   // Increment the VAList.
   if (!ArgSize.isZero()) {
     Address APN = Builder.CreateConstInBoundsByteGEP(AP, ArgSize);
-    Builder.CreateStore(APN.emitRawPointer(CGF), VAListAddr);
+    Builder.CreateStore(APN.getPointer(), VAListAddr);
   }
 
   return Val;
diff --git a/clang/utils/TableGen/MveEmitter.cpp b/clang/utils/TableGen/MveEmitter.cpp
index aa20c758d84a..3a90eee5f1c9 100644
--- a/clang/utils/TableGen/MveEmitter.cpp
+++ b/clang/utils/TableGen/MveEmitter.cpp
@@ -575,7 +575,7 @@ public:
   // Emit code to generate this result as a Value *.
   std::string asValue() override {
     if (AddressType)
-      return "(" + varname() + ".emitRawPointer(*this))";
+      return "(" + varname() + ".getPointer())";
     return Result::asValue();
   }
   bool hasIntegerValue() const override { return Immediate; }
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index 7d98009ec435..a6165ef13fd7 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -2702,7 +2702,6 @@ public:
   IRBuilder(const IRBuilder &) = delete;
 
   InserterTy &getInserter() { return Inserter; }
-  const InserterTy &getInserter() const { return Inserter; }
 };
 
 template <typename FolderTy, typename InserterTy>
-- 
GitLab


From 507e59aa757bdc690ebc0bf69c6d3c84061061a8 Mon Sep 17 00:00:00 2001
From: Julian Nagele <mail@jnagele.net>
Date: Tue, 26 Mar 2024 14:42:08 +0000
Subject: [PATCH 284/404] [TBAA] Tests for invalid tbaa.struct metadata
 (#86167)

These tests show invalid tbaa.struct metadata that is currently accepted
in preparation for a change to the IR Verifier that will then reject it.

PR: https://github.com/llvm/llvm-project/pull/86167
---
 llvm/test/Verifier/tbaa-struct.ll | 40 +++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 llvm/test/Verifier/tbaa-struct.ll

diff --git a/llvm/test/Verifier/tbaa-struct.ll b/llvm/test/Verifier/tbaa-struct.ll
new file mode 100644
index 000000000000..b8ddc7cee496
--- /dev/null
+++ b/llvm/test/Verifier/tbaa-struct.ll
@@ -0,0 +1,40 @@
+; RUN: llvm-as < %s 2>&1
+
+; FIXME: The verifer should reject the invalid !tbaa.struct nodes below.
+
+define void @test_overlapping_regions(ptr %a1) {
+  %ld = load i8, ptr %a1, align 1, !tbaa.struct !0
+  ret void
+}
+
+define void @test_size_not_integer(ptr %a1) {
+  store i8 1, ptr %a1, align 1, !tbaa.struct !5
+  ret void
+}
+
+define void @test_offset_not_integer(ptr %a1, ptr %a2) {
+  tail call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a1, ptr align 8 %a2, i64 16, i1 false), !tbaa.struct !6
+  ret void
+}
+
+define void @test_tbaa_missing(ptr %a1, ptr %a2) {
+  tail call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a1, ptr align 8 %a2, i64 16, i1 false), !tbaa.struct !7
+  ret void
+}
+
+define void @test_tbaa_invalid(ptr %a1) {
+  store i8 1, ptr %a1, align 1, !tbaa.struct !8
+  ret void
+}
+
+declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1) nounwind
+
+!0 = !{i64 0, i64 4, !1, i64 1, i64 4, !1}
+!1 = !{!2, !2, i64 0}
+!2 = !{!"int", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C++ TBAA"}
+!5 = !{i64 0, !2, !1}
+!6 = !{!2, i64 0, !1}
+!7 = !{i64 0, i64 4, null}
+!8 = !{i64 0, i64 4, !2}
-- 
GitLab


From 26d896f3688a8bff6faf85ccce557e320108997f Mon Sep 17 00:00:00 2001
From: Rafael Ubal <rubal@mathworks.com>
Date: Tue, 26 Mar 2024 10:52:55 -0400
Subject: [PATCH 285/404] Fixes in 'tosa.reshape' lowering and folder (#85798)

- Revamped lowering conversion pattern for `tosa.reshape` to handle previously unsupported combinations of dynamic dimensions in input and output tensors. The lowering strategy continues to rely on pairs `tensor.collapse_shape` + `tensor.expand_shape`, which allow for downstream fusion with surrounding `linalg.generic` ops.

- Fixed bug in canonicalization pattern `ReshapeOp::fold()` in `TosaCanonicalizations.cpp`. The input and result types being equal is not a sufficient condition for folding. If there is more than 1 dynamic dimension in the input and result types, a productive reshape could still occur.

- This work exposed the fact that bufferization does not properly handle a `tensor.collapse_shape` op producing a 0D tensor from a dynamically shaped one due to a limitation in `memref.collapse_shape`. While the proper way to address this would involve releasing the `memref.collapse_shape` restriction and verifying correct bufferization, this is left as possible future work. For now, this scenario is avoided by casting the `tosa.reshape` input tensor to a static shape if necessary (see `inferReshapeInputType()`.

- An extended set of tests are intended to cover relevant conversion paths. Tests are named using pattern `test_reshape_<rank>_{up|down|same}_{s2s|s2d|d2s|d2d}_{explicit|auto}[_empty][_identity]`, where:

  - `<rank>` is the input rank (e.g., 3d, 6d)
  - `{up|down|same}` indicates whether the reshape increases, decreases, or retains the input rank.
  - `{s2s|s2d|d2s|d2d}` indicates whether reshape converts a statically shaped input to a statically shaped result (`s2s`), a statically shaped input to a dynamically shaped result (`s2d`), etc.
  - `{explicit|auto}` is used to indicate that all values in the `new_shape` attribute are >=0 (`explicit`) or that a -1 placeholder value is used (`auto`).
  - `empty` is used to indicate that `new_shape` includes a component set to 0.
  - `identity` is used when the input and result shapes are the same.
---
 .../Conversion/TosaToTensor/TosaToTensor.cpp  | 304 ++++++++-------
 .../Dialect/Tosa/IR/TosaCanonicalizations.cpp |   5 +-
 mlir/lib/Dialect/Tosa/IR/TosaOps.cpp          |   5 +
 .../TosaToTensor/tosa-to-tensor.mlir          | 368 +++++++++++++++---
 mlir/test/Dialect/Tosa/canonicalize.mlir      |   8 +
 5 files changed, 495 insertions(+), 195 deletions(-)

diff --git a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
index 505d85f21111..11ba98ddf352 100644
--- a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
+++ b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
@@ -19,24 +19,99 @@
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
 
+#include <numeric>
+
 using namespace mlir;
 using namespace tosa;
 
-static bool findIntermediateShape(ArrayRef<int64_t> lhsShape,
-                                  ArrayRef<int64_t> rhsShape,
-                                  SmallVector<int64_t> &intermediateShape,
-                                  bool isDynamic) {
-  if (isDynamic) {
-    // TODO (natashaknk): Make dynamic intermediate shape not always be rank-1
-    intermediateShape = {ShapedType::kDynamic};
-    return true;
-  }
+namespace {
 
-  if (lhsShape.empty() || rhsShape.empty()) {
-    intermediateShape = {};
-    return true;
-  }
+// Infer the type to which the input of a 'tosa.reshape' op must be cast when
+// lowered.
+TensorType inferReshapeInputType(TypedValue<TensorType> input,
+                                 ArrayRef<int64_t> newShape) {
+  // No need to cast input for non-empty target shape
+  if (!newShape.empty())
+    return input.getType();
+
+  // The input type must be cast into a tensor with the same rank and all static
+  // dimensions set to 1. This prevents the generation of a tensor.collapse_shape
+  // op that converts a dynamically shaped tensor into a 0D tensor. While such
+  // construct is not incorrect on its own, bufferization cannot properly handle
+  // it at the moment, so we avoid it.
+  SmallVector<int64_t> shape(input.getType().getRank(), 1);
+  return input.getType().clone(shape);
+}
+
+// Infer the result type of 'tensor.expand_shape' in the collapse-expand
+// pair emitted for a 'tosa.reshape' op.
+TensorType inferReshapeExpandedType(TensorType inputType,
+                                    ArrayRef<int64_t> newShape) {
+  // Special case for 0D output tensor. Note: Watch out when using Type::clone()
+  // with just '{}', as it will invoke the incorrect overload.
+  if (newShape.empty())
+    return inputType.clone(ArrayRef<int64_t>{});
+
+  // Check if the input is static, and if so, get its total size
+  bool inputIsStatic = inputType.hasStaticShape();
+  int64_t totalSize = inputIsStatic ? inputType.getNumElements() : -1;
+ 
+  // Compute result shape
+  bool resultIsStatic = true;
+  auto resultShape = llvm::map_to_vector(newShape, [&](int64_t size) -> int64_t {
+    // If this is not a placeholder, do not change it
+    if (size >= 0)
+      return size;
+
+    // If we do not know the total size of the tensor, keep this dimension
+    // dynamic in the result shape.
+    if (!inputIsStatic) {
+      resultIsStatic = false;
+      return ShapedType::kDynamic;
+    }
 
+    // Calculate the product of all elements in 'newShape' except for the -1
+    // placeholder, which we discard by negating the result.
+    int64_t totalSizeNoPlaceholder = -std::accumulate(
+        newShape.begin(), newShape.end(), 1, std::multiplies());
+
+    // If there is a 0 component in 'newShape', resolve the placeholder as 0.
+    if (totalSizeNoPlaceholder == 0)
+      return 0;
+
+    // Resolve the placeholder as the quotient between the total tensor size and
+    // the product of all other sizes.
+    return totalSize / totalSizeNoPlaceholder;
+  });
+
+  // A syntactic restriction in 'tensor.expand_shape' forbids a dynamically
+  // shaped input from being reshaped into a statically shaped result. We may
+  // simply turn the first result dimension dynamic to address this.
+  if (!inputIsStatic && resultIsStatic)
+    resultShape[0] = ShapedType::kDynamic;
+  
+  // The 'tensor.expand_shape' op also forbids a statically shaped input from
+  // being reshaped into a dynamically shaped result, but the placeholder
+  // inference algorithm above guarantees that this will never be the case.
+  assert(!inputIsStatic || resultIsStatic);
+
+  // Create result type
+  return inputType.clone(resultShape);
+}
+
+// Infer the result type of 'tensor.collapse_shape' in the collapse-expand
+// pair emitted for a 'tosa.reshape' op.
+TensorType inferReshapeCollapsedType(TensorType lhsType, TensorType rhsType) {
+  auto lhsShape = lhsType.getShape();
+  auto rhsShape = rhsType.getShape();
+
+  if (lhsShape.empty() || rhsShape.empty())
+    return lhsType.clone(ArrayRef<int64_t>{});
+
+  if (ShapedType::isDynamicShape(lhsShape) || ShapedType::isDynamicShape(rhsShape))
+    return lhsType.clone({ShapedType::kDynamic});
+
+  SmallVector<int64_t> intermediateShape;
   unsigned currLhsDim = 0, currRhsDim = 0;
   while (currLhsDim < lhsShape.size() && currRhsDim < rhsShape.size()) {
     int64_t rhsSize = rhsShape[currRhsDim];
@@ -62,174 +137,113 @@ static bool findIntermediateShape(ArrayRef<int64_t> lhsShape,
     currLhsDim++;
   }
 
-  // If the iterators didn't reach the end and their leftover dimensions are not
-  // equal to 1 an intermediate shape was not found.
-  while (currLhsDim < lhsShape.size()) {
-    if (lhsShape[currLhsDim++] != 1) {
-      return false;
-    }
+  // Static shapes are guaranteed to be compatible by the op verifier, so all
+  // leftover dimensions should be 1.
+  for (; currLhsDim < lhsShape.size(); currLhsDim++) {
+    assert(lhsShape[currLhsDim] == 1);
   }
-
-  while (currRhsDim < rhsShape.size()) {
-    if (rhsShape[currRhsDim++] != 1) {
-      return false;
-    }
+  for (; currRhsDim < rhsShape.size(); currRhsDim++) {
+    assert(rhsShape[currRhsDim] == 1);
   }
-
-  return true;
+  
+  return lhsType.clone(intermediateShape);
 }
 
-static bool createReassociationMapsForCollapse(
-    PatternRewriter &rewriter, ArrayRef<int64_t> srcShape,
-    ArrayRef<int64_t> dstShape,
-    SmallVector<ReassociationExprs, 4> &reassociationMap, bool isDynamic) {
+SmallVector<ReassociationExprs>
+createReassociationMapForCollapse(OpBuilder &builder, Type srcType, Type dstType) {
+  auto srcShape = cast<TensorType>(srcType).getShape();
+  auto dstShape = cast<TensorType>(dstType).getShape();
 
-  // If the shape is dynamic, create a map for collapsing into one dimension.
-  if (isDynamic) {
-    SmallVector<AffineExpr, 2> exprs;
-    for (int i = 0, s = srcShape.size(); i < s; ++i)
-      exprs.push_back(rewriter.getAffineDimExpr(i));
-    reassociationMap = {exprs};
-    return true;
-  }
+  if (srcShape.empty() || dstShape.empty())
+    return {};
 
-  if (dstShape.empty()) {
-    reassociationMap = {};
-    return true;
+  if (ShapedType::isDynamicShape(srcShape) || ShapedType::isDynamicShape(dstShape)) {
+    assert(dstShape.size() == 1);
+    SmallVector<AffineExpr, 2> exprs;
+    for (auto i : llvm::seq<int64_t>(srcShape.size()))
+      exprs.push_back(builder.getAffineDimExpr(i));
+    return {exprs};
   }
 
-  reassociationMap.resize(dstShape.size());
+  SmallVector<ReassociationExprs> reassociationMap(dstShape.size());
   unsigned currSrcDim = 0, currDstDim = 0;
   while (currSrcDim < srcShape.size() && currDstDim < dstShape.size()) {
     int64_t dstSize = dstShape[currDstDim];
     int64_t srcSize = srcShape[currSrcDim];
     while (srcSize < dstSize && currSrcDim < srcShape.size()) {
       reassociationMap[currDstDim].push_back(
-          rewriter.getAffineDimExpr(currSrcDim++));
+          builder.getAffineDimExpr(currSrcDim++));
       srcSize *= srcShape[currSrcDim];
     }
     if (srcSize == dstSize) {
       reassociationMap[currDstDim].push_back(
-          rewriter.getAffineDimExpr(currSrcDim++));
+          builder.getAffineDimExpr(currSrcDim++));
       // If the next dim in collapsedShape is not 1, treat subsequent dims in
       // expandedShape which are 1 to be collapsed.
       if (currDstDim == dstShape.size() - 1 || dstShape[currDstDim + 1] != 1) {
         while (currSrcDim < srcShape.size() && srcShape[currSrcDim] == 1) {
           reassociationMap[currDstDim].push_back(
-              rewriter.getAffineDimExpr(currSrcDim++));
+              builder.getAffineDimExpr(currSrcDim++));
         }
       }
     }
     currDstDim++;
   }
 
-  // If both iterators didn't reach the end, we have leftover dimentions which
-  // implies that we have a mismatch in shape.
-  return currSrcDim == srcShape.size() && currDstDim == dstShape.size();
+  // If the source and target shapes are compatible, both iterators must have
+  // reached the end. This condition is guaranteed by the op verifier for
+  // static shapes.
+  assert(currSrcDim == srcShape.size() && currDstDim == dstShape.size());
+  return reassociationMap;
 }
 
-namespace {
-Value createCollapse(ConversionPatternRewriter &rewriter, Location loc,
-                     ShapedType resultTy, Value operand) {
-  ShapedType operandTy = cast<ShapedType>(operand.getType());
-  if (resultTy == operandTy)
-    return operand;
-
-  bool isDynamic = !operandTy.hasStaticShape();
-
-  if (isDynamic && resultTy.getRank() != 1) {
-    (void)rewriter.notifyMatchFailure(
-        loc, "Cannot collapse dynamic dims to more than one dimension");
-    return {};
-  }
-
-  SmallVector<ReassociationExprs, 4> reassociationMap;
-  if (!createReassociationMapsForCollapse(rewriter, operandTy.getShape(),
-                                          resultTy.getShape(),
-                                          reassociationMap, isDynamic)) {
-    (void)rewriter.notifyMatchFailure(
-        loc, "tosa.reshape Attempting to collapse into an incompatible shape");
-    return {};
-  }
-
-  SmallVector<int64_t> intermediateShape;
-  if (!findIntermediateShape(operandTy.getShape(), resultTy.getShape(),
-                             intermediateShape, isDynamic)) {
-    (void)rewriter.notifyMatchFailure(
-        loc, "tosa.reshape Cannot collapse into given shape");
-    return {};
-  }
-  return rewriter.create<tensor::CollapseShapeOp>(loc, resultTy, operand,
-                                                  reassociationMap);
+// Create a tensor.collapse_shape op that reshapes the input into the given
+// result type.
+Value createCollapse(OpBuilder &builder, Location loc, TensorType resultType,
+                     Value input) {
+  auto reassociationMap =
+      createReassociationMapForCollapse(builder, input.getType(), resultType);
+  return builder.createOrFold<tensor::CollapseShapeOp>(loc, resultType, input,
+                                                       reassociationMap);
 }
 
-Value createExpand(ConversionPatternRewriter &rewriter, Location loc,
-                   ShapedType resultTy, Value operand) {
-  ShapedType operandTy = cast<ShapedType>(operand.getType());
-  if (resultTy == operandTy)
-    return operand;
-
-  bool isDynamic = !operandTy.hasStaticShape();
-
-  if (isDynamic && operandTy.getRank() != 1) {
-    (void)rewriter.notifyMatchFailure(
-        loc, "Cannot expand dynamic dims from more than one dimension");
-    return {};
-  }
-
-  SmallVector<ReassociationExprs, 4> reassociationMap;
-  if (!createReassociationMapsForCollapse(rewriter, resultTy.getShape(),
-                                          operandTy.getShape(),
-                                          reassociationMap, isDynamic)) {
-    (void)rewriter.notifyMatchFailure(
-        loc, "tosa.reshape Attempting to expand into an incompatible shape");
-    return {};
-  }
-
-  SmallVector<int64_t> intermediateShape;
-  if (!findIntermediateShape(operandTy.getShape(), resultTy.getShape(),
-                             intermediateShape, isDynamic) ||
-      intermediateShape != operandTy.getShape()) {
-    (void)rewriter.notifyMatchFailure(
-        loc, "tosa.reshape Cannot expand into given shape");
-    return {};
-  }
-  return rewriter.create<tensor::ExpandShapeOp>(loc, resultTy, operand,
-                                                reassociationMap);
+// Create a tensor.expand_shape op that reshapes the input into the given result
+// type.
+Value createExpand(OpBuilder &builder, Location loc, TensorType resultType,
+                   Value input) {
+  auto reassociationMap =
+      createReassociationMapForCollapse(builder, resultType, input.getType());
+  return builder.createOrFold<tensor::ExpandShapeOp>(loc, resultType, input,
+                                                     reassociationMap);
 }
 
-class ReshapeConverterCollapseExpand
-    : public OpConversionPattern<tosa::ReshapeOp> {
+class ReshapeConverter : public OpConversionPattern<tosa::ReshapeOp> {
 public:
   using OpConversionPattern<tosa::ReshapeOp>::OpConversionPattern;
 
   LogicalResult
   matchAndRewrite(tosa::ReshapeOp reshape, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const final {
-    ShapedType operandTy = cast<ShapedType>(adaptor.getInput1().getType());
-    ShapedType resultTy = cast<ShapedType>(reshape.getType());
-    bool isDynamic = !operandTy.hasStaticShape();
-
-    SmallVector<int64_t> intermediateShape;
-    if (!findIntermediateShape(resultTy.getShape(), operandTy.getShape(),
-                               intermediateShape, isDynamic)) {
-      return rewriter.notifyMatchFailure(
-          reshape, "tosa.reshape Cannot identify an intermediate shape between "
-                   "the given two shapes");
-    }
-    auto intermediateTy = RankedTensorType::get(
-        intermediateShape, reshape.getType().getElementType());
-
-    Value collapse = createCollapse(rewriter, reshape.getLoc(), intermediateTy,
-                                    adaptor.getInput1());
-    if (!collapse)
-      return failure();
-
-    Value expand = createExpand(rewriter, reshape.getLoc(), resultTy, collapse);
-    if (!expand)
-      return failure();
-
-    rewriter.replaceOp(reshape, expand);
+    auto loc = reshape.getLoc();
+    auto resultType = reshape.getResult().getType();
+    auto input = reshape.getInput1();
+    auto newShape = reshape.getNewShape();
+
+    // Infer all intermediate types
+    auto inputType = inferReshapeInputType(input, newShape);
+    auto expandedType = inferReshapeExpandedType(inputType, newShape);
+    auto collapsedType = inferReshapeCollapsedType(inputType, expandedType);
+
+    // Cast input if needed
+    auto castInput = rewriter.createOrFold<tensor::CastOp>(loc, inputType, input);
+
+    // Emit collaspe-expand pair
+    auto collapsed = createCollapse(rewriter, loc, collapsedType, castInput);
+    auto expanded = createExpand(rewriter, loc, expandedType, collapsed);
+
+    // Cast to final result type if needed
+    auto result = rewriter.createOrFold<tensor::CastOp>(loc, resultType, expanded);
+    rewriter.replaceOp(reshape, result);
     return success();
   }
 };
@@ -416,8 +430,10 @@ struct ConcatConverter : public OpConversionPattern<tosa::ConcatOp> {
 
 void mlir::tosa::populateTosaToTensorConversionPatterns(
     RewritePatternSet *patterns) {
-  patterns->add<SliceConverter, PadConverter, ConcatConverter>(
-      patterns->getContext());
-
-  patterns->add<ReshapeConverterCollapseExpand>(patterns->getContext());
+  patterns->add<
+    ConcatConverter,
+    PadConverter,
+    ReshapeConverter,
+    SliceConverter
+  >(patterns->getContext());
 }
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
index 4c50aaecfe94..d23c9fe824c9 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
@@ -795,7 +795,10 @@ OpFoldResult ReshapeOp::fold(FoldAdaptor adaptor) {
   if (!inputTy || !outputTy)
     return {};
 
-  if (inputTy == outputTy)
+  // Fold when the input and output types are the same. This is only safe when
+  // there is at most 1 dynamic dimension. For 2 or more dynamic dimensions,
+  // there may still be a productive reshape.
+  if (inputTy == outputTy && inputTy.getNumDynamicDims() < 2)
     return getInput1();
 
   // reshape(reshape(x)) -> reshape(x)
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index f461e7e1a555..6e6e84350738 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -970,6 +970,11 @@ mlir::LogicalResult tosa::ReshapeOp::verify() {
                            << " elements into " << outputElementsNum;
     }
   }
+
+  int missingDims = llvm::count(getNewShape(), -1);
+  if (missingDims > 1)
+    return emitOpError() << "At most one target dimension can be -1";
+
   return mlir::success();
 }
 
diff --git a/mlir/test/Conversion/TosaToTensor/tosa-to-tensor.mlir b/mlir/test/Conversion/TosaToTensor/tosa-to-tensor.mlir
index daaa68a7260b..a8a3c42e1684 100644
--- a/mlir/test/Conversion/TosaToTensor/tosa-to-tensor.mlir
+++ b/mlir/test/Conversion/TosaToTensor/tosa-to-tensor.mlir
@@ -1,95 +1,363 @@
 // RUN: mlir-opt --split-input-file --tosa-to-tensor %s -o -| FileCheck %s
 
-// CHECK-LABEL: @test_reshape_downrank
-// CHECK-SAME: (%[[ARG0:[0-9a-zA-Z_]*]]
-func.func @test_reshape_downrank(%arg0: tensor<2x3xf32>) -> tensor<6xf32> {
-  // CHECK: [[RESHAPE:%.+]] = tensor.collapse_shape %[[ARG0]] {{\[}}[0, 1]]
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 6>} : (tensor<2x3xf32>) -> tensor<6xf32>
-  // CHECK: return [[RESHAPE]]
-  return %0 : tensor<6xf32>
+// -----
+
+// CHECK-LABEL: test_reshape_0d_same_s2s_explicit
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<f32>
+// CHECK: return %[[ARG_0]] : tensor<f32>
+func.func @test_reshape_0d_same_s2s_explicit(%arg0: tensor<f32>) -> tensor<f32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64>} : (tensor<f32>) -> tensor<f32>
+  return %0 : tensor<f32>
 }
 
 // -----
 
-// CHECK-LABEL: @test_reshape_downrank_dyn
-// CHECK-SAME: (%[[ARG0:[0-9a-zA-Z_]*]]
-func.func @test_reshape_downrank_dyn(%arg0: tensor<2x?xf32>) -> tensor<?xf32> {
-  // CHECK: [[RESHAPE:%.+]] = tensor.collapse_shape %[[ARG0]] {{\[}}[0, 1]]
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: -1>} : (tensor<2x?xf32>) -> tensor<?xf32>
-  // CHECK: return [[RESHAPE]]
+// CHECK-LABEL: test_reshape_0d_up_s2d_auto
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<f32>
+// CHECK: %[[VAL_0:.*]] = tensor.expand_shape %[[ARG_0]] [] : tensor<f32> into tensor<1xf32>
+// CHECK: %[[VAL_1:.*]] = tensor.cast %[[VAL_0]] : tensor<1xf32> to tensor<?xf32>
+// CHECK: return %[[VAL_1]] : tensor<?xf32>
+func.func @test_reshape_0d_up_s2d_auto(%arg0: tensor<f32>) -> tensor<?xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: -1>} : (tensor<f32>) -> tensor<?xf32>
   return %0 : tensor<?xf32>
 }
 
 // -----
 
-// CHECK-LABEL: @test_reshape_uprank
-// CHECK-SAME: (%[[ARG0:[0-9a-zA-Z_]*]]
-func.func @test_reshape_uprank(%arg0: tensor<6xf32>) -> tensor<2x3xf32> {
-  // CHECK: [[RESHAPE:%.+]] = tensor.expand_shape %[[ARG0]] {{\[}}[0, 1]]
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 3>} : (tensor<6xf32>) -> tensor<2x3xf32>
-  // CHECK: return [[RESHAPE]]
-  return %0 : tensor<2x3xf32>
+// CHECK-LABEL: test_reshape_0d_up_s2d_explicit
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<f32>
+// CHECK: %[[VAL_0:.*]] = tensor.expand_shape %[[ARG_0]] [] : tensor<f32> into tensor<1xf32>
+// CHECK: %[[VAL_1:.*]] = tensor.cast %[[VAL_0]] : tensor<1xf32> to tensor<?xf32>
+// CHECK: return %[[VAL_1]] : tensor<?xf32>
+func.func @test_reshape_0d_up_s2d_explicit(%arg0: tensor<f32>) -> tensor<?xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 1>} : (tensor<f32>) -> tensor<?xf32>
+  return %0 : tensor<?xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_0d_up_s2s_auto
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<f32>
+// CHECK: %[[VAL_0:.*]] = tensor.expand_shape %[[ARG_0]] [] : tensor<f32> into tensor<1xf32>
+// CHECK: return %[[VAL_0]] : tensor<1xf32>
+func.func @test_reshape_0d_up_s2s_auto(%arg0: tensor<f32>) -> tensor<1xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: -1>} : (tensor<f32>) -> tensor<1xf32>
+  return %0 : tensor<1xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_0d_up_s2s_explicit
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<f32>
+// CHECK: %[[VAL_0:.*]] = tensor.expand_shape %[[ARG_0]] [] : tensor<f32> into tensor<1xf32>
+// CHECK: return %[[VAL_0]] : tensor<1xf32>
+func.func @test_reshape_0d_up_s2s_explicit(%arg0: tensor<f32>) -> tensor<1xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 1>} : (tensor<f32>) -> tensor<1xf32>
+  return %0 : tensor<1xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_1d_down_d2s_explicit
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<?xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.cast %[[ARG_0]] : tensor<?xf32> to tensor<1xf32>
+// CHECK: %[[VAL_1:.*]] = tensor.collapse_shape %[[VAL_0]] [] : tensor<1xf32> into tensor<f32>
+// CHECK: return %[[VAL_1]] : tensor<f32>
+func.func @test_reshape_1d_down_d2s_explicit(%arg0: tensor<?xf32>) -> tensor<f32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64>} : (tensor<?xf32>) -> tensor<f32>
+  return %0 : tensor<f32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_1d_down_s2s_explicit
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<1xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] [] : tensor<1xf32> into tensor<f32>
+// CHECK: return %[[VAL_0]] : tensor<f32>
+func.func @test_reshape_1d_down_s2s_explicit(%arg0: tensor<1xf32>) -> tensor<f32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64>} : (tensor<1xf32>) -> tensor<f32>
+  return %0 : tensor<f32>
 }
 
 // -----
 
-// CHECK-LABEL: @test_reshape_uprank_dyn
-// CHECK-SAME: (%[[ARG0:[0-9a-zA-Z_]*]]
-func.func @test_reshape_uprank_dyn(%arg0: tensor<?xf32>) -> tensor<2x?xf32> {
-  // CHECK: [[RESHAPE:%.+]] = tensor.expand_shape %[[ARG0]] {{\[}}[0, 1]]
+// CHECK-LABEL: test_reshape_1d_up_d2d_auto
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<?xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.expand_shape %[[ARG_0]] {{\[\[}}0, 1]] : tensor<?xf32> into tensor<2x?xf32>
+// CHECK: return %[[VAL_0]] : tensor<2x?xf32>
+func.func @test_reshape_1d_up_d2d_auto(%arg0: tensor<?xf32>) -> tensor<2x?xf32> {
   %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, -1>} : (tensor<?xf32>) -> tensor<2x?xf32>
-  // CHECK: return [[RESHAPE]]
   return %0 : tensor<2x?xf32>
 }
 
 // -----
 
-// CHECK-LABEL: @test_reshape_samerank
-//  CHECK-SAME: (%[[ARG0:.*]]: tensor<3x2xf32>)
-func.func @test_reshape_samerank(%arg0: tensor<3x2xf32>) -> tensor<2x3xf32> {
-  // CHECK-NEXT: %[[RESHAPE1:.*]] = tensor.collapse_shape %[[ARG0]] {{\[}}[0, 1]]
-  // CHECK-NEXT: %[[RESHAPE2:.*]] = tensor.expand_shape %[[RESHAPE1]] {{\[}}[0, 1]]
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 3>} : (tensor<3x2xf32>) -> tensor<2x3xf32>
-  // CHECK-NEXT: return %[[RESHAPE2]]
+// CHECK-LABEL: test_reshape_1d_up_s2s_explicit
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<6xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.expand_shape %[[ARG_0]] {{\[\[}}0, 1]] : tensor<6xf32> into tensor<2x3xf32>
+// CHECK: return %[[VAL_0]] : tensor<2x3xf32>
+func.func @test_reshape_1d_up_s2s_explicit(%arg0: tensor<6xf32>) -> tensor<2x3xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 3>} : (tensor<6xf32>) -> tensor<2x3xf32>
   return %0 : tensor<2x3xf32>
 }
 
 // -----
 
-// CHECK-LABEL: @test_reshape_samerank_dyn
-//  CHECK-SAME: (%[[ARG0:.*]]: tensor<?x2xf32>)
-func.func @test_reshape_samerank_dyn(%arg0: tensor<?x2xf32>) -> tensor<2x?xf32> {
-  // CHECK-NEXT: %[[RESHAPE1:.*]] = tensor.collapse_shape %[[ARG0]] {{\[}}[0, 1]]
-  // CHECK-NEXT: %[[RESHAPE2:.*]] = tensor.expand_shape %[[RESHAPE1]] {{\[}}[0, 1]]
+// CHECK-LABEL: test_reshape_2d_down_d2d_auto
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<2x?xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1]] : tensor<2x?xf32> into tensor<?xf32>
+// CHECK: return %[[VAL_0]] : tensor<?xf32>
+func.func @test_reshape_2d_down_d2d_auto(%arg0: tensor<2x?xf32>) -> tensor<?xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: -1>} : (tensor<2x?xf32>) -> tensor<?xf32>
+  return %0 : tensor<?xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_2d_down_s2s_explicit
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<2x3xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1]] : tensor<2x3xf32> into tensor<6xf32>
+// CHECK: return %[[VAL_0]] : tensor<6xf32>
+func.func @test_reshape_2d_down_s2s_explicit(%arg0: tensor<2x3xf32>) -> tensor<6xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 6>} : (tensor<2x3xf32>) -> tensor<6xf32>
+  return %0 : tensor<6xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_2d_same_d2d_auto
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<?x2xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1]] : tensor<?x2xf32> into tensor<?xf32>
+// CHECK: %[[VAL_1:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1]] : tensor<?xf32> into tensor<2x?xf32>
+// CHECK: return %[[VAL_1]] : tensor<2x?xf32>
+func.func @test_reshape_2d_same_d2d_auto(%arg0: tensor<?x2xf32>) -> tensor<2x?xf32> {
   %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, -1>} : (tensor<?x2xf32>) -> tensor<2x?xf32>
-  // CHECK-NEXT: return %[[RESHAPE2]]
   return %0 : tensor<2x?xf32>
 }
 
 // -----
 
-// CHECK-LABEL: @test_reshape_downrank_6D
-// CHECK-SAME: (%[[ARG0:[0-9a-zA-Z_]*]]:
-func.func @test_reshape_downrank_6D(%arg0: tensor<1x2x3x5x7x11xf32>) -> tensor<6x5x77xf32> {
-  // CHECK: tensor.collapse_shape %[[ARG0]] {{\[}}[0, 1, 2], [3], [4, 5]]
-  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 6, 5, 77>} : (tensor<1x2x3x5x7x11xf32>) -> tensor<6x5x77xf32>
-  return %0 : tensor<6x5x77xf32>
+// CHECK-LABEL: test_reshape_2d_same_s2d_auto
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<2x4xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1]] : tensor<2x4xf32> into tensor<8xf32>
+// CHECK: %[[VAL_1:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1]] : tensor<8xf32> into tensor<4x2xf32>
+// CHECK: %[[VAL_2:.*]] = tensor.cast %[[VAL_1]] : tensor<4x2xf32> to tensor<?x2xf32>
+// CHECK: return %[[VAL_2]] : tensor<?x2xf32>
+func.func @test_reshape_2d_same_s2d_auto(%arg0: tensor<2x4xf32>) -> tensor<?x2xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: -1, 2>} : (tensor<2x4xf32>) -> tensor<?x2xf32>
+  return %0 : tensor<?x2xf32>
 }
 
 // -----
 
-// CHECK-LABEL: @test_reshape_downrank_6D_dyn
-// CHECK-SAME: (%[[ARG0:[0-9a-zA-Z_]*]]:
-func.func @test_reshape_downrank_6D_dyn(%arg0: tensor<1x2x?x5x7x11xf32>) -> tensor<?x5x77xf32> {
-  // CHECK: tensor.collapse_shape %[[ARG0]] {{\[}}[0, 1, 2, 3, 4, 5]]
-  // CHECK: tensor.expand_shape %{{.*}} {{\[}}[0, 1, 2]]
+// CHECK-LABEL: test_reshape_2d_same_s2d_explicit
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<2x4xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1]] : tensor<2x4xf32> into tensor<8xf32>
+// CHECK: %[[VAL_1:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1]] : tensor<8xf32> into tensor<4x2xf32>
+// CHECK: %[[VAL_2:.*]] = tensor.cast %[[VAL_1]] : tensor<4x2xf32> to tensor<?x2xf32>
+// CHECK: return %[[VAL_2]] : tensor<?x2xf32>
+func.func @test_reshape_2d_same_s2d_explicit(%arg0: tensor<2x4xf32>) -> tensor<?x2xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 4, 2>} : (tensor<2x4xf32>) -> tensor<?x2xf32>
+  return %0 : tensor<?x2xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_2d_same_s2s_explicit
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<3x2xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1]] : tensor<3x2xf32> into tensor<6xf32>
+// CHECK: %[[VAL_1:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1]] : tensor<6xf32> into tensor<2x3xf32>
+// CHECK: return %[[VAL_1]] : tensor<2x3xf32>
+func.func @test_reshape_2d_same_s2s_explicit(%arg0: tensor<3x2xf32>) -> tensor<2x3xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 3>} : (tensor<3x2xf32>) -> tensor<2x3xf32>
+  return %0 : tensor<2x3xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_3d_same_d2d_auto_empty
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<3x2x?xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1, 2]] : tensor<3x2x?xf32> into tensor<?xf32>
+// CHECK: %[[VAL_1:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1, 2]] : tensor<?xf32> into tensor<0x3x?xf32>
+// CHECK: %[[VAL_2:.*]] = tensor.cast %[[VAL_1]] : tensor<0x3x?xf32> to tensor<?x?x?xf32>
+// CHECK: return %[[VAL_2]] : tensor<?x?x?xf32>
+func.func @test_reshape_3d_same_d2d_auto_empty(%arg0: tensor<3x2x?xf32>) -> tensor<?x?x?xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 0, 3, -1>} : (tensor<3x2x?xf32>) -> tensor<?x?x?xf32>
+  return %0 : tensor<?x?x?xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_3d_same_d2d_auto
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<2x?x?xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1, 2]] : tensor<2x?x?xf32> into tensor<?xf32>
+// CHECK: %[[VAL_1:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1, 2]] : tensor<?xf32> into tensor<2x?x4xf32>
+// CHECK: %[[VAL_2:.*]] = tensor.cast %[[VAL_1]] : tensor<2x?x4xf32> to tensor<?x?x?xf32>
+// CHECK: return %[[VAL_2]] : tensor<?x?x?xf32>
+func.func @test_reshape_3d_same_d2d_auto(%arg0: tensor<2x?x?xf32>) -> tensor<?x?x?xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, -1, 4>} : (tensor<2x?x?xf32>) -> tensor<?x?x?xf32>
+  return %0 : tensor<?x?x?xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_3d_same_d2d_auto_identity
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<?x3x4xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1, 2]] : tensor<?x3x4xf32> into tensor<?xf32>
+// CHECK: %[[VAL_1:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1, 2]] : tensor<?xf32> into tensor<2x3x?xf32>
+// CHECK: return %[[VAL_1]] : tensor<2x3x?xf32>
+func.func @test_reshape_3d_same_d2d_auto_identity(%arg0: tensor<?x3x4xf32>) -> tensor<2x3x?xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 3, -1>} : (tensor<?x3x4xf32>) -> tensor<2x3x?xf32>
+  return %0 : tensor<2x3x?xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_3d_same_d2d_explicit_empty
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<3x2x?xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1, 2]] : tensor<3x2x?xf32> into tensor<?xf32>
+// CHECK: %[[VAL_1:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1, 2]] : tensor<?xf32> into tensor<?x3x2xf32>
+// CHECK: %[[VAL_2:.*]] = tensor.cast %[[VAL_1]] : tensor<?x3x2xf32> to tensor<?x?x?xf32>
+// CHECK: return %[[VAL_2]] : tensor<?x?x?xf32>
+func.func @test_reshape_3d_same_d2d_explicit_empty(%arg0: tensor<3x2x?xf32>) -> tensor<?x?x?xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 0, 3, 2>} : (tensor<3x2x?xf32>) -> tensor<?x?x?xf32>
+  return %0 : tensor<?x?x?xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_3d_same_d2d_explicit
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<?x?x?xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1, 2]] : tensor<?x?x?xf32> into tensor<?xf32>
+// CHECK: %[[VAL_1:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1, 2]] : tensor<?xf32> into tensor<?x3x4xf32>
+// CHECK: %[[VAL_2:.*]] = tensor.cast %[[VAL_1]] : tensor<?x3x4xf32> to tensor<?x?x?xf32>
+// CHECK: return %[[VAL_2]] : tensor<?x?x?xf32>
+func.func @test_reshape_3d_same_d2d_explicit(%arg0: tensor<?x?x?xf32>) -> tensor<?x?x?xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 3, 4>} : (tensor<?x?x?xf32>) -> tensor<?x?x?xf32>
+  return %0 : tensor<?x?x?xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_3d_same_d2d_explicit_identity
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<?x3x4xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.cast %[[ARG_0]] : tensor<?x3x4xf32> to tensor<2x3x?xf32>
+// CHECK: return %[[VAL_0]] : tensor<2x3x?xf32>
+func.func @test_reshape_3d_same_d2d_explicit_identity(%arg0: tensor<?x3x4xf32>) -> tensor<2x3x?xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 3, 4>} : (tensor<?x3x4xf32>) -> tensor<2x3x?xf32>
+  return %0 : tensor<2x3x?xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_3d_same_d2s_auto
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<?x?x?xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1, 2]] : tensor<?x?x?xf32> into tensor<?xf32>
+// CHECK: %[[VAL_1:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1, 2]] : tensor<?xf32> into tensor<2x?x4xf32>
+// CHECK: %[[VAL_2:.*]] = tensor.cast %[[VAL_1]] : tensor<2x?x4xf32> to tensor<2x3x4xf32>
+// CHECK: return %[[VAL_2]] : tensor<2x3x4xf32>
+func.func @test_reshape_3d_same_d2s_auto(%arg0: tensor<?x?x?xf32>) -> tensor<2x3x4xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, -1, 4>} : (tensor<?x?x?xf32>) -> tensor<2x3x4xf32>
+  return %0 : tensor<2x3x4xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_3d_same_d2s_explicit
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<?x?x?xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1, 2]] : tensor<?x?x?xf32> into tensor<?xf32>
+// CHECK: %[[VAL_1:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1, 2]] : tensor<?xf32> into tensor<?x3x4xf32>
+// CHECK: %[[VAL_2:.*]] = tensor.cast %[[VAL_1]] : tensor<?x3x4xf32> to tensor<2x3x4xf32>
+// CHECK: return %[[VAL_2]] : tensor<2x3x4xf32>
+func.func @test_reshape_3d_same_d2s_explicit(%arg0: tensor<?x?x?xf32>) -> tensor<2x3x4xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 3, 4>} : (tensor<?x?x?xf32>) -> tensor<2x3x4xf32>
+  return %0 : tensor<2x3x4xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_3d_same_s2s_explicit_identity
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<2x3x4xf32>
+// CHECK: return %[[ARG_0]] : tensor<2x3x4xf32>
+func.func @test_reshape_3d_same_s2s_explicit_identity(%arg0: tensor<2x3x4xf32>) -> tensor<2x3x4xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 3, 4>} : (tensor<2x3x4xf32>) -> tensor<2x3x4xf32>
+  return %0 : tensor<2x3x4xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_3d_up_d2s_explicit
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<?x?x?xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1, 2]] : tensor<?x?x?xf32> into tensor<?xf32>
+// CHECK: %[[VAL_1:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1, 2, 3]] : tensor<?xf32> into tensor<?x3x2x1xf32>
+// CHECK: %[[VAL_2:.*]] = tensor.cast %[[VAL_1]] : tensor<?x3x2x1xf32> to tensor<1x3x2x1xf32>
+// CHECK: return %[[VAL_2]] : tensor<1x3x2x1xf32>
+func.func @test_reshape_3d_up_d2s_explicit(%input: tensor<?x?x?xf32>) -> tensor<1x3x2x1xf32> {
+  %0 = tosa.reshape %input {new_shape = array<i64: 1, 3, 2, 1>} : (tensor<?x?x?xf32>) -> tensor<1x3x2x1xf32>
+  return %0 : tensor<1x3x2x1xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_4d_down_d2s_explicit
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<?x?x?x?xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.cast %[[ARG_0]] : tensor<?x?x?x?xf32> to tensor<1x1x1x1xf32>
+// CHECK: %[[VAL_1:.*]] = tensor.collapse_shape %[[VAL_0]] [] : tensor<1x1x1x1xf32> into tensor<f32>
+// CHECK: return %[[VAL_1]] : tensor<f32>
+func.func @test_reshape_4d_down_d2s_explicit(%arg0: tensor<?x?x?x?xf32>) -> tensor<f32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64>} : (tensor<?x?x?x?xf32>) -> tensor<f32>
+  return %0 : tensor<f32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_5d_down_d2d_auto
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<?x?x?x2x3xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1, 2, 3, 4]] : tensor<?x?x?x2x3xf32> into tensor<?xf32>
+// CHECK: %[[VAL_1:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1, 2]] : tensor<?xf32> into tensor<?x2x3xf32>
+// CHECK: return %[[VAL_1]] : tensor<?x2x3xf32>
+func.func @test_reshape_5d_down_d2d_auto(%arg0: tensor<?x?x?x2x3xf32>) -> tensor<?x2x3xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: -1, 2, 3>} : (tensor<?x?x?x2x3xf32>) -> tensor<?x2x3xf32>
+  return %0 : tensor<?x2x3xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_6d_down_d2d_auto
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<1x2x?x5x7x11xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1, 2, 3, 4, 5]] : tensor<1x2x?x5x7x11xf32> into tensor<?xf32>
+// CHECK: %[[VAL_1:.*]] = tensor.expand_shape %[[VAL_0]] {{\[\[}}0, 1, 2]] : tensor<?xf32> into tensor<?x5x77xf32>
+// CHECK: return %[[VAL_1]] : tensor<?x5x77xf32>
+func.func @test_reshape_6d_down_d2d_auto(%arg0: tensor<1x2x?x5x7x11xf32>) -> tensor<?x5x77xf32> {
   %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: -1, 5, 77>} : (tensor<1x2x?x5x7x11xf32>) -> tensor<?x5x77xf32>
   return %0 : tensor<?x5x77xf32>
 }
 
 // -----
 
-// CHECK-LABLE: func @slice
+// CHECK-LABEL: test_reshape_6d_down_s2s_auto
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<1x2x3x5x7x11xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1, 2], [3], [4, 5]] : tensor<1x2x3x5x7x11xf32> into tensor<6x5x77xf32>
+// CHECK: return %[[VAL_0]] : tensor<6x5x77xf32>
+func.func @test_reshape_6d_down_s2s_auto(%arg0: tensor<1x2x3x5x7x11xf32>) -> tensor<6x5x77xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 6, 5, -1>} : (tensor<1x2x3x5x7x11xf32>) -> tensor<6x5x77xf32>
+  return %0 : tensor<6x5x77xf32>
+}
+
+// -----
+
+// CHECK-LABEL: test_reshape_6d_down_s2s_explicit
+// CHECK-SAME: %[[ARG_0:[a-zA-Z0-9_]+]]: tensor<1x2x3x5x7x11xf32>
+// CHECK: %[[VAL_0:.*]] = tensor.collapse_shape %[[ARG_0]] {{\[\[}}0, 1, 2], [3], [4, 5]] : tensor<1x2x3x5x7x11xf32> into tensor<6x5x77xf32>
+// CHECK: return %[[VAL_0]] : tensor<6x5x77xf32>
+func.func @test_reshape_6d_down_s2s_explicit(%arg0: tensor<1x2x3x5x7x11xf32>) -> tensor<6x5x77xf32> {
+  %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 6, 5, 77>} : (tensor<1x2x3x5x7x11xf32>) -> tensor<6x5x77xf32>
+  return %0 : tensor<6x5x77xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @slice
 func.func @slice(%arg0: tensor<6xf32>) ->() {
   // CHECK: [[SLICE:%.+]] = tensor.extract_slice %arg0[2] [1] [1]
   %0 = "tosa.slice"(%arg0) {start = array<i64: 2>, size = array<i64: 1>} : (tensor<6xf32>)  -> (tensor<1xf32>)
diff --git a/mlir/test/Dialect/Tosa/canonicalize.mlir b/mlir/test/Dialect/Tosa/canonicalize.mlir
index e7ede2e0ccef..6eac759a0836 100644
--- a/mlir/test/Dialect/Tosa/canonicalize.mlir
+++ b/mlir/test/Dialect/Tosa/canonicalize.mlir
@@ -365,6 +365,14 @@ func.func @reshape_canonicalize(%arg0: tensor<?x10xf32>) -> tensor<?x10xf32> {
   return %0 : tensor<?x10xf32>
 }
 
+// CHECK-LABEL: @reshape_canonicalize_dyn_nofold
+func.func @reshape_canonicalize_dyn_nofold(%arg0: tensor<?x?x10xf32>) -> tensor<?x?x10xf32> {
+  // CHECK: %[[VAR0:.+]] = tosa.reshape %arg0 {new_shape = array<i64: -1, 2, 10>} : (tensor<?x?x10xf32>) -> tensor<?x?x10xf32>
+  // CHECK: return %[[VAR0]] : tensor<?x?x10xf32>
+  %0 = tosa.reshape %arg0 {new_shape = array<i64: -1, 2, 10>} : (tensor<?x?x10xf32>) -> tensor<?x?x10xf32>
+  return %0 : tensor<?x?x10xf32>
+}
+
 // CHECK-LABEL: @reshape_canonicalize_double
 func.func @reshape_canonicalize_double(%arg0: tensor<?x10xf32>) -> tensor<?x5xf32> {
   // CHECK: %[[VAL_1:.*]] = tosa.reshape %arg0 {new_shape = array<i64: -1, 5>}
-- 
GitLab


From 06bb8c9f202e37f215b26ca0dd9b2d8adaf5a83d Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Tue, 26 Mar 2024 16:01:57 +0100
Subject: [PATCH 286/404] [VPlan] Explicitly handle scalar pointer inductions.
 (#83068)

Add a new PtrAdd opcode to VPInstruction that corresponds to
IRBuilder::CreatePtrAdd, which creates a GEP with source element type
i8.

This is then used to model scalarizing VPWidenPointerInductionRecipe by
introducing scalar-steps to model the index increment followed by a
PtrAdd.

Note that PtrAdd needs to be able to generate code for only the first
lane or for all lanes. This may warrant introducing a separate recipe
for scalarizing that can be created without relying on the underlying
IR.

Depends on https://github.com/llvm/llvm-project/pull/80271

PR: https://github.com/llvm/llvm-project/pull/83068
---
 .../Transforms/Vectorize/LoopVectorize.cpp    |  35 +--
 llvm/lib/Transforms/Vectorize/VPlan.cpp       |   7 +-
 llvm/lib/Transforms/Vectorize/VPlan.h         |  41 ++-
 .../Transforms/Vectorize/VPlanAnalysis.cpp    |   3 +
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp |  79 +++++-
 .../Transforms/Vectorize/VPlanTransforms.cpp  |  66 ++++-
 .../versioning-scev-invalidation.ll           |   4 +-
 .../epilog-vectorization-widen-inductions.ll  | 100 ++++---
 .../LoopVectorize/AArch64/sve-epilog-vect.ll  | 148 +++++-----
 .../AArch64/sve-live-out-pointer-induction.ll |  74 ++---
 .../LoopVectorize/AArch64/sve-widen-gep.ll    |  12 +-
 .../LoopVectorize/AArch64/sve-widen-phi.ll    |  49 ++--
 .../LoopVectorize/X86/gather_scatter.ll       | 154 +++++------
 .../X86/interleave-opaque-pointers.ll         |  19 +-
 .../LoopVectorize/X86/small-size.ll           | 236 ++++++++--------
 .../LoopVectorize/consecutive-ptr-uniforms.ll |  18 +-
 .../LoopVectorize/first-order-recurrence.ll   | 261 +++++++++---------
 .../LoopVectorize/pointer-induction-unroll.ll |  61 ++--
 .../LoopVectorize/pointer-induction.ll        |  99 ++++---
 .../tail-folding-vectorization-factor-1.ll    |  25 +-
 .../vplan-printing-before-execute.ll          |  14 +-
 .../vplan-sink-scalars-and-merge.ll           |   8 +-
 .../PhaseOrdering/ARM/arm_mult_q15.ll         |   2 +-
 23 files changed, 782 insertions(+), 733 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 28748142ae92..d7be9343d830 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9111,42 +9111,11 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
          "Not a pointer induction according to InductionDescriptor!");
   assert(cast<PHINode>(getUnderlyingInstr())->getType()->isPointerTy() &&
          "Unexpected type.");
+  assert(!onlyScalarsGenerated(State.VF.isScalable()) &&
+         "Recipe should have been replaced");
 
   auto *IVR = getParent()->getPlan()->getCanonicalIV();
   PHINode *CanonicalIV = cast<PHINode>(State.get(IVR, 0, /*IsScalar*/ true));
-
-  if (onlyScalarsGenerated(State.VF.isScalable())) {
-    // This is the normalized GEP that starts counting at zero.
-    Value *PtrInd = State.Builder.CreateSExtOrTrunc(
-        CanonicalIV, IndDesc.getStep()->getType());
-    // Determine the number of scalars we need to generate for each unroll
-    // iteration. If the instruction is uniform, we only need to generate the
-    // first lane. Otherwise, we generate all VF values.
-    bool IsUniform = vputils::onlyFirstLaneUsed(this);
-    assert((IsUniform || !State.VF.isScalable()) &&
-           "Cannot scalarize a scalable VF");
-    unsigned Lanes = IsUniform ? 1 : State.VF.getFixedValue();
-
-    for (unsigned Part = 0; Part < State.UF; ++Part) {
-      Value *PartStart =
-          createStepForVF(State.Builder, PtrInd->getType(), State.VF, Part);
-
-      for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
-        Value *Idx = State.Builder.CreateAdd(
-            PartStart, ConstantInt::get(PtrInd->getType(), Lane));
-        Value *GlobalIdx = State.Builder.CreateAdd(PtrInd, Idx);
-
-        Value *Step = State.get(getOperand(1), VPIteration(Part, Lane));
-        Value *SclrGep = emitTransformedIndex(
-            State.Builder, GlobalIdx, IndDesc.getStartValue(), Step,
-            IndDesc.getKind(), IndDesc.getInductionBinOp());
-        SclrGep->setName("next.gep");
-        State.set(this, SclrGep, VPIteration(Part, Lane));
-      }
-    }
-    return;
-  }
-
   Type *PhiType = IndDesc.getStep()->getType();
 
   // Build a pointer phi
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 9768e4b7aa0a..8c664081c40c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -860,11 +860,8 @@ void VPlan::execute(VPTransformState *State) {
         Phi = cast<PHINode>(State->get(R.getVPSingleValue(), 0));
       } else {
         auto *WidenPhi = cast<VPWidenPointerInductionRecipe>(&R);
-        // TODO: Split off the case that all users of a pointer phi are scalar
-        // from the VPWidenPointerInductionRecipe.
-        if (WidenPhi->onlyScalarsGenerated(State->VF.isScalable()))
-          continue;
-
+        assert(!WidenPhi->onlyScalarsGenerated(State->VF.isScalable()) &&
+               "recipe generating only scalars should have been replaced");
         auto *GEP = cast<GetElementPtrInst>(State->get(WidenPhi, 0));
         Phi = cast<PHINode>(GEP->getPointerOperand());
       }
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index ba4992a7dcc9..f70ccf8270b2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1155,6 +1155,10 @@ public:
     BranchOnCount,
     BranchOnCond,
     ComputeReductionResult,
+    // Add an offset in bytes (second operand) to a base pointer (first
+    // operand). Only generates scalar values (either for the first lane only or
+    // for all lanes, depending on its uses).
+    PtrAdd,
   };
 
 private:
@@ -1164,11 +1168,28 @@ private:
   /// An optional name that can be used for the generated IR instruction.
   const std::string Name;
 
-  /// Utility method serving execute(): generates a single instance of the
-  /// modeled instruction. \returns the generated value for \p Part.
-  /// In some cases an existing value is returned rather than a generated
+  /// Returns true if this VPInstruction generates scalar values for all lanes.
+  /// Most VPInstructions generate a single value per part, either vector or
+  /// scalar. VPReplicateRecipe takes care of generating multiple (scalar)
+  /// values per all lanes, stemming from an original ingredient. This method
+  /// identifies the (rare) cases of VPInstructions that do so as well, w/o an
+  /// underlying ingredient.
+  bool doesGeneratePerAllLanes() const;
+
+  /// Returns true if we can generate a scalar for the first lane only if
+  /// needed.
+  bool canGenerateScalarForFirstLane() const;
+
+  /// Utility methods serving execute(): generates a single instance of the
+  /// modeled instruction for a given part. \returns the generated value for \p
+  /// Part. In some cases an existing value is returned rather than a generated
   /// one.
-  Value *generateInstruction(VPTransformState &State, unsigned Part);
+  Value *generatePerPart(VPTransformState &State, unsigned Part);
+
+  /// Utility methods serving execute(): generates a scalar single instance of
+  /// the modeled instruction for a given lane. \returns the scalar generated
+  /// value for lane \p Lane.
+  Value *generatePerLane(VPTransformState &State, const VPIteration &Lane);
 
 #if !defined(NDEBUG)
   /// Return true if the VPInstruction is a floating point math operation, i.e.
@@ -2491,12 +2512,6 @@ class VPDerivedIVRecipe : public VPSingleDefRecipe {
   /// for floating point inductions.
   const FPMathOperator *FPBinOp;
 
-  VPDerivedIVRecipe(InductionDescriptor::InductionKind Kind,
-                    const FPMathOperator *FPBinOp, VPValue *Start,
-                    VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step)
-      : VPSingleDefRecipe(VPDef::VPDerivedIVSC, {Start, CanonicalIV, Step}),
-        Kind(Kind), FPBinOp(FPBinOp) {}
-
 public:
   VPDerivedIVRecipe(const InductionDescriptor &IndDesc, VPValue *Start,
                     VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step)
@@ -2505,6 +2520,12 @@ public:
             dyn_cast_or_null<FPMathOperator>(IndDesc.getInductionBinOp()),
             Start, CanonicalIV, Step) {}
 
+  VPDerivedIVRecipe(InductionDescriptor::InductionKind Kind,
+                    const FPMathOperator *FPBinOp, VPValue *Start,
+                    VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step)
+      : VPSingleDefRecipe(VPDef::VPDerivedIVSC, {Start, CanonicalIV, Step}),
+        Kind(Kind), FPBinOp(FPBinOp) {}
+
   ~VPDerivedIVRecipe() override = default;
 
   VPRecipeBase *clone() override {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index f55beac2047c..04e30312dc23 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -44,6 +44,9 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
     CachedTypes[OtherV] = ResTy;
     return ResTy;
   }
+  case VPInstruction::PtrAdd:
+    // Return the type based on the pointer argument (i.e. first operand).
+    return inferScalarType(R->getOperand(0));
   default:
     break;
   }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f91997a37993..23d025cf33ea 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -127,6 +127,7 @@ bool VPRecipeBase::mayHaveSideEffects() const {
     case VPInstruction::Not:
     case VPInstruction::CalculateTripCountMinusVF:
     case VPInstruction::CanonicalIVIncrementForPart:
+    case VPInstruction::PtrAdd:
       return false;
     default:
       return true;
@@ -270,10 +271,39 @@ VPInstruction::VPInstruction(unsigned Opcode,
   assert(isFPMathOp() && "this op can't take fast-math flags");
 }
 
-Value *VPInstruction::generateInstruction(VPTransformState &State,
-                                          unsigned Part) {
+bool VPInstruction::doesGeneratePerAllLanes() const {
+  return Opcode == VPInstruction::PtrAdd && !vputils::onlyFirstLaneUsed(this);
+}
+
+bool VPInstruction::canGenerateScalarForFirstLane() const {
+  if (Instruction::isBinaryOp(getOpcode()))
+    return true;
+
+  switch (Opcode) {
+  case VPInstruction::BranchOnCond:
+  case VPInstruction::BranchOnCount:
+  case VPInstruction::CalculateTripCountMinusVF:
+  case VPInstruction::CanonicalIVIncrementForPart:
+  case VPInstruction::ComputeReductionResult:
+  case VPInstruction::PtrAdd:
+    return true;
+  default:
+    return false;
+  }
+}
+
+Value *VPInstruction::generatePerLane(VPTransformState &State,
+                                      const VPIteration &Lane) {
+  IRBuilderBase &Builder = State.Builder;
+
+  assert(getOpcode() == VPInstruction::PtrAdd &&
+         "only PtrAdd opcodes are supported for now");
+  return Builder.CreatePtrAdd(State.get(getOperand(0), Lane),
+                              State.get(getOperand(1), Lane), Name);
+}
+
+Value *VPInstruction::generatePerPart(VPTransformState &State, unsigned Part) {
   IRBuilderBase &Builder = State.Builder;
-  Builder.SetCurrentDebugLocation(getDebugLoc());
 
   if (Instruction::isBinaryOp(getOpcode())) {
     bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
@@ -490,6 +520,13 @@ Value *VPInstruction::generateInstruction(VPTransformState &State,
 
     return ReducedPartRdx;
   }
+  case VPInstruction::PtrAdd: {
+    assert(vputils::onlyFirstLaneUsed(this) &&
+           "can only generate first lane for PtrAdd");
+    Value *Ptr = State.get(getOperand(0), Part, /* IsScalar */ true);
+    Value *Addend = State.get(getOperand(1), Part, /* IsScalar */ true);
+    return Builder.CreatePtrAdd(Ptr, Addend, Name);
+  }
   default:
     llvm_unreachable("Unsupported opcode for instruction");
   }
@@ -514,17 +551,33 @@ void VPInstruction::execute(VPTransformState &State) {
          "Recipe not a FPMathOp but has fast-math flags?");
   if (hasFastMathFlags())
     State.Builder.setFastMathFlags(getFastMathFlags());
+  State.Builder.SetCurrentDebugLocation(getDebugLoc());
+  bool GeneratesPerFirstLaneOnly =
+      canGenerateScalarForFirstLane() &&
+      (vputils::onlyFirstLaneUsed(this) ||
+       getOpcode() == VPInstruction::ComputeReductionResult);
+  bool GeneratesPerAllLanes = doesGeneratePerAllLanes();
   for (unsigned Part = 0; Part < State.UF; ++Part) {
-    Value *GeneratedValue = generateInstruction(State, Part);
-    if (!hasResult())
+    if (GeneratesPerAllLanes) {
+      for (unsigned Lane = 0, NumLanes = State.VF.getKnownMinValue();
+           Lane != NumLanes; ++Lane) {
+        Value *GeneratedValue = generatePerLane(State, VPIteration(Part, Lane));
+        assert(GeneratedValue && "generatePerLane must produce a value");
+        State.set(this, GeneratedValue, VPIteration(Part, Lane));
+      }
       continue;
-    assert(GeneratedValue && "generateInstruction must produce a value");
+    }
 
-    bool IsVector = GeneratedValue->getType()->isVectorTy();
-    State.set(this, GeneratedValue, Part, !IsVector);
-    assert((IsVector || getOpcode() == VPInstruction::ComputeReductionResult ||
-            State.VF.isScalar() || vputils::onlyFirstLaneUsed(this)) &&
-           "scalar value but not only first lane used");
+    Value *GeneratedValue = generatePerPart(State, Part);
+    if (!hasResult())
+      continue;
+    assert(GeneratedValue && "generatePerPart must produce a value");
+    assert((GeneratedValue->getType()->isVectorTy() ==
+                !GeneratesPerFirstLaneOnly ||
+            State.VF.isScalar()) &&
+           "scalar value but not only first lane defined");
+    State.set(this, GeneratedValue, Part,
+              /*IsScalar*/ GeneratesPerFirstLaneOnly);
   }
 }
 
@@ -537,6 +590,7 @@ bool VPInstruction::onlyFirstLaneUsed(const VPValue *Op) const {
   default:
     return false;
   case Instruction::ICmp:
+  case VPInstruction::PtrAdd:
     // TODO: Cover additional opcodes.
     return vputils::onlyFirstLaneUsed(this);
   case VPInstruction::ActiveLaneMask:
@@ -594,6 +648,9 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
   case VPInstruction::ComputeReductionResult:
     O << "compute-reduction-result";
     break;
+  case VPInstruction::PtrAdd:
+    O << "ptradd";
+    break;
   default:
     O << Instruction::getOpcodeName(getOpcode());
   }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 00cec1148528..e6ab27a918bc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -498,15 +498,18 @@ static void removeDeadRecipes(VPlan &Plan) {
   }
 }
 
-static VPValue *createScalarIVSteps(VPlan &Plan, const InductionDescriptor &ID,
+static VPValue *createScalarIVSteps(VPlan &Plan,
+                                    InductionDescriptor::InductionKind Kind,
+                                    Instruction::BinaryOps InductionOpcode,
+                                    FPMathOperator *FPBinOp,
                                     ScalarEvolution &SE, Instruction *TruncI,
                                     VPValue *StartV, VPValue *Step,
                                     VPBasicBlock::iterator IP) {
   VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
   VPCanonicalIVPHIRecipe *CanonicalIV = Plan.getCanonicalIV();
   VPSingleDefRecipe *BaseIV = CanonicalIV;
-  if (!CanonicalIV->isCanonical(ID.getKind(), StartV, Step)) {
-    BaseIV = new VPDerivedIVRecipe(ID, StartV, CanonicalIV, Step);
+  if (!CanonicalIV->isCanonical(Kind, StartV, Step)) {
+    BaseIV = new VPDerivedIVRecipe(Kind, FPBinOp, StartV, CanonicalIV, Step);
     HeaderVPBB->insert(BaseIV, IP);
   }
 
@@ -536,21 +539,56 @@ static VPValue *createScalarIVSteps(VPlan &Plan, const InductionDescriptor &ID,
     VecPreheader->appendRecipe(Step->getDefiningRecipe());
   }
 
-  VPScalarIVStepsRecipe *Steps = new VPScalarIVStepsRecipe(ID, BaseIV, Step);
+  VPScalarIVStepsRecipe *Steps = new VPScalarIVStepsRecipe(
+      BaseIV, Step, InductionOpcode,
+      FPBinOp ? FPBinOp->getFastMathFlags() : FastMathFlags());
   HeaderVPBB->insert(Steps, IP);
   return Steps;
 }
 
-/// If any user of a VPWidenIntOrFpInductionRecipe needs scalar values,
-/// provide them by building scalar steps off of the canonical scalar IV and
-/// update the original IV's users. This is an optional optimization to reduce
-/// the needs of vector extracts.
-static void optimizeInductions(VPlan &Plan, ScalarEvolution &SE) {
+/// Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd
+/// (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as
+/// VPWidenPointerInductionRecipe will generate vectors only. If some users
+/// require vectors while other require scalars, the scalar uses need to extract
+/// the scalars from the generated vectors (Note that this is different to how
+/// int/fp inductions are handled). Also optimize VPWidenIntOrFpInductionRecipe,
+/// if any of its users needs scalar values, by providing them scalar steps
+/// built on the canonical scalar IV and update the original IV's users. This is
+/// an optional optimization to reduce the needs of vector extracts.
+static void legalizeAndOptimizeInductions(VPlan &Plan, ScalarEvolution &SE) {
   SmallVector<VPRecipeBase *> ToRemove;
   VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
   bool HasOnlyVectorVFs = !Plan.hasVF(ElementCount::getFixed(1));
   VPBasicBlock::iterator InsertPt = HeaderVPBB->getFirstNonPhi();
   for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
+    // Replace wide pointer inductions which have only their scalars used by
+    // PtrAdd(IndStart, ScalarIVSteps (0, Step)).
+    if (auto *PtrIV = dyn_cast<VPWidenPointerInductionRecipe>(&Phi)) {
+      if (!PtrIV->onlyScalarsGenerated(Plan.hasScalableVF()))
+        continue;
+
+      const InductionDescriptor &ID = PtrIV->getInductionDescriptor();
+      VPValue *StartV = Plan.getVPValueOrAddLiveIn(
+          ConstantInt::get(ID.getStep()->getType(), 0));
+      VPValue *StepV = PtrIV->getOperand(1);
+      VPRecipeBase *Steps =
+          createScalarIVSteps(Plan, InductionDescriptor::IK_IntInduction,
+                              Instruction::Add, nullptr, SE, nullptr, StartV,
+                              StepV, InsertPt)
+              ->getDefiningRecipe();
+
+      auto *Recipe =
+          new VPInstruction(VPInstruction::PtrAdd,
+                            {PtrIV->getStartValue(), Steps->getVPSingleValue()},
+                            PtrIV->getDebugLoc(), "next.gep");
+
+      Recipe->insertAfter(Steps);
+      PtrIV->replaceAllUsesWith(Recipe);
+      continue;
+    }
+
+    // Replace widened induction with scalar steps for users that only use
+    // scalars.
     auto *WideIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
     if (!WideIV)
       continue;
@@ -560,9 +598,11 @@ static void optimizeInductions(VPlan &Plan, ScalarEvolution &SE) {
       continue;
 
     const InductionDescriptor &ID = WideIV->getInductionDescriptor();
-    VPValue *Steps = createScalarIVSteps(Plan, ID, SE, WideIV->getTruncInst(),
-                                         WideIV->getStartValue(),
-                                         WideIV->getStepValue(), InsertPt);
+    VPValue *Steps = createScalarIVSteps(
+        Plan, ID.getKind(), ID.getInductionOpcode(),
+        dyn_cast_or_null<FPMathOperator>(ID.getInductionBinOp()), SE,
+        WideIV->getTruncInst(), WideIV->getStartValue(), WideIV->getStepValue(),
+        InsertPt);
 
     // Update scalar users of IV to use Step instead.
     if (!HasOnlyVectorVFs)
@@ -1025,7 +1065,7 @@ void VPlanTransforms::optimize(VPlan &Plan, ScalarEvolution &SE) {
   removeRedundantInductionCasts(Plan);
 
   simplifyRecipes(Plan, SE.getContext());
-  optimizeInductions(Plan, SE);
+  legalizeAndOptimizeInductions(Plan, SE);
   removeDeadRecipes(Plan);
 
   createAndOptimizeReplicateRegions(Plan);
diff --git a/llvm/test/Transforms/LoopLoadElim/versioning-scev-invalidation.ll b/llvm/test/Transforms/LoopLoadElim/versioning-scev-invalidation.ll
index 4b0bc908eddf..63470a943515 100644
--- a/llvm/test/Transforms/LoopLoadElim/versioning-scev-invalidation.ll
+++ b/llvm/test/Transforms/LoopLoadElim/versioning-scev-invalidation.ll
@@ -63,8 +63,8 @@ define void @g(ptr %dst.1, ptr %start, i64 %N) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[LCSSA_PTR_IV_1]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr double, ptr [[NEXT_GEP]], i32 0
 ; CHECK-NEXT:    store <4 x double> zeroinitializer, ptr [[TMP5]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll
index 24c59fdb47b6..00ec396107dc 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll
@@ -11,76 +11,74 @@ define void @test_widen_ptr_induction(ptr %ptr.start.1) {
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[PTR_START_1:%.*]], i64 10000
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PTR_START_1]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[PTR_START_1]], i64 [[TMP1]]
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x ptr> poison, ptr [[NEXT_GEP]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x ptr> [[TMP2]], ptr [[NEXT_GEP1]], i32 1
-; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[PTR_START_1]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[PTR_START_1]], i64 [[TMP5]]
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[NEXT_GEP2]], i32 0
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[NEXT_GEP3]], i32 1
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <2 x ptr> [[TMP3]], zeroinitializer
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <2 x ptr> [[TMP7]], zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP8]], i32 0
-; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP10]])
-; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i1> [[TMP8]], i32 1
-; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP11]])
-; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i1> [[TMP9]], i32 0
-; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP12]])
-; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i1> [[TMP9]], i32 1
-; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP13]])
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0
-; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 2
-; CHECK-NEXT:    store <2 x i8> zeroinitializer, ptr [[TMP14]], align 1
-; CHECK-NEXT:    store <2 x i8> zeroinitializer, ptr [[TMP15]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[PTR_START_1:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[PTR_START_1]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[PTR_START_1]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[PTR_START_1]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x ptr> [[TMP10]], ptr [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <2 x ptr> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne <2 x ptr> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i1> [[TMP12]], i32 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP14]])
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i1> [[TMP12]], i32 1
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP15]])
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i1> [[TMP13]], i32 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP16]])
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i1> [[TMP13]], i32 1
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP17]])
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr i8, ptr [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[TMP4]], i32 2
+; CHECK-NEXT:    store <2 x i8> zeroinitializer, ptr [[TMP18]], align 1
+; CHECK-NEXT:    store <2 x i8> zeroinitializer, ptr [[TMP19]], align 1
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
-; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
+; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
 ; CHECK:       vec.epilog.iter.check:
-; CHECK-NEXT:    [[IND_END6:%.*]] = getelementptr i8, ptr [[PTR_START_1]], i64 10000
+; CHECK-NEXT:    [[IND_END1:%.*]] = getelementptr i8, ptr [[PTR_START_1]], i64 10000
 ; CHECK-NEXT:    br i1 true, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PTR_START_1]], [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT:    [[IND_END5:%.*]] = getelementptr i8, ptr [[PTR_START_1]], i64 10000
+; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[PTR_START_1]], i64 10000
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
-; CHECK-NEXT:    [[INDEX8:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX8]], 0
-; CHECK-NEXT:    [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[PTR_START_1]], i64 [[TMP17]]
-; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[INDEX8]], 1
-; CHECK-NEXT:    [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[PTR_START_1]], i64 [[TMP18]]
-; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <2 x ptr> poison, ptr [[NEXT_GEP9]], i32 0
-; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <2 x ptr> [[TMP19]], ptr [[NEXT_GEP10]], i32 1
-; CHECK-NEXT:    [[TMP21:%.*]] = icmp ne <2 x ptr> [[TMP20]], zeroinitializer
-; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x i1> [[TMP21]], i32 0
-; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP22]])
-; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <2 x i1> [[TMP21]], i32 1
-; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP23]])
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[NEXT_GEP9]], i32 0
-; CHECK-NEXT:    store <2 x i8> zeroinitializer, ptr [[TMP24]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT11]] = add nuw i64 [[INDEX8]], 2
-; CHECK-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT11]], 10000
-; CHECK-NEXT:    br i1 [[TMP25]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
+; CHECK-NEXT:    [[INDEX3:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT4:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[INDEX3]], 0
+; CHECK-NEXT:    [[TMP22:%.*]] = add i64 [[INDEX3]], 1
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr i8, ptr [[PTR_START_1]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[PTR_START_1]], i64 [[TMP22]]
+; CHECK-NEXT:    [[TMP25:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP23]], i32 0
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <2 x ptr> [[TMP25]], ptr [[TMP24]], i32 1
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp ne <2 x ptr> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <2 x i1> [[TMP27]], i32 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP28]])
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <2 x i1> [[TMP27]], i32 1
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP29]])
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[TMP23]], i32 0
+; CHECK-NEXT:    store <2 x i8> zeroinitializer, ptr [[TMP30]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT4]] = add nuw i64 [[INDEX3]], 2
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT4]], 10000
+; CHECK-NEXT:    br i1 [[TMP31]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], {{!llvm.loop ![0-9]+}}
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 10000, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL7:%.*]] = phi ptr [ [[IND_END5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END6]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PTR_START_1]], [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 10000, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END1]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PTR_START_1]], [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL4]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL7]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL2]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[CMP_I_I_I_I:%.*]] = icmp ne ptr [[PTR_IV]], null
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP_I_I_I_I]])
 ; CHECK-NEXT:    store i8 0, ptr [[PTR_IV]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll
index 24d2127ee171..12889c2acc8e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll
@@ -146,13 +146,13 @@ define void @main_vf_vscale_x_16(ptr %A) #0 {
 ; CHECK-VF8-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-VF8-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK-VF8:       vec.epilog.vector.body:
-; CHECK-VF8-NEXT:    [[INDEX2:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-VF8-NEXT:    [[TMP19:%.*]] = add i64 [[INDEX2]], 0
+; CHECK-VF8-NEXT:    [[INDEX1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VF8-NEXT:    [[TMP19:%.*]] = add i64 [[INDEX1]], 0
 ; CHECK-VF8-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP19]]
 ; CHECK-VF8-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP20]], i32 0
 ; CHECK-VF8-NEXT:    store <8 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>, ptr [[TMP21]], align 1
-; CHECK-VF8-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX2]], 8
-; CHECK-VF8-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 1024
+; CHECK-VF8-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 8
+; CHECK-VF8-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 1024
 ; CHECK-VF8-NEXT:    br i1 [[TMP22]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK-VF8:       vec.epilog.middle.block:
 ; CHECK-VF8-NEXT:    br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
@@ -246,13 +246,13 @@ define void @main_vf_vscale_x_2(ptr %A) #0 vscale_range(8, 8) {
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
-; CHECK-NEXT:    [[INDEX2:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP19:%.*]] = add i64 [[INDEX2]], 0
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP19:%.*]] = add i64 [[INDEX1]], 0
 ; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP19]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[TMP20]], i32 0
 ; CHECK-NEXT:    store <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>, ptr [[TMP21]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX2]], 8
-; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 1024
+; CHECK-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 8
+; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 1024
 ; CHECK-NEXT:    br i1 [[TMP22]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
@@ -315,13 +315,13 @@ define void @main_vf_vscale_x_2(ptr %A) #0 vscale_range(8, 8) {
 ; CHECK-VF8-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-VF8-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK-VF8:       vec.epilog.vector.body:
-; CHECK-VF8-NEXT:    [[INDEX2:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-VF8-NEXT:    [[TMP19:%.*]] = add i64 [[INDEX2]], 0
+; CHECK-VF8-NEXT:    [[INDEX1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VF8-NEXT:    [[TMP19:%.*]] = add i64 [[INDEX1]], 0
 ; CHECK-VF8-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP19]]
 ; CHECK-VF8-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[TMP20]], i32 0
 ; CHECK-VF8-NEXT:    store <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>, ptr [[TMP21]], align 1
-; CHECK-VF8-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX2]], 8
-; CHECK-VF8-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 1024
+; CHECK-VF8-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 8
+; CHECK-VF8-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 1024
 ; CHECK-VF8-NEXT:    br i1 [[TMP22]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-VF8:       vec.epilog.middle.block:
 ; CHECK-VF8-NEXT:    br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
@@ -374,66 +374,65 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 10000, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 32
-; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[N_VEC]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP8]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 16
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], [[TMP11]]
-; CHECK-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP12]]
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0
-; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 16
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 [[TMP15]]
-; CHECK-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP13]], align 1
+; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 1
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[INDEX]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[TMP14]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP17]], 16
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP18]]
 ; CHECK-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP16]], align 1
+; CHECK-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP19]], align 1
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 10000, [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
 ; CHECK:       vec.epilog.iter.check:
-; CHECK-NEXT:    [[IND_END7:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC]]
+; CHECK-NEXT:    [[IND_END4:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC]]
 ; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 10000, [[N_VEC]]
-; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 8
-; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP21]], 8
+; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], [[TMP22]]
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 8
-; CHECK-NEXT:    [[N_MOD_VF3:%.*]] = urem i64 10000, [[TMP21]]
-; CHECK-NEXT:    [[N_VEC4:%.*]] = sub i64 10000, [[N_MOD_VF3]]
-; CHECK-NEXT:    [[IND_END6:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC4]]
-; CHECK-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 8
+; CHECK-NEXT:    [[TMP23:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP24:%.*]] = mul i64 [[TMP23]], 8
+; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 10000, [[TMP24]]
+; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 10000, [[N_MOD_VF2]]
+; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC3]]
+; CHECK-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 8
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
-; CHECK-NEXT:    [[INDEX10:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT12:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP24:%.*]] = add i64 [[INDEX10]], 0
-; CHECK-NEXT:    [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP24]]
-; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[NEXT_GEP11]], i32 0
-; CHECK-NEXT:    store <vscale x 8 x i8> zeroinitializer, ptr [[TMP25]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT12]] = add nuw i64 [[INDEX10]], [[TMP23]]
-; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT12]], [[N_VEC4]]
-; CHECK-NEXT:    br i1 [[TMP26]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP27:%.*]] = add i64 [[INDEX7]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP28]], i32 0
+; CHECK-NEXT:    store <vscale x 8 x i8> zeroinitializer, ptr [[TMP29]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT8]] = add nuw i64 [[INDEX7]], [[TMP26]]
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT8]], [[N_VEC3]]
+; CHECK-NEXT:    br i1 [[TMP30]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       vec.epilog.middle.block:
-; CHECK-NEXT:    [[CMP_N9:%.*]] = icmp eq i64 10000, [[N_VEC4]]
-; CHECK-NEXT:    br i1 [[CMP_N9]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
+; CHECK-NEXT:    [[CMP_N6:%.*]] = icmp eq i64 10000, [[N_VEC3]]
+; CHECK-NEXT:    br i1 [[CMP_N6]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi i64 [ [[N_VEC4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi ptr [ [[IND_END6]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END7]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL5]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL8]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL5]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    store i8 0, ptr [[PTR_IV]], align 1
 ; CHECK-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i8, ptr [[PTR_IV]], i64 1
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
@@ -457,57 +456,56 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
 ; CHECK-VF8-NEXT:    [[N_VEC:%.*]] = sub i64 10000, [[N_MOD_VF]]
 ; CHECK-VF8-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-VF8-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 32
-; CHECK-VF8-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[N_VEC]]
 ; CHECK-VF8-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-VF8:       vector.body:
 ; CHECK-VF8-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-VF8-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 0
-; CHECK-VF8-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP6]]
 ; CHECK-VF8-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-VF8-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 16
 ; CHECK-VF8-NEXT:    [[TMP9:%.*]] = add i64 [[TMP8]], 0
-; CHECK-VF8-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP9]]
-; CHECK-VF8-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP10]]
-; CHECK-VF8-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0
-; CHECK-VF8-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 16
-; CHECK-VF8-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 [[TMP13]]
-; CHECK-VF8-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP11]], align 1
+; CHECK-VF8-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 1
+; CHECK-VF8-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]]
+; CHECK-VF8-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[TMP6]]
+; CHECK-VF8-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP11]]
+; CHECK-VF8-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[TMP12]], i32 0
+; CHECK-VF8-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-VF8-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 16
+; CHECK-VF8-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP16]]
 ; CHECK-VF8-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP14]], align 1
+; CHECK-VF8-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP17]], align 1
 ; CHECK-VF8-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-VF8-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-VF8-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-VF8-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-VF8-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK-VF8:       middle.block:
 ; CHECK-VF8-NEXT:    [[CMP_N:%.*]] = icmp eq i64 10000, [[N_VEC]]
 ; CHECK-VF8-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
 ; CHECK-VF8:       vec.epilog.iter.check:
-; CHECK-VF8-NEXT:    [[IND_END4:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC]]
+; CHECK-VF8-NEXT:    [[IND_END1:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC]]
 ; CHECK-VF8-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 10000, [[N_VEC]]
 ; CHECK-VF8-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 8
 ; CHECK-VF8-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK-VF8:       vec.epilog.ph:
-; CHECK-VF8-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-VF8-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-VF8-NEXT:    [[IND_END3:%.*]] = getelementptr i8, ptr [[START]], i64 10000
+; CHECK-VF8-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[START]], i64 10000
 ; CHECK-VF8-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK-VF8:       vec.epilog.vector.body:
-; CHECK-VF8-NEXT:    [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
-; CHECK-VF8-NEXT:    [[TMP16:%.*]] = add i64 [[INDEX7]], 0
-; CHECK-VF8-NEXT:    [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP16]]
-; CHECK-VF8-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i32 0
-; CHECK-VF8-NEXT:    store <8 x i8> zeroinitializer, ptr [[TMP17]], align 1
-; CHECK-VF8-NEXT:    [[INDEX_NEXT9]] = add nuw i64 [[INDEX7]], 8
-; CHECK-VF8-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT9]], 10000
-; CHECK-VF8-NEXT:    br i1 [[TMP18]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-VF8-NEXT:    [[INDEX3:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT4:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-VF8-NEXT:    [[TMP19:%.*]] = add i64 [[INDEX3]], 0
+; CHECK-VF8-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP19]]
+; CHECK-VF8-NEXT:    [[TMP21:%.*]] = getelementptr i8, ptr [[TMP20]], i32 0
+; CHECK-VF8-NEXT:    store <8 x i8> zeroinitializer, ptr [[TMP21]], align 1
+; CHECK-VF8-NEXT:    [[INDEX_NEXT4]] = add nuw i64 [[INDEX3]], 8
+; CHECK-VF8-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT4]], 10000
+; CHECK-VF8-NEXT:    br i1 [[TMP22]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK-VF8:       vec.epilog.middle.block:
 ; CHECK-VF8-NEXT:    br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK-VF8:       vec.epilog.scalar.ph:
-; CHECK-VF8-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; CHECK-VF8-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[IND_END3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK]] ]
+; CHECK-VF8-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-VF8-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END1]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK]] ]
 ; CHECK-VF8-NEXT:    br label [[LOOP:%.*]]
 ; CHECK-VF8:       loop:
-; CHECK-VF8-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL2]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-VF8-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL5]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-VF8-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-VF8-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL2]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-VF8-NEXT:    store i8 0, ptr [[PTR_IV]], align 1
 ; CHECK-VF8-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i8, ptr [[PTR_IV]], i64 1
 ; CHECK-VF8-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll
index cfb0f9e59ecb..8b64d7a08366 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll
@@ -23,54 +23,54 @@ define ptr @test(ptr %start.1, ptr %start.2, ptr %end) {
 ; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[START_1:%.*]], i64 [[TMP8]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[N_VEC]], 8
 ; CHECK-NEXT:    [[IND_END3:%.*]] = getelementptr i8, ptr [[START_2]], i64 [[TMP9]]
-; CHECK-NEXT:    [[TMP33:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP34:%.*]] = mul i64 [[TMP33]], 4
+; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[START_1]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
-; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 2
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 8, [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP11]], 0
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP14]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 2
+; CHECK-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 2
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 8, [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP13]], 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP16]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP15:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = add <vscale x 2 x i64> [[DOTSPLAT]], [[TMP15]]
-; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = mul <vscale x 2 x i64> [[TMP16]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 8, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[VECTOR_GEP]]
-; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP11]], 1
-; CHECK-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP18]], i64 0
+; CHECK-NEXT:    [[TMP17:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+; CHECK-NEXT:    [[TMP18:%.*]] = add <vscale x 2 x i64> [[DOTSPLAT]], [[TMP17]]
+; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = mul <vscale x 2 x i64> [[TMP18]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 8, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[VECTOR_GEP]]
+; CHECK-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP13]], 1
+; CHECK-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP20]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT6:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT5]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP19:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
-; CHECK-NEXT:    [[TMP20:%.*]] = add <vscale x 2 x i64> [[DOTSPLAT6]], [[TMP19]]
-; CHECK-NEXT:    [[VECTOR_GEP7:%.*]] = mul <vscale x 2 x i64> [[TMP20]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 8, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[VECTOR_GEP7]]
-; CHECK-NEXT:    [[TMP22:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 8
-; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START_2]], i64 [[TMP23]]
-; CHECK-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP24]], 2
-; CHECK-NEXT:    [[TMP26:%.*]] = add i64 [[TMP25]], 0
-; CHECK-NEXT:    [[TMP27:%.*]] = add i64 [[INDEX]], [[TMP26]]
+; CHECK-NEXT:    [[TMP21:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
+; CHECK-NEXT:    [[TMP22:%.*]] = add <vscale x 2 x i64> [[DOTSPLAT6]], [[TMP21]]
+; CHECK-NEXT:    [[VECTOR_GEP7:%.*]] = mul <vscale x 2 x i64> [[TMP22]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 8, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[VECTOR_GEP7]]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP24:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 2
+; CHECK-NEXT:    [[TMP27:%.*]] = add i64 [[TMP26]], 0
 ; CHECK-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 8
-; CHECK-NEXT:    [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[START_2]], i64 [[TMP28]]
-; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr i64, ptr [[NEXT_GEP]], i32 0
-; CHECK-NEXT:    [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP31:%.*]] = mul i64 [[TMP30]], 2
-; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr i64, ptr [[NEXT_GEP]], i64 [[TMP31]]
-; CHECK-NEXT:    store <vscale x 2 x i64> zeroinitializer, ptr [[TMP29]], align 8
+; CHECK-NEXT:    [[TMP29:%.*]] = add i64 [[OFFSET_IDX]], [[TMP28]]
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[START_2]], i64 [[TMP24]]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[START_2]], i64 [[TMP29]]
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr i64, ptr [[TMP30]], i32 0
+; CHECK-NEXT:    [[TMP33:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP34:%.*]] = mul i64 [[TMP33]], 2
+; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr i64, ptr [[TMP30]], i64 [[TMP34]]
 ; CHECK-NEXT:    store <vscale x 2 x i64> zeroinitializer, ptr [[TMP32]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP34]]
-; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP13]]
-; CHECK-NEXT:    [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP35]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    store <vscale x 2 x i64> zeroinitializer, ptr [[TMP35]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
+; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
 ; CHECK-NEXT:    [[CMO:%.*]] = sub i64 [[N_VEC]], 1
-; CHECK-NEXT:    [[TMP36:%.*]] = mul i64 [[CMO]], 8
-; CHECK-NEXT:    [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[START_1]], i64 [[TMP36]]
+; CHECK-NEXT:    [[TMP37:%.*]] = mul i64 [[CMO]], 8
+; CHECK-NEXT:    [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[START_1]], i64 [[TMP37]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[START_1]], [[ENTRY:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
index 899fcce5c02a..3bab341e1c24 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
@@ -19,10 +19,12 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-NEXT:   <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
 ; CHECK-NEXT:     EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT:     EMIT ir<%ptr.iv.1> = WIDEN-POINTER-INDUCTION ir<%start.1>, 8
 ; CHECK-NEXT:     EMIT ir<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, 1
+; CHECK-NEXT:     vp<[[PTR_IDX:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * ir<8>
+; CHECK-NEXT:     vp<[[PTR_IDX_STEPS:%.+]]> = SCALAR-STEPS vp<[[PTR_IDX]]>, ir<8>
+; CHECK-NEXT:     EMIT vp<[[PTR_IV_1:%.+]]> = ptradd ir<%start.1>, vp<[[PTR_IDX_STEPS]]>
 ; CHECK-NEXT:     WIDEN-GEP Var[Inv] ir<%ptr.iv.2.next> = getelementptr inbounds ir<%ptr.iv.2>, ir<1>
-; CHECK-NEXT:     vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%ptr.iv.1>
+; CHECK-NEXT:     vp<[[VEC_PTR:%.+]]> = vector-pointer vp<[[PTR_IV_1]]>
 ; CHECK-NEXT:     WIDEN store vp<[[VEC_PTR]]>, ir<%ptr.iv.2.next>
 ; CHECK-NEXT:     vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%ptr.iv.2>
 ; CHECK-NEXT:     WIDEN ir<%lv> = load vp<[[VEC_PTR2]]>
@@ -59,9 +61,6 @@ define void @pointer_induction_used_as_vector(ptr noalias %start.1, ptr noalias
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[START_2]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
-; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START_1]], i64 [[TMP6]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
 ; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 1
@@ -73,6 +72,9 @@ define void @pointer_induction_used_as_vector(ptr noalias %start.1, ptr noalias
 ; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 2 x i64> [[DOTSPLAT]], [[TMP12]]
 ; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = mul <vscale x 2 x i64> [[TMP13]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 1, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[VECTOR_GEP]]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START_1]], i64 [[TMP5]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, <vscale x 2 x ptr> [[TMP14]], i64 1
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr ptr, ptr [[NEXT_GEP]], i32 0
 ; CHECK-NEXT:    store <vscale x 2 x ptr> [[TMP15]], ptr [[TMP16]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll
index 7226048c478d..126ceac7325a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll
@@ -32,21 +32,20 @@ define void @widen_ptr_phi_unrolled(ptr noalias nocapture %a, ptr noalias nocapt
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 3
-; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 5
-; CHECK-NEXT:    [[TMP9:%.*]] = shl i64 [[INDEX]], 3
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[C]], i64 [[TMP8]]
-; CHECK-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP9]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[NEXT_GEP]], align 4
-; CHECK-NEXT:    [[WIDE_VEC3:%.*]] = load <vscale x 8 x i32>, ptr [[NEXT_GEP2]], align 4
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 5
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[C]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[C]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i64 [[TMP7]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[WIDE_VEC2:%.*]] = load <vscale x 8 x i32>, ptr [[TMP10]], align 4
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.experimental.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
-; CHECK-NEXT:    [[STRIDED_VEC4:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.experimental.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC3]])
-; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC4]], 0
-; CHECK-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC4]], 1
+; CHECK-NEXT:    [[STRIDED_VEC3:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.experimental.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC2]])
+; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC3]], 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC3]], 1
 ; CHECK-NEXT:    [[TMP15:%.*]] = add nsw <vscale x 4 x i32> [[TMP11]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[TMP16:%.*]] = add nsw <vscale x 4 x i32> [[TMP13]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
@@ -148,21 +147,21 @@ define void @widen_2ptrs_phi_unrolled(ptr noalias nocapture %dst, ptr noalias no
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP7]]
-; CHECK-NEXT:    [[TMP8:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[OFFSET_IDX4:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[OFFSET_IDX4]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP10:%.*]] = shl nuw nsw i64 [[TMP9]], 2
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i32, ptr [[NEXT_GEP]], i64 [[TMP10]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[NEXT_GEP]], align 4
-; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 4 x i32>, ptr [[TMP11]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i32, ptr [[TMP7]], i64 [[TMP10]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 4 x i32>, ptr [[TMP11]], align 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = shl nsw <vscale x 4 x i32> [[WIDE_LOAD]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP13:%.*]] = shl nsw <vscale x 4 x i32> [[WIDE_LOAD7]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP13:%.*]] = shl nsw <vscale x 4 x i32> [[WIDE_LOAD5]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr i32, ptr [[NEXT_GEP5]], i64 [[TMP15]]
-; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP12]], ptr [[NEXT_GEP5]], align 4
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr i32, ptr [[TMP8]], i64 [[TMP15]]
+; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP12]], ptr [[TMP8]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP13]], ptr [[TMP16]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -246,12 +245,12 @@ define i32 @pointer_iv_mixed(ptr noalias %a, ptr noalias %b, i64 %n) #0 {
 ; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.experimental.stepvector.nxv2i64()
 ; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = shl <vscale x 2 x i64> [[TMP9]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 2, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[VECTOR_GEP]]
-; CHECK-NEXT:    [[TMP11:%.*]] = shl i64 [[INDEX]], 3
-; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP11]]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[B]], i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <vscale x 2 x ptr> [[TMP10]], i64 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i32>, ptr [[TMP12]], align 8
 ; CHECK-NEXT:    [[TMP13]] = add <vscale x 2 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT:    store <vscale x 2 x ptr> [[TMP10]], ptr [[NEXT_GEP]], align 8
+; CHECK-NEXT:    store <vscale x 2 x ptr> [[TMP10]], ptr [[TMP11]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP8]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll
index e9541c1ee035..6516b05ab4ed 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll
@@ -639,87 +639,84 @@ define void @test_gather_not_profitable_pr48429(i32 %d, ptr readonly noalias %pt
 ; AVX512:       vector.ph:
 ; AVX512-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 16
 ; AVX512-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
-; AVX512-NEXT:    [[TMP13:%.*]] = mul i64 [[N_VEC]], 4
-; AVX512-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP13]]
-; AVX512-NEXT:    [[TMP14:%.*]] = mul i64 [[N_VEC]], 64
-; AVX512-NEXT:    [[IND_END9:%.*]] = getelementptr i8, ptr [[DEST]], i64 [[TMP14]]
+; AVX512-NEXT:    [[TMP13:%.*]] = mul i64 [[N_VEC]], 64
+; AVX512-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[DEST]], i64 [[TMP13]]
 ; AVX512-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; AVX512:       vector.body:
 ; AVX512-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[DEST]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
 ; AVX512-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; AVX512-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 0
-; AVX512-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 4
-; AVX512-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP16]]
-; AVX512-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <16 x i64> <i64 0, i64 64, i64 128, i64 192, i64 256, i64 320, i64 384, i64 448, i64 512, i64 576, i64 640, i64 704, i64 768, i64 832, i64 896, i64 960>
-; AVX512-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[NEXT_GEP]], i64 [[IDXPROM]]
-; AVX512-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i32 0
-; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x float>, ptr [[TMP19]], align 4, !alias.scope !8
-; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> [[WIDE_LOAD]], <16 x ptr> [[TMP17]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !11, !noalias !13
-; AVX512-NEXT:    [[TMP20:%.*]] = getelementptr float, ptr [[NEXT_GEP]], i32 0
-; AVX512-NEXT:    [[WIDE_LOAD8:%.*]] = load <16 x float>, ptr [[TMP20]], align 4, !alias.scope !15
-; AVX512-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, <16 x ptr> [[TMP17]], i64 1
-; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> [[WIDE_LOAD8]], <16 x ptr> [[TMP21]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !11, !noalias !13
+; AVX512-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <16 x i64> <i64 0, i64 64, i64 128, i64 192, i64 256, i64 320, i64 384, i64 448, i64 512, i64 576, i64 640, i64 704, i64 768, i64 832, i64 896, i64 960>
+; AVX512-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 4
+; AVX512-NEXT:    [[TMP15:%.*]] = add i64 [[OFFSET_IDX]], 0
+; AVX512-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP15]]
+; AVX512-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[IDXPROM]]
+; AVX512-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i32 0
+; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x float>, ptr [[TMP18]], align 4, !alias.scope [[META8:![0-9]+]]
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> [[WIDE_LOAD]], <16 x ptr> [[TMP14]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope [[META11:![0-9]+]], !noalias [[META13:![0-9]+]]
+; AVX512-NEXT:    [[TMP19:%.*]] = getelementptr float, ptr [[TMP16]], i32 0
+; AVX512-NEXT:    [[WIDE_LOAD8:%.*]] = load <16 x float>, ptr [[TMP19]], align 4, !alias.scope [[META15:![0-9]+]]
+; AVX512-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, <16 x ptr> [[TMP14]], i64 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> [[WIDE_LOAD8]], <16 x ptr> [[TMP20]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope [[META11]], !noalias [[META13]]
 ; AVX512-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; AVX512-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 1024
-; AVX512-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; AVX512-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; AVX512-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX512-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; AVX512:       middle.block:
 ; AVX512-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
 ; AVX512-NEXT:    br i1 [[CMP_N]], label [[FOR_END]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
 ; AVX512:       vec.epilog.iter.check:
-; AVX512-NEXT:    [[TMP23:%.*]] = mul i64 [[N_VEC]], 64
-; AVX512-NEXT:    [[IND_END17:%.*]] = getelementptr i8, ptr [[DEST]], i64 [[TMP23]]
-; AVX512-NEXT:    [[TMP24:%.*]] = mul i64 [[N_VEC]], 4
-; AVX512-NEXT:    [[IND_END14:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP24]]
+; AVX512-NEXT:    [[TMP22:%.*]] = mul i64 [[N_VEC]], 64
+; AVX512-NEXT:    [[IND_END15:%.*]] = getelementptr i8, ptr [[DEST]], i64 [[TMP22]]
+; AVX512-NEXT:    [[TMP23:%.*]] = mul i64 [[N_VEC]], 4
+; AVX512-NEXT:    [[IND_END12:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP23]]
 ; AVX512-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[TMP3]], [[N_VEC]]
 ; AVX512-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 8
 ; AVX512-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; AVX512:       vec.epilog.ph:
-; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PTR]], [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; AVX512-NEXT:    [[BC_RESUME_VAL10:%.*]] = phi ptr [ [[IND_END9]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[DEST]], [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[DEST]], [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; AVX512-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; AVX512-NEXT:    [[N_MOD_VF11:%.*]] = urem i64 [[TMP3]], 8
-; AVX512-NEXT:    [[N_VEC12:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF11]]
-; AVX512-NEXT:    [[TMP25:%.*]] = mul i64 [[N_VEC12]], 4
-; AVX512-NEXT:    [[IND_END13:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP25]]
-; AVX512-NEXT:    [[TMP26:%.*]] = mul i64 [[N_VEC12]], 64
-; AVX512-NEXT:    [[IND_END16:%.*]] = getelementptr i8, ptr [[DEST]], i64 [[TMP26]]
+; AVX512-NEXT:    [[N_MOD_VF9:%.*]] = urem i64 [[TMP3]], 8
+; AVX512-NEXT:    [[N_VEC10:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF9]]
+; AVX512-NEXT:    [[TMP24:%.*]] = mul i64 [[N_VEC10]], 4
+; AVX512-NEXT:    [[IND_END11:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP24]]
+; AVX512-NEXT:    [[TMP25:%.*]] = mul i64 [[N_VEC10]], 64
+; AVX512-NEXT:    [[IND_END14:%.*]] = getelementptr i8, ptr [[DEST]], i64 [[TMP25]]
 ; AVX512-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; AVX512:       vec.epilog.vector.body:
-; AVX512-NEXT:    [[POINTER_PHI22:%.*]] = phi ptr [ [[BC_RESUME_VAL10]], [[VEC_EPILOG_PH]] ], [ [[PTR_IND23:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
-; AVX512-NEXT:    [[INDEX20:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT26:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
-; AVX512-NEXT:    [[TMP27:%.*]] = add i64 [[INDEX20]], 0
-; AVX512-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 4
-; AVX512-NEXT:    [[NEXT_GEP21:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP28]]
-; AVX512-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[POINTER_PHI22]], <8 x i64> <i64 0, i64 64, i64 128, i64 192, i64 256, i64 320, i64 384, i64 448>
-; AVX512-NEXT:    [[TMP30:%.*]] = getelementptr inbounds float, ptr [[NEXT_GEP21]], i64 [[IDXPROM]]
-; AVX512-NEXT:    [[TMP31:%.*]] = getelementptr inbounds float, ptr [[TMP30]], i32 0
-; AVX512-NEXT:    [[WIDE_LOAD24:%.*]] = load <8 x float>, ptr [[TMP31]], align 4, !alias.scope !17
-; AVX512-NEXT:    call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> [[WIDE_LOAD24]], <8 x ptr> [[TMP29]], i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !20, !noalias !22
-; AVX512-NEXT:    [[TMP32:%.*]] = getelementptr float, ptr [[NEXT_GEP21]], i32 0
-; AVX512-NEXT:    [[WIDE_LOAD25:%.*]] = load <8 x float>, ptr [[TMP32]], align 4, !alias.scope !24
-; AVX512-NEXT:    [[TMP33:%.*]] = getelementptr inbounds float, <8 x ptr> [[TMP29]], i64 1
-; AVX512-NEXT:    call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> [[WIDE_LOAD25]], <8 x ptr> [[TMP33]], i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !20, !noalias !22
-; AVX512-NEXT:    [[INDEX_NEXT26]] = add nuw i64 [[INDEX20]], 8
-; AVX512-NEXT:    [[PTR_IND23]] = getelementptr i8, ptr [[POINTER_PHI22]], i64 512
-; AVX512-NEXT:    [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT26]], [[N_VEC12]]
-; AVX512-NEXT:    br i1 [[TMP34]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; AVX512-NEXT:    [[POINTER_PHI19:%.*]] = phi ptr [ [[BC_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[PTR_IND20:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; AVX512-NEXT:    [[INDEX18:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT24:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; AVX512-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[POINTER_PHI19]], <8 x i64> <i64 0, i64 64, i64 128, i64 192, i64 256, i64 320, i64 384, i64 448>
+; AVX512-NEXT:    [[OFFSET_IDX21:%.*]] = mul i64 [[INDEX18]], 4
+; AVX512-NEXT:    [[TMP27:%.*]] = add i64 [[OFFSET_IDX21]], 0
+; AVX512-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP27]]
+; AVX512-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, ptr [[TMP28]], i64 [[IDXPROM]]
+; AVX512-NEXT:    [[TMP30:%.*]] = getelementptr inbounds float, ptr [[TMP29]], i32 0
+; AVX512-NEXT:    [[WIDE_LOAD22:%.*]] = load <8 x float>, ptr [[TMP30]], align 4, !alias.scope [[META17:![0-9]+]]
+; AVX512-NEXT:    call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> [[WIDE_LOAD22]], <8 x ptr> [[TMP26]], i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope [[META20:![0-9]+]], !noalias [[META22:![0-9]+]]
+; AVX512-NEXT:    [[TMP31:%.*]] = getelementptr float, ptr [[TMP28]], i32 0
+; AVX512-NEXT:    [[WIDE_LOAD23:%.*]] = load <8 x float>, ptr [[TMP31]], align 4, !alias.scope [[META24:![0-9]+]]
+; AVX512-NEXT:    [[TMP32:%.*]] = getelementptr inbounds float, <8 x ptr> [[TMP26]], i64 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> [[WIDE_LOAD23]], <8 x ptr> [[TMP32]], i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope [[META20]], !noalias [[META22]]
+; AVX512-NEXT:    [[INDEX_NEXT24]] = add nuw i64 [[INDEX18]], 8
+; AVX512-NEXT:    [[PTR_IND20]] = getelementptr i8, ptr [[POINTER_PHI19]], i64 512
+; AVX512-NEXT:    [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT24]], [[N_VEC10]]
+; AVX512-NEXT:    br i1 [[TMP33]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
 ; AVX512:       vec.epilog.middle.block:
-; AVX512-NEXT:    [[CMP_N19:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC12]]
-; AVX512-NEXT:    br i1 [[CMP_N19]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
+; AVX512-NEXT:    [[CMP_N17:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC10]]
+; AVX512-NEXT:    br i1 [[CMP_N17]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; AVX512:       vec.epilog.scalar.ph:
-; AVX512-NEXT:    [[BC_RESUME_VAL15:%.*]] = phi ptr [ [[IND_END13]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END14]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PTR]], [[VECTOR_MEMCHECK]] ], [ [[PTR]], [[ITER_CHECK]] ]
-; AVX512-NEXT:    [[BC_RESUME_VAL18:%.*]] = phi ptr [ [[IND_END16]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END17]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[DEST]], [[VECTOR_MEMCHECK]] ], [ [[DEST]], [[ITER_CHECK]] ]
+; AVX512-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi ptr [ [[IND_END11]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END12]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PTR]], [[VECTOR_MEMCHECK]] ], [ [[PTR]], [[ITER_CHECK]] ]
+; AVX512-NEXT:    [[BC_RESUME_VAL16:%.*]] = phi ptr [ [[IND_END14]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END15]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[DEST]], [[VECTOR_MEMCHECK]] ], [ [[DEST]], [[ITER_CHECK]] ]
 ; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX512:       for.body:
-; AVX512-NEXT:    [[PTR_ADDR_012:%.*]] = phi ptr [ [[BC_RESUME_VAL15]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ]
-; AVX512-NEXT:    [[DEST_ADDR_011:%.*]] = phi ptr [ [[BC_RESUME_VAL18]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[ADD_PTR6:%.*]], [[FOR_BODY]] ]
+; AVX512-NEXT:    [[PTR_ADDR_012:%.*]] = phi ptr [ [[BC_RESUME_VAL13]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ]
+; AVX512-NEXT:    [[DEST_ADDR_011:%.*]] = phi ptr [ [[BC_RESUME_VAL16]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[ADD_PTR6:%.*]], [[FOR_BODY]] ]
 ; AVX512-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[PTR_ADDR_012]], i64 [[IDXPROM]]
-; AVX512-NEXT:    [[TMP35:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; AVX512-NEXT:    store float [[TMP35]], ptr [[DEST_ADDR_011]], align 4
-; AVX512-NEXT:    [[TMP36:%.*]] = load float, ptr [[PTR_ADDR_012]], align 4
+; AVX512-NEXT:    [[TMP34:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; AVX512-NEXT:    store float [[TMP34]], ptr [[DEST_ADDR_011]], align 4
+; AVX512-NEXT:    [[TMP35:%.*]] = load float, ptr [[PTR_ADDR_012]], align 4
 ; AVX512-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[DEST_ADDR_011]], i64 1
-; AVX512-NEXT:    store float [[TMP36]], ptr [[ARRAYIDX5]], align 4
+; AVX512-NEXT:    store float [[TMP35]], ptr [[ARRAYIDX5]], align 4
 ; AVX512-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds float, ptr [[PTR_ADDR_012]], i64 1
 ; AVX512-NEXT:    [[ADD_PTR6]] = getelementptr inbounds float, ptr [[DEST_ADDR_011]], i64 16
 ; AVX512-NEXT:    [[CMP_NOT:%.*]] = icmp eq ptr [[INCDEC_PTR]], [[ADD_PTR]]
@@ -774,30 +771,29 @@ define void @test_gather_not_profitable_pr48429(i32 %d, ptr readonly noalias %pt
 ; FVW2-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; FVW2:       vector.body:
 ; FVW2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FVW2-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 0
-; FVW2-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 4
-; FVW2-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP16]]
-; FVW2-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX]], 0
-; FVW2-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP17]], 64
-; FVW2-NEXT:    [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[DEST]], i64 [[TMP18]]
-; FVW2-NEXT:    [[TMP19:%.*]] = add i64 [[INDEX]], 1
-; FVW2-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 64
-; FVW2-NEXT:    [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[DEST]], i64 [[TMP20]]
-; FVW2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[NEXT_GEP]], i64 [[IDXPROM]]
+; FVW2-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 4
+; FVW2-NEXT:    [[TMP15:%.*]] = add i64 [[OFFSET_IDX]], 0
+; FVW2-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP15]]
+; FVW2-NEXT:    [[OFFSET_IDX9:%.*]] = mul i64 [[INDEX]], 64
+; FVW2-NEXT:    [[TMP17:%.*]] = add i64 [[OFFSET_IDX9]], 0
+; FVW2-NEXT:    [[TMP18:%.*]] = add i64 [[OFFSET_IDX9]], 64
+; FVW2-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[DEST]], i64 [[TMP17]]
+; FVW2-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[DEST]], i64 [[TMP18]]
+; FVW2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[IDXPROM]]
 ; FVW2-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i32 0
-; FVW2-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP22]], align 4, !alias.scope !8
+; FVW2-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP22]], align 4, !alias.scope [[META8:![0-9]+]]
 ; FVW2-NEXT:    [[TMP23:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i32 0
-; FVW2-NEXT:    store float [[TMP23]], ptr [[NEXT_GEP9]], align 4, !alias.scope !11, !noalias !13
+; FVW2-NEXT:    store float [[TMP23]], ptr [[TMP19]], align 4, !alias.scope [[META11:![0-9]+]], !noalias [[META13:![0-9]+]]
 ; FVW2-NEXT:    [[TMP24:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i32 1
-; FVW2-NEXT:    store float [[TMP24]], ptr [[NEXT_GEP10]], align 4, !alias.scope !11, !noalias !13
-; FVW2-NEXT:    [[TMP25:%.*]] = getelementptr float, ptr [[NEXT_GEP]], i32 0
-; FVW2-NEXT:    [[WIDE_LOAD11:%.*]] = load <2 x float>, ptr [[TMP25]], align 4, !alias.scope !15
-; FVW2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, ptr [[NEXT_GEP9]], i64 1
-; FVW2-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, ptr [[NEXT_GEP10]], i64 1
-; FVW2-NEXT:    [[TMP28:%.*]] = extractelement <2 x float> [[WIDE_LOAD11]], i32 0
-; FVW2-NEXT:    store float [[TMP28]], ptr [[TMP26]], align 4, !alias.scope !11, !noalias !13
-; FVW2-NEXT:    [[TMP29:%.*]] = extractelement <2 x float> [[WIDE_LOAD11]], i32 1
-; FVW2-NEXT:    store float [[TMP29]], ptr [[TMP27]], align 4, !alias.scope !11, !noalias !13
+; FVW2-NEXT:    store float [[TMP24]], ptr [[TMP20]], align 4, !alias.scope [[META11]], !noalias [[META13]]
+; FVW2-NEXT:    [[TMP25:%.*]] = getelementptr float, ptr [[TMP16]], i32 0
+; FVW2-NEXT:    [[WIDE_LOAD10:%.*]] = load <2 x float>, ptr [[TMP25]], align 4, !alias.scope [[META15:![0-9]+]]
+; FVW2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, ptr [[TMP19]], i64 1
+; FVW2-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, ptr [[TMP20]], i64 1
+; FVW2-NEXT:    [[TMP28:%.*]] = extractelement <2 x float> [[WIDE_LOAD10]], i32 0
+; FVW2-NEXT:    store float [[TMP28]], ptr [[TMP26]], align 4, !alias.scope [[META11]], !noalias [[META13]]
+; FVW2-NEXT:    [[TMP29:%.*]] = extractelement <2 x float> [[WIDE_LOAD10]], i32 1
+; FVW2-NEXT:    store float [[TMP29]], ptr [[TMP27]], align 4, !alias.scope [[META11]], !noalias [[META13]]
 ; FVW2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; FVW2-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; FVW2-NEXT:    br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/interleave-opaque-pointers.ll b/llvm/test/Transforms/LoopVectorize/X86/interleave-opaque-pointers.ll
index 022912f3b855..1b0118e137e7 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/interleave-opaque-pointers.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleave-opaque-pointers.ll
@@ -25,15 +25,14 @@ define void @test_pr55375_interleave_opaque_ptr(ptr %start, ptr %end) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
-; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 16
-; CHECK-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP8]]
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x ptr> poison, ptr [[NEXT_GEP]], i32 0
-; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x ptr> [[TMP9]], ptr [[NEXT_GEP3]], i32 1
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr ptr, ptr [[NEXT_GEP]], i32 0
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x ptr> [[TMP9]], ptr [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr ptr, ptr [[TMP7]], i32 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x ptr> zeroinitializer, <2 x ptr> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x ptr> [[TMP12]], <4 x ptr> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
 ; CHECK-NEXT:    store <4 x ptr> [[INTERLEAVED_VEC]], ptr [[TMP11]], align 8
@@ -53,7 +52,7 @@ define void @test_pr55375_interleave_opaque_ptr(ptr %start, ptr %end) {
 ; CHECK-NEXT:    store ptr null, ptr [[IV]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = getelementptr inbounds [[PAIR]], ptr [[IV]], i64 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq ptr [[IV_NEXT]], [[END]]
-; CHECK-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
index 51d264820503..dc474fbf67ce 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
@@ -273,64 +273,58 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE19:%.*]] ]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT13:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT12]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[VEC_IV:%.*]] = or disjoint <4 x i64> [[BROADCAST_SPLAT13]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE20:%.*]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-NEXT:    [[OFFSET_IDX8:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT13:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT14:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT13]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[VEC_IV:%.*]] = or disjoint <4 x i64> [[BROADCAST_SPLAT14]], <i64 0, i64 1, i64 2, i64 3>
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0
 ; CHECK-NEXT:    br i1 [[TMP4]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
-; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 [[TMP5]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT:    [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[Q:%.*]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[NEXT_GEP8]], align 16
-; CHECK-NEXT:    store i32 [[TMP7]], ptr [[NEXT_GEP]], align 16
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[Q:%.*]], i64 [[OFFSET_IDX8]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[NEXT_GEP9]], align 16
+; CHECK-NEXT:    store i32 [[TMP5]], ptr [[NEXT_GEP]], align 16
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; CHECK:       pred.store.continue:
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1
-; CHECK-NEXT:    br i1 [[TMP8]], label [[PRED_STORE_IF14:%.*]], label [[PRED_STORE_CONTINUE15:%.*]]
-; CHECK:       pred.store.if14:
-; CHECK-NEXT:    [[TMP9:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP10:%.*]] = or disjoint i64 [[TMP9]], 4
-; CHECK-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP10]]
-; CHECK-NEXT:    [[TMP11:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP12:%.*]] = or disjoint i64 [[TMP11]], 4
-; CHECK-NEXT:    [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP12]]
-; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[NEXT_GEP9]], align 16
-; CHECK-NEXT:    store i32 [[TMP13]], ptr [[NEXT_GEP5]], align 16
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE15]]
-; CHECK:       pred.store.continue15:
-; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2
-; CHECK-NEXT:    br i1 [[TMP14]], label [[PRED_STORE_IF16:%.*]], label [[PRED_STORE_CONTINUE17:%.*]]
-; CHECK:       pred.store.if16:
-; CHECK-NEXT:    [[TMP15:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP16:%.*]] = or disjoint i64 [[TMP15]], 8
-; CHECK-NEXT:    [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP16]]
-; CHECK-NEXT:    [[TMP17:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP18:%.*]] = or disjoint i64 [[TMP17]], 8
-; CHECK-NEXT:    [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP18]]
-; CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[NEXT_GEP10]], align 16
-; CHECK-NEXT:    store i32 [[TMP19]], ptr [[NEXT_GEP6]], align 16
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE17]]
-; CHECK:       pred.store.continue17:
-; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3
-; CHECK-NEXT:    br i1 [[TMP20]], label [[PRED_STORE_IF18:%.*]], label [[PRED_STORE_CONTINUE19]]
-; CHECK:       pred.store.if18:
-; CHECK-NEXT:    [[TMP21:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP22:%.*]] = or disjoint i64 [[TMP21]], 12
-; CHECK-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP22]]
-; CHECK-NEXT:    [[TMP23:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP24:%.*]] = or disjoint i64 [[TMP23]], 12
-; CHECK-NEXT:    [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP24]]
-; CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[NEXT_GEP11]], align 16
-; CHECK-NEXT:    store i32 [[TMP25]], ptr [[NEXT_GEP7]], align 16
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE19]]
-; CHECK:       pred.store.continue19:
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1
+; CHECK-NEXT:    br i1 [[TMP6]], label [[PRED_STORE_IF15:%.*]], label [[PRED_STORE_CONTINUE16:%.*]]
+; CHECK:       pred.store.if15:
+; CHECK-NEXT:    [[TMP7:%.*]] = or disjoint i64 [[OFFSET_IDX]], 4
+; CHECK-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or disjoint i64 [[OFFSET_IDX8]], 4
+; CHECK-NEXT:    [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[NEXT_GEP10]], align 16
+; CHECK-NEXT:    store i32 [[TMP9]], ptr [[NEXT_GEP5]], align 16
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE16]]
+; CHECK:       pred.store.continue16:
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2
+; CHECK-NEXT:    br i1 [[TMP10]], label [[PRED_STORE_IF17:%.*]], label [[PRED_STORE_CONTINUE18:%.*]]
+; CHECK:       pred.store.if17:
+; CHECK-NEXT:    [[TMP11:%.*]] = or disjoint i64 [[OFFSET_IDX]], 8
+; CHECK-NEXT:    [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or disjoint i64 [[OFFSET_IDX8]], 8
+; CHECK-NEXT:    [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[NEXT_GEP11]], align 16
+; CHECK-NEXT:    store i32 [[TMP13]], ptr [[NEXT_GEP6]], align 16
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE18]]
+; CHECK:       pred.store.continue18:
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3
+; CHECK-NEXT:    br i1 [[TMP14]], label [[PRED_STORE_IF19:%.*]], label [[PRED_STORE_CONTINUE20]]
+; CHECK:       pred.store.if19:
+; CHECK-NEXT:    [[TMP15:%.*]] = or disjoint i64 [[OFFSET_IDX]], 12
+; CHECK-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or disjoint i64 [[OFFSET_IDX8]], 12
+; CHECK-NEXT:    [[NEXT_GEP12:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[NEXT_GEP12]], align 16
+; CHECK-NEXT:    store i32 [[TMP17]], ptr [[NEXT_GEP7]], align 16
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE20]]
+; CHECK:       pred.store.continue20:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
@@ -410,24 +404,24 @@ define void @example23b(ptr noalias nocapture %src, ptr noalias nocapture %dst)
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[TMP1]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[TMP2]]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[OFFSET_IDX4:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[OFFSET_IDX4]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[NEXT_GEP]], align 2
-; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i16> [[WIDE_LOAD]] to <4 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw <4 x i32> [[TMP3]], <i32 7, i32 7, i32 7, i32 7>
-; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[NEXT_GEP4]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <4 x i16> [[WIDE_LOAD]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw <4 x i32> [[TMP1]], <i32 7, i32 7, i32 7, i32 7>
+; CHECK-NEXT:    store <4 x i32> [[TMP2]], ptr [[NEXT_GEP5]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
-; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    br i1 true, label [[TMP7:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br i1 true, label [[TMP5:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    br label [[TMP6:%.*]]
-; CHECK:       6:
-; CHECK-NEXT:    br i1 poison, label [[TMP7]], label [[TMP6]], !llvm.loop [[LOOP11:![0-9]+]]
-; CHECK:       7:
+; CHECK-NEXT:    br label [[TMP4:%.*]]
+; CHECK:       4:
+; CHECK-NEXT:    br i1 poison, label [[TMP5]], label [[TMP4]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       5:
 ; CHECK-NEXT:    ret void
 ;
   br label %1
@@ -457,7 +451,9 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst)
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE16:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE17:%.*]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
+; CHECK-NEXT:    [[OFFSET_IDX7:%.*]] = shl i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[VEC_IV:%.*]] = or disjoint <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
@@ -465,71 +461,63 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst)
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[NEXT_GEP]], align 2
-; CHECK-NEXT:    [[TMP6:%.*]] = zext i16 [[TMP5]] to i32
-; CHECK-NEXT:    [[TMP7:%.*]] = shl nuw nsw i32 [[TMP6]], 7
-; CHECK-NEXT:    store i32 [[TMP7]], ptr [[NEXT_GEP7]], align 4
+; CHECK-NEXT:    [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[OFFSET_IDX7]]
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr [[NEXT_GEP]], align 2
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP3]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i32 [[TMP4]], 7
+; CHECK-NEXT:    store i32 [[TMP5]], ptr [[NEXT_GEP8]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; CHECK:       pred.store.continue:
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
-; CHECK-NEXT:    br i1 [[TMP8]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12:%.*]]
-; CHECK:       pred.store.if11:
-; CHECK-NEXT:    [[TMP9:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP10:%.*]] = or disjoint i64 [[TMP9]], 4
-; CHECK-NEXT:    [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP10]]
-; CHECK-NEXT:    [[TMP11:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP12:%.*]] = or disjoint i64 [[TMP11]], 2
-; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP12]]
-; CHECK-NEXT:    [[TMP13:%.*]] = load i16, ptr [[NEXT_GEP4]], align 2
-; CHECK-NEXT:    [[TMP14:%.*]] = zext i16 [[TMP13]] to i32
-; CHECK-NEXT:    [[TMP15:%.*]] = shl nuw nsw i32 [[TMP14]], 7
-; CHECK-NEXT:    store i32 [[TMP15]], ptr [[NEXT_GEP8]], align 4
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE12]]
-; CHECK:       pred.store.continue12:
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
-; CHECK-NEXT:    br i1 [[TMP16]], label [[PRED_STORE_IF13:%.*]], label [[PRED_STORE_CONTINUE14:%.*]]
-; CHECK:       pred.store.if13:
-; CHECK-NEXT:    [[TMP17:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP18:%.*]] = or disjoint i64 [[TMP17]], 8
-; CHECK-NEXT:    [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP18]]
-; CHECK-NEXT:    [[TMP19:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP20:%.*]] = or disjoint i64 [[TMP19]], 4
-; CHECK-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP20]]
-; CHECK-NEXT:    [[TMP21:%.*]] = load i16, ptr [[NEXT_GEP5]], align 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
+; CHECK-NEXT:    br i1 [[TMP6]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13:%.*]]
+; CHECK:       pred.store.if12:
+; CHECK-NEXT:    [[TMP7:%.*]] = or disjoint i64 [[OFFSET_IDX7]], 4
+; CHECK-NEXT:    [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or disjoint i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i16, ptr [[NEXT_GEP4]], align 2
+; CHECK-NEXT:    [[TMP10:%.*]] = zext i16 [[TMP9]] to i32
+; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw nsw i32 [[TMP10]], 7
+; CHECK-NEXT:    store i32 [[TMP11]], ptr [[NEXT_GEP9]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE13]]
+; CHECK:       pred.store.continue13:
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
+; CHECK-NEXT:    br i1 [[TMP12]], label [[PRED_STORE_IF14:%.*]], label [[PRED_STORE_CONTINUE15:%.*]]
+; CHECK:       pred.store.if14:
+; CHECK-NEXT:    [[TMP13:%.*]] = or disjoint i64 [[OFFSET_IDX7]], 8
+; CHECK-NEXT:    [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = or disjoint i64 [[OFFSET_IDX]], 4
+; CHECK-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = load i16, ptr [[NEXT_GEP5]], align 2
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i16 [[TMP15]] to i32
+; CHECK-NEXT:    [[TMP17:%.*]] = shl nuw nsw i32 [[TMP16]], 7
+; CHECK-NEXT:    store i32 [[TMP17]], ptr [[NEXT_GEP10]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE15]]
+; CHECK:       pred.store.continue15:
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; CHECK-NEXT:    br i1 [[TMP18]], label [[PRED_STORE_IF16:%.*]], label [[PRED_STORE_CONTINUE17]]
+; CHECK:       pred.store.if16:
+; CHECK-NEXT:    [[TMP19:%.*]] = or disjoint i64 [[OFFSET_IDX7]], 12
+; CHECK-NEXT:    [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = or disjoint i64 [[OFFSET_IDX]], 6
+; CHECK-NEXT:    [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP20]]
+; CHECK-NEXT:    [[TMP21:%.*]] = load i16, ptr [[NEXT_GEP6]], align 2
 ; CHECK-NEXT:    [[TMP22:%.*]] = zext i16 [[TMP21]] to i32
 ; CHECK-NEXT:    [[TMP23:%.*]] = shl nuw nsw i32 [[TMP22]], 7
-; CHECK-NEXT:    store i32 [[TMP23]], ptr [[NEXT_GEP9]], align 4
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE14]]
-; CHECK:       pred.store.continue14:
-; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
-; CHECK-NEXT:    br i1 [[TMP24]], label [[PRED_STORE_IF15:%.*]], label [[PRED_STORE_CONTINUE16]]
-; CHECK:       pred.store.if15:
-; CHECK-NEXT:    [[TMP25:%.*]] = shl i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP26:%.*]] = or disjoint i64 [[TMP25]], 12
-; CHECK-NEXT:    [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP26]]
-; CHECK-NEXT:    [[TMP27:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP28:%.*]] = or disjoint i64 [[TMP27]], 6
-; CHECK-NEXT:    [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP28]]
-; CHECK-NEXT:    [[TMP29:%.*]] = load i16, ptr [[NEXT_GEP6]], align 2
-; CHECK-NEXT:    [[TMP30:%.*]] = zext i16 [[TMP29]] to i32
-; CHECK-NEXT:    [[TMP31:%.*]] = shl nuw nsw i32 [[TMP30]], 7
-; CHECK-NEXT:    store i32 [[TMP31]], ptr [[NEXT_GEP10]], align 4
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE16]]
-; CHECK:       pred.store.continue16:
+; CHECK-NEXT:    store i32 [[TMP23]], ptr [[NEXT_GEP11]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE17]]
+; CHECK:       pred.store.continue17:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
-; CHECK-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
+; CHECK-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    br i1 true, label [[TMP34:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br i1 true, label [[TMP26:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    br label [[TMP33:%.*]]
-; CHECK:       33:
-; CHECK-NEXT:    br i1 poison, label [[TMP34]], label [[TMP33]], !llvm.loop [[LOOP13:![0-9]+]]
-; CHECK:       34:
+; CHECK-NEXT:    br label [[TMP25:%.*]]
+; CHECK:       25:
+; CHECK-NEXT:    br i1 poison, label [[TMP26]], label [[TMP25]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK:       26:
 ; CHECK-NEXT:    ret void
 ;
   br label %1
diff --git a/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll b/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
index cc7c1d8a6188..1e23f02ee2b1 100644
--- a/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
@@ -311,15 +311,12 @@ for.end:
 ; INTER:     vector.body
 ; INTER:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
 ; INTER:       %[[I0:.+]] = shl i64 %index, 4
+; INTER:       %[[I1:.+]] = or disjoint i64 %[[I0]], 16
+; INTER:       %[[I2:.+]] = or disjoint i64 %[[I0]], 32
+; INTER:       %[[I3:.+]] = or disjoint i64 %[[I0]], 48
 ; INTER:       %next.gep = getelementptr i8, ptr %a, i64 %[[I0]]
-; INTER:       %[[S1:.+]] = shl i64 %index, 4
-; INTER:       %[[I1:.+]] = or disjoint i64 %[[S1]], 16
 ; INTER:       %next.gep2 = getelementptr i8, ptr %a, i64 %[[I1]]
-; INTER:       %[[S2:.+]] = shl i64 %index, 4
-; INTER:       %[[I2:.+]] = or disjoint i64 %[[S2]], 32
 ; INTER:       %next.gep3 = getelementptr i8, ptr %a, i64 %[[I2]]
-; INTER:       %[[S3:.+]] = shl i64 %index, 4
-; INTER:       %[[I3:.+]] = or disjoint i64 %[[S3]], 48
 ; INTER:       %next.gep4 = getelementptr i8, ptr %a, i64 %[[I3]]
 ; INTER:       br i1 {{.*}}, label %middle.block, label %vector.body
 ;
@@ -361,15 +358,12 @@ for.end:
 ; CHECK:     vector.body
 ; CHECK:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
 ; CHECK:       [[SHL1:%.+]] = shl i64 %index, 4
+; CHECK:       %[[I1:.+]] = or disjoint i64 [[SHL1]], 16
+; CHECK:       %[[I2:.+]] = or disjoint i64 [[SHL1]], 32
+; CHECK:       %[[I3:.+]] = or disjoint i64 [[SHL1]], 48
 ; CHECK:       %next.gep = getelementptr i8, ptr %a, i64 [[SHL1]]
-; CHECK:       [[SHL2:%.+]] = shl i64 %index, 4
-; CHECK:       %[[I1:.+]] = or disjoint i64 [[SHL2]], 16
 ; CHECK:       %next.gep2 = getelementptr i8, ptr %a, i64 %[[I1]]
-; CHECK:       [[SHL3:%.+]] = shl i64 %index, 4
-; CHECK:       %[[I2:.+]] = or disjoint i64 [[SHL3]], 32
 ; CHECK:       %next.gep3 = getelementptr i8, ptr %a, i64 %[[I2]]
-; CHECK:       [[SHL4:%.+]] = shl i64 %index, 4
-; CHECK:       %[[I3:.+]] = or disjoint i64 [[SHL4]], 48
 ; CHECK:       %next.gep4 = getelementptr i8, ptr %a, i64 %[[I3]]
 ; CHECK:       br i1 {{.*}}, label %middle.block, label %vector.body
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
index b451d4b4e546..0a37e5ea0ca0 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
@@ -1537,92 +1537,85 @@ define i32 @PR33613(ptr %b, double %j, i32 %d) {
 ; UNROLL-NO-IC-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; UNROLL-NO-IC:       vector.body:
 ; UNROLL-NO-IC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-IC-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP48:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-IC-NEXT:    [[VEC_PHI9:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP49:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x double> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP39:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-IC-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; UNROLL-NO-IC-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 200
-; UNROLL-NO-IC-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP1]]
-; UNROLL-NO-IC-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 1
-; UNROLL-NO-IC-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 200
-; UNROLL-NO-IC-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP3]]
-; UNROLL-NO-IC-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 2
-; UNROLL-NO-IC-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 200
-; UNROLL-NO-IC-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP5]]
-; UNROLL-NO-IC-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 3
-; UNROLL-NO-IC-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 200
-; UNROLL-NO-IC-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP7]]
-; UNROLL-NO-IC-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 4
-; UNROLL-NO-IC-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 200
-; UNROLL-NO-IC-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP9]]
-; UNROLL-NO-IC-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 5
-; UNROLL-NO-IC-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 200
-; UNROLL-NO-IC-NEXT:    [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP11]]
-; UNROLL-NO-IC-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 6
-; UNROLL-NO-IC-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 200
-; UNROLL-NO-IC-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP13]]
-; UNROLL-NO-IC-NEXT:    [[TMP14:%.*]] = add i64 [[INDEX]], 7
-; UNROLL-NO-IC-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 200
-; UNROLL-NO-IC-NEXT:    [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP15]]
-; UNROLL-NO-IC-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP]], i64 [[IDXPROM]]
-; UNROLL-NO-IC-NEXT:    [[TMP17:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP2]], i64 [[IDXPROM]]
-; UNROLL-NO-IC-NEXT:    [[TMP18:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP3]], i64 [[IDXPROM]]
-; UNROLL-NO-IC-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP4]], i64 [[IDXPROM]]
-; UNROLL-NO-IC-NEXT:    [[TMP20:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP5]], i64 [[IDXPROM]]
-; UNROLL-NO-IC-NEXT:    [[TMP21:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP6]], i64 [[IDXPROM]]
-; UNROLL-NO-IC-NEXT:    [[TMP22:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP7]], i64 [[IDXPROM]]
-; UNROLL-NO-IC-NEXT:    [[TMP23:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP8]], i64 [[IDXPROM]]
-; UNROLL-NO-IC-NEXT:    [[TMP24:%.*]] = load double, ptr [[TMP16]], align 8
-; UNROLL-NO-IC-NEXT:    [[TMP25:%.*]] = load double, ptr [[TMP17]], align 8
-; UNROLL-NO-IC-NEXT:    [[TMP26:%.*]] = load double, ptr [[TMP18]], align 8
-; UNROLL-NO-IC-NEXT:    [[TMP27:%.*]] = load double, ptr [[TMP19]], align 8
+; UNROLL-NO-IC-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP40:%.*]], [[VECTOR_BODY]] ]
+; UNROLL-NO-IC-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP41:%.*]], [[VECTOR_BODY]] ]
+; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x double> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP31:%.*]], [[VECTOR_BODY]] ]
+; UNROLL-NO-IC-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 200
+; UNROLL-NO-IC-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; UNROLL-NO-IC-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 200
+; UNROLL-NO-IC-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 400
+; UNROLL-NO-IC-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 600
+; UNROLL-NO-IC-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 800
+; UNROLL-NO-IC-NEXT:    [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 1000
+; UNROLL-NO-IC-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 1200
+; UNROLL-NO-IC-NEXT:    [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 1400
+; UNROLL-NO-IC-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
+; UNROLL-NO-IC-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP1]]
+; UNROLL-NO-IC-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
+; UNROLL-NO-IC-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP3]]
+; UNROLL-NO-IC-NEXT:    [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP4]]
+; UNROLL-NO-IC-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP5]]
+; UNROLL-NO-IC-NEXT:    [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP6]]
+; UNROLL-NO-IC-NEXT:    [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP7]]
+; UNROLL-NO-IC-NEXT:    [[TMP8:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP]], i64 [[IDXPROM]]
+; UNROLL-NO-IC-NEXT:    [[TMP9:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP3]], i64 [[IDXPROM]]
+; UNROLL-NO-IC-NEXT:    [[TMP10:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP4]], i64 [[IDXPROM]]
+; UNROLL-NO-IC-NEXT:    [[TMP11:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP5]], i64 [[IDXPROM]]
+; UNROLL-NO-IC-NEXT:    [[TMP12:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP6]], i64 [[IDXPROM]]
+; UNROLL-NO-IC-NEXT:    [[TMP13:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP7]], i64 [[IDXPROM]]
+; UNROLL-NO-IC-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP8]], i64 [[IDXPROM]]
+; UNROLL-NO-IC-NEXT:    [[TMP15:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP9]], i64 [[IDXPROM]]
+; UNROLL-NO-IC-NEXT:    [[TMP16:%.*]] = load double, ptr [[TMP8]], align 8
+; UNROLL-NO-IC-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP9]], align 8
+; UNROLL-NO-IC-NEXT:    [[TMP18:%.*]] = load double, ptr [[TMP10]], align 8
+; UNROLL-NO-IC-NEXT:    [[TMP19:%.*]] = load double, ptr [[TMP11]], align 8
+; UNROLL-NO-IC-NEXT:    [[TMP20:%.*]] = insertelement <4 x double> poison, double [[TMP16]], i32 0
+; UNROLL-NO-IC-NEXT:    [[TMP21:%.*]] = insertelement <4 x double> [[TMP20]], double [[TMP17]], i32 1
+; UNROLL-NO-IC-NEXT:    [[TMP22:%.*]] = insertelement <4 x double> [[TMP21]], double [[TMP18]], i32 2
+; UNROLL-NO-IC-NEXT:    [[TMP23:%.*]] = insertelement <4 x double> [[TMP22]], double [[TMP19]], i32 3
+; UNROLL-NO-IC-NEXT:    [[TMP24:%.*]] = load double, ptr [[TMP12]], align 8
+; UNROLL-NO-IC-NEXT:    [[TMP25:%.*]] = load double, ptr [[TMP13]], align 8
+; UNROLL-NO-IC-NEXT:    [[TMP26:%.*]] = load double, ptr [[TMP14]], align 8
+; UNROLL-NO-IC-NEXT:    [[TMP27:%.*]] = load double, ptr [[TMP15]], align 8
 ; UNROLL-NO-IC-NEXT:    [[TMP28:%.*]] = insertelement <4 x double> poison, double [[TMP24]], i32 0
 ; UNROLL-NO-IC-NEXT:    [[TMP29:%.*]] = insertelement <4 x double> [[TMP28]], double [[TMP25]], i32 1
 ; UNROLL-NO-IC-NEXT:    [[TMP30:%.*]] = insertelement <4 x double> [[TMP29]], double [[TMP26]], i32 2
-; UNROLL-NO-IC-NEXT:    [[TMP31:%.*]] = insertelement <4 x double> [[TMP30]], double [[TMP27]], i32 3
-; UNROLL-NO-IC-NEXT:    [[TMP32:%.*]] = load double, ptr [[TMP20]], align 8
-; UNROLL-NO-IC-NEXT:    [[TMP33:%.*]] = load double, ptr [[TMP21]], align 8
-; UNROLL-NO-IC-NEXT:    [[TMP34:%.*]] = load double, ptr [[TMP22]], align 8
-; UNROLL-NO-IC-NEXT:    [[TMP35:%.*]] = load double, ptr [[TMP23]], align 8
-; UNROLL-NO-IC-NEXT:    [[TMP36:%.*]] = insertelement <4 x double> poison, double [[TMP32]], i32 0
-; UNROLL-NO-IC-NEXT:    [[TMP37:%.*]] = insertelement <4 x double> [[TMP36]], double [[TMP33]], i32 1
-; UNROLL-NO-IC-NEXT:    [[TMP38:%.*]] = insertelement <4 x double> [[TMP37]], double [[TMP34]], i32 2
-; UNROLL-NO-IC-NEXT:    [[TMP39]] = insertelement <4 x double> [[TMP38]], double [[TMP35]], i32 3
-; UNROLL-NO-IC-NEXT:    [[TMP40:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[TMP31]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; UNROLL-NO-IC-NEXT:    [[TMP41:%.*]] = shufflevector <4 x double> [[TMP31]], <4 x double> [[TMP39]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; UNROLL-NO-IC-NEXT:    [[TMP42:%.*]] = fmul <4 x double> [[TMP40]], [[TMP31]]
-; UNROLL-NO-IC-NEXT:    [[TMP43:%.*]] = fmul <4 x double> [[TMP41]], [[TMP39]]
-; UNROLL-NO-IC-NEXT:    [[TMP44:%.*]] = fcmp une <4 x double> [[TMP42]], zeroinitializer
-; UNROLL-NO-IC-NEXT:    [[TMP45:%.*]] = fcmp une <4 x double> [[TMP43]], zeroinitializer
-; UNROLL-NO-IC-NEXT:    [[TMP46:%.*]] = zext <4 x i1> [[TMP44]] to <4 x i32>
-; UNROLL-NO-IC-NEXT:    [[TMP47:%.*]] = zext <4 x i1> [[TMP45]] to <4 x i32>
-; UNROLL-NO-IC-NEXT:    [[TMP48]] = add <4 x i32> [[VEC_PHI]], [[TMP46]]
-; UNROLL-NO-IC-NEXT:    [[TMP49]] = add <4 x i32> [[VEC_PHI9]], [[TMP47]]
+; UNROLL-NO-IC-NEXT:    [[TMP31]] = insertelement <4 x double> [[TMP30]], double [[TMP27]], i32 3
+; UNROLL-NO-IC-NEXT:    [[TMP32:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[TMP23]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; UNROLL-NO-IC-NEXT:    [[TMP33:%.*]] = shufflevector <4 x double> [[TMP23]], <4 x double> [[TMP31]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; UNROLL-NO-IC-NEXT:    [[TMP34:%.*]] = fmul <4 x double> [[TMP32]], [[TMP23]]
+; UNROLL-NO-IC-NEXT:    [[TMP35:%.*]] = fmul <4 x double> [[TMP33]], [[TMP31]]
+; UNROLL-NO-IC-NEXT:    [[TMP36:%.*]] = fcmp une <4 x double> [[TMP34]], zeroinitializer
+; UNROLL-NO-IC-NEXT:    [[TMP37:%.*]] = fcmp une <4 x double> [[TMP35]], zeroinitializer
+; UNROLL-NO-IC-NEXT:    [[TMP38:%.*]] = zext <4 x i1> [[TMP36]] to <4 x i32>
+; UNROLL-NO-IC-NEXT:    [[TMP39:%.*]] = zext <4 x i1> [[TMP37]] to <4 x i32>
+; UNROLL-NO-IC-NEXT:    [[TMP40]] = add <4 x i32> [[VEC_PHI]], [[TMP38]]
+; UNROLL-NO-IC-NEXT:    [[TMP41]] = add <4 x i32> [[VEC_PHI2]], [[TMP39]]
 ; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; UNROLL-NO-IC-NEXT:    [[TMP50:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10240
-; UNROLL-NO-IC-NEXT:    br i1 [[TMP50]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; UNROLL-NO-IC-NEXT:    [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10240
+; UNROLL-NO-IC-NEXT:    br i1 [[TMP42]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; UNROLL-NO-IC:       middle.block:
-; UNROLL-NO-IC-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP49]], [[TMP48]]
-; UNROLL-NO-IC-NEXT:    [[TMP51:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
-; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x double> [[TMP39]], i32 3
+; UNROLL-NO-IC-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP41]], [[TMP40]]
+; UNROLL-NO-IC-NEXT:    [[TMP43:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x double> [[TMP31]], i32 3
 ; UNROLL-NO-IC-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NO-IC:       scalar.ph:
 ; UNROLL-NO-IC-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi double [ [[J]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]
 ; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ]
 ; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 10240, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; UNROLL-NO-IC-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP51]], [[MIDDLE_BLOCK]] ]
+; UNROLL-NO-IC-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP43]], [[MIDDLE_BLOCK]] ]
 ; UNROLL-NO-IC-NEXT:    br label [[FOR_BODY:%.*]]
 ; UNROLL-NO-IC:       for.cond.cleanup:
-; UNROLL-NO-IC-NEXT:    [[A_1_LCSSA:%.*]] = phi i32 [ [[A_1:%.*]], [[FOR_BODY]] ], [ [[TMP51]], [[MIDDLE_BLOCK]] ]
+; UNROLL-NO-IC-NEXT:    [[A_1_LCSSA:%.*]] = phi i32 [ [[A_1:%.*]], [[FOR_BODY]] ], [ [[TMP43]], [[MIDDLE_BLOCK]] ]
 ; UNROLL-NO-IC-NEXT:    ret i32 [[A_1_LCSSA]]
 ; UNROLL-NO-IC:       for.body:
 ; UNROLL-NO-IC-NEXT:    [[B_ADDR_012:%.*]] = phi ptr [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD_PTR:%.*]], [[FOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[I_011:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[INC1:%.*]], [[FOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[A_010:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[A_1]], [[FOR_BODY]] ]
-; UNROLL-NO-IC-NEXT:    [[SCALAR_RECUR:%.*]] = phi double [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP52:%.*]], [[FOR_BODY]] ]
+; UNROLL-NO-IC-NEXT:    [[SCALAR_RECUR:%.*]] = phi double [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP44:%.*]], [[FOR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[B_ADDR_012]], i64 [[IDXPROM]]
-; UNROLL-NO-IC-NEXT:    [[TMP52]] = load double, ptr [[ARRAYIDX]], align 8
-; UNROLL-NO-IC-NEXT:    [[MUL:%.*]] = fmul double [[SCALAR_RECUR]], [[TMP52]]
+; UNROLL-NO-IC-NEXT:    [[TMP44]] = load double, ptr [[ARRAYIDX]], align 8
+; UNROLL-NO-IC-NEXT:    [[MUL:%.*]] = fmul double [[SCALAR_RECUR]], [[TMP44]]
 ; UNROLL-NO-IC-NEXT:    [[TOBOOL:%.*]] = fcmp une double [[MUL]], 0.000000e+00
 ; UNROLL-NO-IC-NEXT:    [[INC:%.*]] = zext i1 [[TOBOOL]] to i32
 ; UNROLL-NO-IC-NEXT:    [[A_1]] = add nsw i32 [[A_010]], [[INC]]
@@ -1640,35 +1633,34 @@ define i32 @PR33613(ptr %b, double %j, i32 %d) {
 ; UNROLL-NO-VF-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; UNROLL-NO-VF:       vector.body:
 ; UNROLL-NO-VF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-VF-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-VF-NEXT:    [[VEC_PHI3:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-VF-NEXT:    [[VECTOR_RECUR:%.*]] = phi double [ [[J:%.*]], [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
-; UNROLL-NO-VF-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; UNROLL-NO-VF-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 200
-; UNROLL-NO-VF-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP1]]
-; UNROLL-NO-VF-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 1
-; UNROLL-NO-VF-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 200
-; UNROLL-NO-VF-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP3]]
-; UNROLL-NO-VF-NEXT:    [[TMP4:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP]], i64 [[IDXPROM]]
-; UNROLL-NO-VF-NEXT:    [[TMP5:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP2]], i64 [[IDXPROM]]
-; UNROLL-NO-VF-NEXT:    [[TMP6:%.*]] = load double, ptr [[TMP4]], align 8
-; UNROLL-NO-VF-NEXT:    [[TMP7]] = load double, ptr [[TMP5]], align 8
-; UNROLL-NO-VF-NEXT:    [[TMP8:%.*]] = fmul double [[VECTOR_RECUR]], [[TMP6]]
-; UNROLL-NO-VF-NEXT:    [[TMP9:%.*]] = fmul double [[TMP6]], [[TMP7]]
-; UNROLL-NO-VF-NEXT:    [[TMP10:%.*]] = fcmp une double [[TMP8]], 0.000000e+00
-; UNROLL-NO-VF-NEXT:    [[TMP11:%.*]] = fcmp une double [[TMP9]], 0.000000e+00
-; UNROLL-NO-VF-NEXT:    [[TMP12:%.*]] = zext i1 [[TMP10]] to i32
-; UNROLL-NO-VF-NEXT:    [[TMP13:%.*]] = zext i1 [[TMP11]] to i32
-; UNROLL-NO-VF-NEXT:    [[TMP14]] = add i32 [[VEC_PHI]], [[TMP12]]
-; UNROLL-NO-VF-NEXT:    [[TMP15]] = add i32 [[VEC_PHI3]], [[TMP13]]
+; UNROLL-NO-VF-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
+; UNROLL-NO-VF-NEXT:    [[VEC_PHI2:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; UNROLL-NO-VF-NEXT:    [[VECTOR_RECUR:%.*]] = phi double [ [[J:%.*]], [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; UNROLL-NO-VF-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 200
+; UNROLL-NO-VF-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; UNROLL-NO-VF-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 200
+; UNROLL-NO-VF-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
+; UNROLL-NO-VF-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP1]]
+; UNROLL-NO-VF-NEXT:    [[TMP2:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP]], i64 [[IDXPROM]]
+; UNROLL-NO-VF-NEXT:    [[TMP3:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP3]], i64 [[IDXPROM]]
+; UNROLL-NO-VF-NEXT:    [[TMP4:%.*]] = load double, ptr [[TMP2]], align 8
+; UNROLL-NO-VF-NEXT:    [[TMP5]] = load double, ptr [[TMP3]], align 8
+; UNROLL-NO-VF-NEXT:    [[TMP6:%.*]] = fmul double [[VECTOR_RECUR]], [[TMP4]]
+; UNROLL-NO-VF-NEXT:    [[TMP7:%.*]] = fmul double [[TMP4]], [[TMP5]]
+; UNROLL-NO-VF-NEXT:    [[TMP8:%.*]] = fcmp une double [[TMP6]], 0.000000e+00
+; UNROLL-NO-VF-NEXT:    [[TMP9:%.*]] = fcmp une double [[TMP7]], 0.000000e+00
+; UNROLL-NO-VF-NEXT:    [[TMP10:%.*]] = zext i1 [[TMP8]] to i32
+; UNROLL-NO-VF-NEXT:    [[TMP11:%.*]] = zext i1 [[TMP9]] to i32
+; UNROLL-NO-VF-NEXT:    [[TMP12]] = add i32 [[VEC_PHI]], [[TMP10]]
+; UNROLL-NO-VF-NEXT:    [[TMP13]] = add i32 [[VEC_PHI2]], [[TMP11]]
 ; UNROLL-NO-VF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; UNROLL-NO-VF-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10240
-; UNROLL-NO-VF-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; UNROLL-NO-VF-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10240
+; UNROLL-NO-VF-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; UNROLL-NO-VF:       middle.block:
-; UNROLL-NO-VF-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP15]], [[TMP14]]
+; UNROLL-NO-VF-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP13]], [[TMP12]]
 ; UNROLL-NO-VF-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NO-VF:       scalar.ph:
-; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi double [ [[J]], [[ENTRY:%.*]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
+; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi double [ [[J]], [[ENTRY:%.*]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]
 ; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ]
 ; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 10240, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; UNROLL-NO-VF-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
@@ -1680,10 +1672,10 @@ define i32 @PR33613(ptr %b, double %j, i32 %d) {
 ; UNROLL-NO-VF-NEXT:    [[B_ADDR_012:%.*]] = phi ptr [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD_PTR:%.*]], [[FOR_BODY]] ]
 ; UNROLL-NO-VF-NEXT:    [[I_011:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[INC1:%.*]], [[FOR_BODY]] ]
 ; UNROLL-NO-VF-NEXT:    [[A_010:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[A_1]], [[FOR_BODY]] ]
-; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR:%.*]] = phi double [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP17:%.*]], [[FOR_BODY]] ]
+; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR:%.*]] = phi double [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP15:%.*]], [[FOR_BODY]] ]
 ; UNROLL-NO-VF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[B_ADDR_012]], i64 [[IDXPROM]]
-; UNROLL-NO-VF-NEXT:    [[TMP17]] = load double, ptr [[ARRAYIDX]], align 8
-; UNROLL-NO-VF-NEXT:    [[MUL:%.*]] = fmul double [[SCALAR_RECUR]], [[TMP17]]
+; UNROLL-NO-VF-NEXT:    [[TMP15]] = load double, ptr [[ARRAYIDX]], align 8
+; UNROLL-NO-VF-NEXT:    [[MUL:%.*]] = fmul double [[SCALAR_RECUR]], [[TMP15]]
 ; UNROLL-NO-VF-NEXT:    [[TOBOOL:%.*]] = fcmp une double [[MUL]], 0.000000e+00
 ; UNROLL-NO-VF-NEXT:    [[INC:%.*]] = zext i1 [[TOBOOL]] to i32
 ; UNROLL-NO-VF-NEXT:    [[A_1]] = add nsw i32 [[A_010]], [[INC]]
@@ -1702,61 +1694,58 @@ define i32 @PR33613(ptr %b, double %j, i32 %d) {
 ; SINK-AFTER-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SINK-AFTER:       vector.body:
 ; SINK-AFTER-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SINK-AFTER-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
-; SINK-AFTER-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x double> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
-; SINK-AFTER-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; SINK-AFTER-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 200
-; SINK-AFTER-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP1]]
-; SINK-AFTER-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 1
-; SINK-AFTER-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 200
-; SINK-AFTER-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP3]]
-; SINK-AFTER-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 2
-; SINK-AFTER-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 200
-; SINK-AFTER-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP5]]
-; SINK-AFTER-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 3
-; SINK-AFTER-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 200
-; SINK-AFTER-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP7]]
-; SINK-AFTER-NEXT:    [[TMP8:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP]], i64 [[IDXPROM]]
-; SINK-AFTER-NEXT:    [[TMP9:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP2]], i64 [[IDXPROM]]
-; SINK-AFTER-NEXT:    [[TMP10:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP3]], i64 [[IDXPROM]]
-; SINK-AFTER-NEXT:    [[TMP11:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP4]], i64 [[IDXPROM]]
-; SINK-AFTER-NEXT:    [[TMP12:%.*]] = load double, ptr [[TMP8]], align 8
-; SINK-AFTER-NEXT:    [[TMP13:%.*]] = load double, ptr [[TMP9]], align 8
-; SINK-AFTER-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP10]], align 8
-; SINK-AFTER-NEXT:    [[TMP15:%.*]] = load double, ptr [[TMP11]], align 8
-; SINK-AFTER-NEXT:    [[TMP16:%.*]] = insertelement <4 x double> poison, double [[TMP12]], i32 0
-; SINK-AFTER-NEXT:    [[TMP17:%.*]] = insertelement <4 x double> [[TMP16]], double [[TMP13]], i32 1
-; SINK-AFTER-NEXT:    [[TMP18:%.*]] = insertelement <4 x double> [[TMP17]], double [[TMP14]], i32 2
-; SINK-AFTER-NEXT:    [[TMP19]] = insertelement <4 x double> [[TMP18]], double [[TMP15]], i32 3
-; SINK-AFTER-NEXT:    [[TMP20:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[TMP19]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; SINK-AFTER-NEXT:    [[TMP21:%.*]] = fmul <4 x double> [[TMP20]], [[TMP19]]
-; SINK-AFTER-NEXT:    [[TMP22:%.*]] = fcmp une <4 x double> [[TMP21]], zeroinitializer
-; SINK-AFTER-NEXT:    [[TMP23:%.*]] = zext <4 x i1> [[TMP22]] to <4 x i32>
-; SINK-AFTER-NEXT:    [[TMP24]] = add <4 x i32> [[VEC_PHI]], [[TMP23]]
+; SINK-AFTER-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ]
+; SINK-AFTER-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x double> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; SINK-AFTER-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 200
+; SINK-AFTER-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; SINK-AFTER-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 200
+; SINK-AFTER-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 400
+; SINK-AFTER-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 600
+; SINK-AFTER-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
+; SINK-AFTER-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP1]]
+; SINK-AFTER-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
+; SINK-AFTER-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP3]]
+; SINK-AFTER-NEXT:    [[TMP4:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP]], i64 [[IDXPROM]]
+; SINK-AFTER-NEXT:    [[TMP5:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP2]], i64 [[IDXPROM]]
+; SINK-AFTER-NEXT:    [[TMP6:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP3]], i64 [[IDXPROM]]
+; SINK-AFTER-NEXT:    [[TMP7:%.*]] = getelementptr inbounds double, ptr [[NEXT_GEP4]], i64 [[IDXPROM]]
+; SINK-AFTER-NEXT:    [[TMP8:%.*]] = load double, ptr [[TMP4]], align 8
+; SINK-AFTER-NEXT:    [[TMP9:%.*]] = load double, ptr [[TMP5]], align 8
+; SINK-AFTER-NEXT:    [[TMP10:%.*]] = load double, ptr [[TMP6]], align 8
+; SINK-AFTER-NEXT:    [[TMP11:%.*]] = load double, ptr [[TMP7]], align 8
+; SINK-AFTER-NEXT:    [[TMP12:%.*]] = insertelement <4 x double> poison, double [[TMP8]], i32 0
+; SINK-AFTER-NEXT:    [[TMP13:%.*]] = insertelement <4 x double> [[TMP12]], double [[TMP9]], i32 1
+; SINK-AFTER-NEXT:    [[TMP14:%.*]] = insertelement <4 x double> [[TMP13]], double [[TMP10]], i32 2
+; SINK-AFTER-NEXT:    [[TMP15]] = insertelement <4 x double> [[TMP14]], double [[TMP11]], i32 3
+; SINK-AFTER-NEXT:    [[TMP16:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[TMP15]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; SINK-AFTER-NEXT:    [[TMP17:%.*]] = fmul <4 x double> [[TMP16]], [[TMP15]]
+; SINK-AFTER-NEXT:    [[TMP18:%.*]] = fcmp une <4 x double> [[TMP17]], zeroinitializer
+; SINK-AFTER-NEXT:    [[TMP19:%.*]] = zext <4 x i1> [[TMP18]] to <4 x i32>
+; SINK-AFTER-NEXT:    [[TMP20]] = add <4 x i32> [[VEC_PHI]], [[TMP19]]
 ; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; SINK-AFTER-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10240
-; SINK-AFTER-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; SINK-AFTER-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10240
+; SINK-AFTER-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; SINK-AFTER:       middle.block:
-; SINK-AFTER-NEXT:    [[TMP26:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP24]])
-; SINK-AFTER-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x double> [[TMP19]], i32 3
+; SINK-AFTER-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP20]])
+; SINK-AFTER-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x double> [[TMP15]], i32 3
 ; SINK-AFTER-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
 ; SINK-AFTER:       scalar.ph:
 ; SINK-AFTER-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi double [ [[J]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]
 ; SINK-AFTER-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ]
 ; SINK-AFTER-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 10240, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; SINK-AFTER-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP26]], [[MIDDLE_BLOCK]] ]
+; SINK-AFTER-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP22]], [[MIDDLE_BLOCK]] ]
 ; SINK-AFTER-NEXT:    br label [[FOR_BODY:%.*]]
 ; SINK-AFTER:       for.cond.cleanup:
-; SINK-AFTER-NEXT:    [[A_1_LCSSA:%.*]] = phi i32 [ [[A_1:%.*]], [[FOR_BODY]] ], [ [[TMP26]], [[MIDDLE_BLOCK]] ]
+; SINK-AFTER-NEXT:    [[A_1_LCSSA:%.*]] = phi i32 [ [[A_1:%.*]], [[FOR_BODY]] ], [ [[TMP22]], [[MIDDLE_BLOCK]] ]
 ; SINK-AFTER-NEXT:    ret i32 [[A_1_LCSSA]]
 ; SINK-AFTER:       for.body:
 ; SINK-AFTER-NEXT:    [[B_ADDR_012:%.*]] = phi ptr [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD_PTR:%.*]], [[FOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[I_011:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[INC1:%.*]], [[FOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[A_010:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[A_1]], [[FOR_BODY]] ]
-; SINK-AFTER-NEXT:    [[SCALAR_RECUR:%.*]] = phi double [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP27:%.*]], [[FOR_BODY]] ]
+; SINK-AFTER-NEXT:    [[SCALAR_RECUR:%.*]] = phi double [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[TMP23:%.*]], [[FOR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[B_ADDR_012]], i64 [[IDXPROM]]
-; SINK-AFTER-NEXT:    [[TMP27]] = load double, ptr [[ARRAYIDX]], align 8
-; SINK-AFTER-NEXT:    [[MUL:%.*]] = fmul double [[SCALAR_RECUR]], [[TMP27]]
+; SINK-AFTER-NEXT:    [[TMP23]] = load double, ptr [[ARRAYIDX]], align 8
+; SINK-AFTER-NEXT:    [[MUL:%.*]] = fmul double [[SCALAR_RECUR]], [[TMP23]]
 ; SINK-AFTER-NEXT:    [[TOBOOL:%.*]] = fcmp une double [[MUL]], 0.000000e+00
 ; SINK-AFTER-NEXT:    [[INC:%.*]] = zext i1 [[TOBOOL]] to i32
 ; SINK-AFTER-NEXT:    [[A_1]] = add nsw i32 [[A_010]], [[INC]]
diff --git a/llvm/test/Transforms/LoopVectorize/pointer-induction-unroll.ll b/llvm/test/Transforms/LoopVectorize/pointer-induction-unroll.ll
index 410947704fea..31d862a3438a 100644
--- a/llvm/test/Transforms/LoopVectorize/pointer-induction-unroll.ll
+++ b/llvm/test/Transforms/LoopVectorize/pointer-induction-unroll.ll
@@ -32,35 +32,36 @@ define void @non_constant_scalar_expansion(i32 %0, ptr %call) {
 ; STRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; STRIDED:       vector.body:
 ; STRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; STRIDED-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 0
-; STRIDED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], [[TMP1]]
-; STRIDED-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr null, i64 [[TMP5]]
-; STRIDED-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 1
-; STRIDED-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], [[TMP1]]
-; STRIDED-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr null, i64 [[TMP7]]
-; STRIDED-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 2
-; STRIDED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], [[TMP1]]
-; STRIDED-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr null, i64 [[TMP9]]
-; STRIDED-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 3
-; STRIDED-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], [[TMP1]]
-; STRIDED-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr null, i64 [[TMP11]]
 ; STRIDED-NEXT:    [[DOTCAST:%.*]] = trunc i64 [[INDEX]] to i32
 ; STRIDED-NEXT:    [[OFFSET_IDX:%.*]] = add i32 30, [[DOTCAST]]
-; STRIDED-NEXT:    [[TMP12:%.*]] = add i32 [[OFFSET_IDX]], 0
-; STRIDED-NEXT:    [[TMP13:%.*]] = add i32 [[OFFSET_IDX]], 1
-; STRIDED-NEXT:    [[TMP14:%.*]] = add i32 [[OFFSET_IDX]], 2
-; STRIDED-NEXT:    [[TMP15:%.*]] = add i32 [[OFFSET_IDX]], 3
-; STRIDED-NEXT:    [[TMP16:%.*]] = getelementptr ptr, ptr [[CALL:%.*]], i32 [[TMP12]]
-; STRIDED-NEXT:    [[TMP17:%.*]] = getelementptr ptr, ptr [[CALL]], i32 [[TMP13]]
-; STRIDED-NEXT:    [[TMP18:%.*]] = getelementptr ptr, ptr [[CALL]], i32 [[TMP14]]
-; STRIDED-NEXT:    [[TMP19:%.*]] = getelementptr ptr, ptr [[CALL]], i32 [[TMP15]]
-; STRIDED-NEXT:    store ptr [[NEXT_GEP]], ptr [[TMP16]], align 4
-; STRIDED-NEXT:    store ptr [[NEXT_GEP2]], ptr [[TMP17]], align 4
-; STRIDED-NEXT:    store ptr [[NEXT_GEP3]], ptr [[TMP18]], align 4
-; STRIDED-NEXT:    store ptr [[NEXT_GEP4]], ptr [[TMP19]], align 4
+; STRIDED-NEXT:    [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], 0
+; STRIDED-NEXT:    [[TMP4:%.*]] = add i32 [[OFFSET_IDX]], 1
+; STRIDED-NEXT:    [[TMP5:%.*]] = add i32 [[OFFSET_IDX]], 2
+; STRIDED-NEXT:    [[TMP6:%.*]] = add i32 [[OFFSET_IDX]], 3
+; STRIDED-NEXT:    [[OFFSET_IDX2:%.*]] = mul i64 [[INDEX]], [[TMP1]]
+; STRIDED-NEXT:    [[TMP7:%.*]] = mul i64 0, [[TMP1]]
+; STRIDED-NEXT:    [[TMP8:%.*]] = add i64 [[OFFSET_IDX2]], [[TMP7]]
+; STRIDED-NEXT:    [[TMP9:%.*]] = mul i64 1, [[TMP1]]
+; STRIDED-NEXT:    [[TMP10:%.*]] = add i64 [[OFFSET_IDX2]], [[TMP9]]
+; STRIDED-NEXT:    [[TMP11:%.*]] = mul i64 2, [[TMP1]]
+; STRIDED-NEXT:    [[TMP12:%.*]] = add i64 [[OFFSET_IDX2]], [[TMP11]]
+; STRIDED-NEXT:    [[TMP13:%.*]] = mul i64 3, [[TMP1]]
+; STRIDED-NEXT:    [[TMP14:%.*]] = add i64 [[OFFSET_IDX2]], [[TMP13]]
+; STRIDED-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr null, i64 [[TMP8]]
+; STRIDED-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr null, i64 [[TMP10]]
+; STRIDED-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr null, i64 [[TMP12]]
+; STRIDED-NEXT:    [[TMP18:%.*]] = getelementptr i8, ptr null, i64 [[TMP14]]
+; STRIDED-NEXT:    [[TMP19:%.*]] = getelementptr ptr, ptr [[CALL:%.*]], i32 [[TMP3]]
+; STRIDED-NEXT:    [[TMP20:%.*]] = getelementptr ptr, ptr [[CALL]], i32 [[TMP4]]
+; STRIDED-NEXT:    [[TMP21:%.*]] = getelementptr ptr, ptr [[CALL]], i32 [[TMP5]]
+; STRIDED-NEXT:    [[TMP22:%.*]] = getelementptr ptr, ptr [[CALL]], i32 [[TMP6]]
+; STRIDED-NEXT:    store ptr [[TMP15]], ptr [[TMP19]], align 4
+; STRIDED-NEXT:    store ptr [[TMP16]], ptr [[TMP20]], align 4
+; STRIDED-NEXT:    store ptr [[TMP17]], ptr [[TMP21]], align 4
+; STRIDED-NEXT:    store ptr [[TMP18]], ptr [[TMP22]], align 4
 ; STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; STRIDED-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4294967264
-; STRIDED-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; STRIDED-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4294967264
+; STRIDED-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; STRIDED:       middle.block:
 ; STRIDED-NEXT:    br i1 false, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; STRIDED:       scalar.ph:
@@ -68,13 +69,13 @@ define void @non_constant_scalar_expansion(i32 %0, ptr %call) {
 ; STRIDED-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ null, [[ENTRY]] ]
 ; STRIDED-NEXT:    br label [[FOR_COND:%.*]]
 ; STRIDED:       for.cond:
-; STRIDED-NEXT:    [[TMP21:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_COND]] ]
+; STRIDED-NEXT:    [[TMP24:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_COND]] ]
 ; STRIDED-NEXT:    [[P_0:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[ADD_PTR:%.*]], [[FOR_COND]] ]
 ; STRIDED-NEXT:    [[ADD_PTR]] = getelementptr i8, ptr [[P_0]], i32 [[MUL]]
-; STRIDED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr ptr, ptr [[CALL]], i32 [[TMP21]]
+; STRIDED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr ptr, ptr [[CALL]], i32 [[TMP24]]
 ; STRIDED-NEXT:    store ptr [[P_0]], ptr [[ARRAYIDX]], align 4
-; STRIDED-NEXT:    [[INC]] = add i32 [[TMP21]], 1
-; STRIDED-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP21]], 0
+; STRIDED-NEXT:    [[INC]] = add i32 [[TMP24]], 1
+; STRIDED-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP24]], 0
 ; STRIDED-NEXT:    br i1 [[TOBOOL_NOT]], label [[FOR_END]], label [[FOR_COND]], !llvm.loop [[LOOP3:![0-9]+]]
 ; STRIDED:       for.end:
 ; STRIDED-NEXT:    ret void
diff --git a/llvm/test/Transforms/LoopVectorize/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
index 27f6f7b1cb4e..35037968160c 100644
--- a/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
@@ -23,56 +23,53 @@ define void @a(ptr readnone %b) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE10:%.*]] ]
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], -1
-; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr null, i64 [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[NEXT_GEP]], i64 -1
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i32 -3
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP6]], align 1
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 0, [[INDEX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr null, i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[NEXT_GEP]], i64 -1
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 -3
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP5]], align 1
 ; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq <4 x i8> [[REVERSE]], zeroinitializer
-; CHECK-NEXT:    [[TMP8:%.*]] = xor <4 x i1> [[TMP7]], <i1 true, i1 true, i1 true, i1 true>
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP8]], i32 0
-; CHECK-NEXT:    br i1 [[TMP9]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq <4 x i8> [[REVERSE]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i1> [[TMP6]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i1> [[TMP7]], i32 0
+; CHECK-NEXT:    br i1 [[TMP8]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[NEXT_GEP]], i64 -1
-; CHECK-NEXT:    store i8 95, ptr [[TMP10]], align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[NEXT_GEP]], i64 -1
+; CHECK-NEXT:    store i8 95, ptr [[TMP9]], align 1
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; CHECK:       pred.store.continue:
-; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP8]], i32 1
-; CHECK-NEXT:    br i1 [[TMP11]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6:%.*]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP7]], i32 1
+; CHECK-NEXT:    br i1 [[TMP10]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6:%.*]]
 ; CHECK:       pred.store.if5:
-; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], -1
-; CHECK-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr null, i64 [[TMP13]]
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[NEXT_GEP2]], i64 -1
-; CHECK-NEXT:    store i8 95, ptr [[TMP14]], align 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], -1
+; CHECK-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr null, i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[NEXT_GEP2]], i64 -1
+; CHECK-NEXT:    store i8 95, ptr [[TMP12]], align 1
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; CHECK:       pred.store.continue6:
-; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i1> [[TMP8]], i32 2
-; CHECK-NEXT:    br i1 [[TMP15]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i1> [[TMP7]], i32 2
+; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
 ; CHECK:       pred.store.if7:
-; CHECK-NEXT:    [[TMP16:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], -1
-; CHECK-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr null, i64 [[TMP17]]
-; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[NEXT_GEP3]], i64 -1
-; CHECK-NEXT:    store i8 95, ptr [[TMP18]], align 1
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[OFFSET_IDX]], -2
+; CHECK-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr null, i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[NEXT_GEP3]], i64 -1
+; CHECK-NEXT:    store i8 95, ptr [[TMP15]], align 1
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE8]]
 ; CHECK:       pred.store.continue8:
-; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <4 x i1> [[TMP8]], i32 3
-; CHECK-NEXT:    br i1 [[TMP19]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP7]], i32 3
+; CHECK-NEXT:    br i1 [[TMP16]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10]]
 ; CHECK:       pred.store.if9:
-; CHECK-NEXT:    [[TMP20:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], -1
-; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr null, i64 [[TMP21]]
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[NEXT_GEP4]], i64 -1
-; CHECK-NEXT:    store i8 95, ptr [[TMP22]], align 1
+; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[OFFSET_IDX]], -3
+; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr null, i64 [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[NEXT_GEP4]], i64 -1
+; CHECK-NEXT:    store i8 95, ptr [[TMP18]], align 1
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE10]]
 ; CHECK:       pred.store.continue10:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
@@ -86,8 +83,8 @@ define void @a(ptr readnone %b) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[C_05:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[IF_END:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[C_05]], i64 -1
-; CHECK-NEXT:    [[TMP24:%.*]] = load i8, ptr [[INCDEC_PTR]], align 1
-; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i8 [[TMP24]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = load i8, ptr [[INCDEC_PTR]], align 1
+; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i8 [[TMP20]], 0
 ; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label [[IF_END]], label [[IF_THEN:%.*]]
 ; CHECK:       if.then:
 ; CHECK-NEXT:    store i8 95, ptr [[INCDEC_PTR]], align 1
@@ -145,22 +142,22 @@ define void @pointer_induction_used_as_vector(ptr noalias %start.1, ptr noalias
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[START_2]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START_1]], i64 [[TMP2]]
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> <i64 0, i64 1, i64 2, i64 3>
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, <4 x ptr> [[TMP3]], i64 1
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr ptr, ptr [[NEXT_GEP]], i32 0
-; CHECK-NEXT:    store <4 x ptr> [[TMP4]], ptr [[TMP5]], align 8
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x ptr> [[TMP3]], i32 0
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP7]], align 1
-; CHECK-NEXT:    [[TMP8:%.*]] = add <4 x i8> [[WIDE_LOAD]], <i8 1, i8 1, i8 1, i8 1>
-; CHECK-NEXT:    store <4 x i8> [[TMP8]], ptr [[TMP7]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, <4 x ptr> [[TMP1]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr ptr, ptr [[NEXT_GEP]], i32 0
+; CHECK-NEXT:    store <4 x ptr> [[TMP3]], ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x ptr> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP6]], align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = add <4 x i8> [[WIDE_LOAD]], <i8 1, i8 1, i8 1, i8 1>
+; CHECK-NEXT:    store <4 x i8> [[TMP7]], ptr [[TMP6]], align 1
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 4
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
index 0c659a550b31..92ca77bc841c 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
@@ -92,6 +92,7 @@ define void @VF1-VPWidenCanonicalIVRecipeExe(ptr %ptr1) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE12:%.*]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[VEC_IV:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-NEXT:    [[VEC_IV4:%.*]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[VEC_IV5:%.*]] = add i64 [[INDEX]], 2
@@ -102,39 +103,35 @@ define void @VF1-VPWidenCanonicalIVRecipeExe(ptr %ptr1) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i64 [[VEC_IV6]], 14
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
-; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
-; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP4]]
 ; CHECK-NEXT:    store double 0.000000e+00, ptr [[NEXT_GEP]], align 8
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; CHECK:       pred.store.continue:
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
 ; CHECK:       pred.store.if7:
-; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 8
-; CHECK-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 8
+; CHECK-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP5]]
 ; CHECK-NEXT:    store double 0.000000e+00, ptr [[NEXT_GEP1]], align 8
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE8]]
 ; CHECK:       pred.store.continue8:
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]
 ; CHECK:       pred.store.if9:
-; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
-; CHECK-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 16
+; CHECK-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP6]]
 ; CHECK-NEXT:    store double 0.000000e+00, ptr [[NEXT_GEP2]], align 8
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE10]]
 ; CHECK:       pred.store.continue10:
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12]]
 ; CHECK:       pred.store.if11:
-; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 8
-; CHECK-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 24
+; CHECK-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP7]]
 ; CHECK-NEXT:    store double 0.000000e+00, ptr [[NEXT_GEP3]], align 8
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE12]]
 ; CHECK:       pred.store.continue12:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
-; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
index 1dddbfe20a2e..ca9dfdc6f6d2 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
@@ -23,11 +23,12 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
 ; CHECK-NEXT:     EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
-; CHECK-NEXT:     EMIT ir<%p.src> = WIDEN-POINTER-INDUCTION ir<%A>, 1
-; CHECK-NEXT:     vp<[[VPTR:%.]]> = vector-pointer ir<%p.src>
+; CHECK-NEXT:     vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
+; CHECK-NEXT:     EMIT vp<[[PADD:%.+]]> = ptradd ir<%A>, vp<[[STEPS]]>
+; CHECK-NEXT:     vp<[[VPTR:%.]]> = vector-pointer vp<[[PADD]]>
 ; CHECK-NEXT:     WIDEN ir<%l> = load vp<[[VPTR]]>
 ; CHECK-NEXT:     WIDEN ir<%add> = add nsw ir<%l>, ir<10>
-; CHECK-NEXT:     vp<[[VPTR2:%.+]]> = vector-pointer ir<%p.src>
+; CHECK-NEXT:     vp<[[VPTR2:%.+]]> = vector-pointer vp<[[PADD]]>
 ; CHECK-NEXT:     WIDEN store vp<[[VPTR2]]>, ir<%add>
 ; CHECK-NEXT:     EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:     EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
@@ -54,11 +55,12 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
 ; CHECK-NEXT:     EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
-; CHECK-NEXT:     EMIT ir<%p.src> = WIDEN-POINTER-INDUCTION ir<%A>, 1
-; CHECK-NEXT:     vp<[[VPTR:%.]]> = vector-pointer ir<%p.src>
+; CHECK-NEXT:     vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
+; CHECK-NEXT:     EMIT vp<[[PADD:%.+]]> = ptradd ir<%A>, vp<[[STEPS]]>
+; CHECK-NEXT:     vp<[[VPTR:%.]]> = vector-pointer vp<[[PADD]]>
 ; CHECK-NEXT:     WIDEN ir<%l> = load vp<[[VPTR]]>
 ; CHECK-NEXT:     WIDEN ir<%add> = add nsw ir<%l>, ir<10>
-; CHECK-NEXT:     vp<[[VPTR2:%.+]]> = vector-pointer ir<%p.src>
+; CHECK-NEXT:     vp<[[VPTR2:%.+]]> = vector-pointer vp<[[PADD]]>
 ; CHECK-NEXT:     WIDEN store vp<[[VPTR2]]>, ir<%add>
 ; CHECK-NEXT:     EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:     EMIT branch-on-cond ir<true>
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index 9b9c3e704852..89b3a6da16c1 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -1113,8 +1113,10 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
 ; CHECK-NEXT:     EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
-; CHECK-NEXT:     EMIT ir<%ptr.iv> = WIDEN-POINTER-INDUCTION ir<%start>, -1
-; CHECK-NEXT:     CLONE ir<%ptr.iv.next> = getelementptr inbounds ir<%ptr.iv>, ir<-1>
+; CHECK-NEXT:     vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<0> + vp<%3> * ir<-1>
+; CHECK-NEXT:     vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
+; CHECK-NEXT:     EMIT vp<[[PTR_IV:%.+]]> = ptradd ir<%start>, vp<[[STEPS]]>
+; CHECK-NEXT:     CLONE ir<%ptr.iv.next> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1>
 ; CHECK-NEXT:     vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%ptr.iv.next>
 ; CHECK-NEXT:     WIDEN ir<%l> = load vp<[[VEC_PTR]]>
 ; CHECK-NEXT:     WIDEN ir<%c.1> = icmp eq ir<%l>, ir<0>
@@ -1127,7 +1129,7 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
 ; CHECK-NEXT:     Successor(s): pred.store.if, pred.store.continue
 ; CHECK-EMPTY:
 ; CHECK-NEXT:     pred.store.if:
-; CHECK-NEXT:       REPLICATE ir<%ptr.iv.next> = getelementptr inbounds ir<%ptr.iv>, ir<-1>
+; CHECK-NEXT:       REPLICATE ir<%ptr.iv.next> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1>
 ; CHECK-NEXT:       REPLICATE store ir<95>, ir<%ptr.iv.next>
 ; CHECK-NEXT:     Successor(s): pred.store.continue
 ; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll b/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll
index 73bcee5fb74f..36bcda4c43ca 100644
--- a/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll
+++ b/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll
@@ -49,7 +49,7 @@ define void @arm_mult_q15(ptr %pSrcA, ptr %pSrcB, ptr noalias %pDst, i32 %blockS
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_VEC]], [[BLOCKSIZE]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[WHILE_END]], label [[WHILE_BODY_PREHEADER16]]
-; CHECK:       while.body.preheader16:
+; CHECK:       while.body.preheader18:
 ; CHECK-NEXT:    [[BLKCNT_06_PH:%.*]] = phi i32 [ [[BLOCKSIZE]], [[WHILE_BODY_PREHEADER]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    [[PSRCA_ADDR_05_PH:%.*]] = phi ptr [ [[PSRCA]], [[WHILE_BODY_PREHEADER]] ], [ [[IND_END7]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    [[PDST_ADDR_04_PH:%.*]] = phi ptr [ [[PDST]], [[WHILE_BODY_PREHEADER]] ], [ [[IND_END9]], [[MIDDLE_BLOCK]] ]
-- 
GitLab


From e933c05cd2cb97974cfca9544ebbdb9af349d602 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 26 Mar 2024 13:55:54 +0000
Subject: [PATCH 287/404] [X86] Add fadd/fsub/fmul tests showing failure to
 concat operands together and perform as a wider vector

We don't want to concat fadd/fsub/fmul if both operands would need concatenating (as the fp op is usually cheaper than the concat), but if at least one operand is free to concat (i.e. constant or extracted from a wider vector), then we should try to concat the fp op.
---
 llvm/test/CodeGen/X86/widen_fadd.ll | 74 +++++++++++++++++++++++++++--
 llvm/test/CodeGen/X86/widen_fmul.ll | 74 +++++++++++++++++++++++++++--
 llvm/test/CodeGen/X86/widen_fsub.ll | 74 +++++++++++++++++++++++++++--
 3 files changed, 210 insertions(+), 12 deletions(-)

diff --git a/llvm/test/CodeGen/X86/widen_fadd.ll b/llvm/test/CodeGen/X86/widen_fadd.ll
index 68f2ed436804..8277d532bfce 100644
--- a/llvm/test/CodeGen/X86/widen_fadd.ll
+++ b/llvm/test/CodeGen/X86/widen_fadd.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=SSE
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx  | FileCheck %s --check-prefixes=AVX,AVX1OR2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX1OR2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX,AVX512VL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx  | FileCheck %s --check-prefixes=AVX,AVX1OR2,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX1OR2,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX,AVX512,AVX512VL
 
 define void @widen_fadd_v2f32_v4f32(ptr %a0, ptr %b0, ptr %c0) {
 ; SSE-LABEL: widen_fadd_v2f32_v4f32:
@@ -364,3 +364,69 @@ define void @widen_fadd_v2f32_v16f32(ptr %a0, ptr %b0, ptr %c0) {
   store <2 x float> %vc14, ptr %c14, align 4
   ret void
 }
+
+define <8 x float> @widen_fadd_v4f32_v8f32_const(<4 x float> %x, <4 x float> %y) {
+; SSE-LABEL: widen_fadd_v4f32_v8f32_const:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm2 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; SSE-NEXT:    addps %xmm2, %xmm0
+; SSE-NEXT:    addps %xmm2, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: widen_fadd_v4f32_v8f32_const:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm2 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; AVX-NEXT:    vaddps %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vaddps %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    retq
+  %x2 = fadd <4 x float> %x, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %y2 = fadd <4 x float> %y, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %r = shufflevector <4 x float> %x2, <4 x float> %y2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x float> %r
+}
+
+define <16 x float> @widen_fadd_v4f32_v16f32_const(<4 x float> %x, <4 x float> %y, <4 x float> %z, <4 x float> %w) {
+; SSE-LABEL: widen_fadd_v4f32_v16f32_const:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm4 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; SSE-NEXT:    addps %xmm4, %xmm0
+; SSE-NEXT:    addps %xmm4, %xmm1
+; SSE-NEXT:    addps %xmm4, %xmm2
+; SSE-NEXT:    addps %xmm4, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX1OR2-LABEL: widen_fadd_v4f32_v16f32_const:
+; AVX1OR2:       # %bb.0:
+; AVX1OR2-NEXT:    vbroadcastss {{.*#+}} xmm4 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; AVX1OR2-NEXT:    vaddps %xmm4, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vaddps %xmm4, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vaddps %xmm4, %xmm2, %xmm2
+; AVX1OR2-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; AVX1OR2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm1
+; AVX1OR2-NEXT:    retq
+;
+; AVX512-LABEL: widen_fadd_v4f32_v16f32_const:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm4 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; AVX512-NEXT:    vaddps %xmm4, %xmm0, %xmm0
+; AVX512-NEXT:    vaddps %xmm4, %xmm1, %xmm1
+; AVX512-NEXT:    vaddps %xmm4, %xmm2, %xmm2
+; AVX512-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; AVX512-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX512-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-NEXT:    retq
+  %x2 = fadd <4 x float> %x, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %y2 = fadd <4 x float> %y, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %z2 = fadd <4 x float> %z, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %w2 = fadd <4 x float> %w, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %r0 = shufflevector <4 x float> %x2, <4 x float> %y2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %r1 = shufflevector <4 x float> %z2, <4 x float> %w2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %r = shufflevector <8 x float> %r0, <8 x float> %r1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x float> %r
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX1: {{.*}}
+; AVX2: {{.*}}
diff --git a/llvm/test/CodeGen/X86/widen_fmul.ll b/llvm/test/CodeGen/X86/widen_fmul.ll
index ac208da9ee11..3adc5b48539b 100644
--- a/llvm/test/CodeGen/X86/widen_fmul.ll
+++ b/llvm/test/CodeGen/X86/widen_fmul.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=SSE
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx  | FileCheck %s --check-prefixes=AVX,AVX1OR2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX1OR2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX,AVX512VL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx  | FileCheck %s --check-prefixes=AVX,AVX1OR2,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX1OR2,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX,AVX512,AVX512VL
 
 define void @widen_fmul_v2f32_v4f32(ptr %a0, ptr %b0, ptr %c0) {
 ; SSE-LABEL: widen_fmul_v2f32_v4f32:
@@ -364,3 +364,69 @@ define void @widen_fmul_v2f32_v16f32(ptr %a0, ptr %b0, ptr %c0) {
   store <2 x float> %vc14, ptr %c14, align 4
   ret void
 }
+
+define <8 x float> @widen_fmul_v4f32_v8f32_const(<4 x float> %x, <4 x float> %y) {
+; SSE-LABEL: widen_fmul_v4f32_v8f32_const:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm2 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; SSE-NEXT:    mulps %xmm2, %xmm0
+; SSE-NEXT:    mulps %xmm2, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: widen_fmul_v4f32_v8f32_const:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm2 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; AVX-NEXT:    vmulps %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vmulps %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    retq
+  %x2 = fmul <4 x float> %x, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
+  %y2 = fmul <4 x float> %y, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
+  %r = shufflevector <4 x float> %x2, <4 x float> %y2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x float> %r
+}
+
+define <16 x float> @widen_fmul_v4f32_v16f32_const(<4 x float> %x, <4 x float> %y, <4 x float> %z, <4 x float> %w) {
+; SSE-LABEL: widen_fmul_v4f32_v16f32_const:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm4 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; SSE-NEXT:    mulps %xmm4, %xmm0
+; SSE-NEXT:    mulps %xmm4, %xmm1
+; SSE-NEXT:    mulps %xmm4, %xmm2
+; SSE-NEXT:    mulps %xmm4, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX1OR2-LABEL: widen_fmul_v4f32_v16f32_const:
+; AVX1OR2:       # %bb.0:
+; AVX1OR2-NEXT:    vbroadcastss {{.*#+}} xmm4 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; AVX1OR2-NEXT:    vmulps %xmm4, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vmulps %xmm4, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vmulps %xmm4, %xmm2, %xmm2
+; AVX1OR2-NEXT:    vmulps %xmm4, %xmm3, %xmm3
+; AVX1OR2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm1
+; AVX1OR2-NEXT:    retq
+;
+; AVX512-LABEL: widen_fmul_v4f32_v16f32_const:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm4 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; AVX512-NEXT:    vmulps %xmm4, %xmm0, %xmm0
+; AVX512-NEXT:    vmulps %xmm4, %xmm1, %xmm1
+; AVX512-NEXT:    vmulps %xmm4, %xmm2, %xmm2
+; AVX512-NEXT:    vmulps %xmm4, %xmm3, %xmm3
+; AVX512-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX512-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-NEXT:    retq
+  %x2 = fmul <4 x float> %x, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
+  %y2 = fmul <4 x float> %y, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
+  %z2 = fmul <4 x float> %z, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
+  %w2 = fmul <4 x float> %w, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
+  %r0 = shufflevector <4 x float> %x2, <4 x float> %y2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %r1 = shufflevector <4 x float> %z2, <4 x float> %w2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %r = shufflevector <8 x float> %r0, <8 x float> %r1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x float> %r
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX1: {{.*}}
+; AVX2: {{.*}}
diff --git a/llvm/test/CodeGen/X86/widen_fsub.ll b/llvm/test/CodeGen/X86/widen_fsub.ll
index 90cf455ba61f..eb318181c698 100644
--- a/llvm/test/CodeGen/X86/widen_fsub.ll
+++ b/llvm/test/CodeGen/X86/widen_fsub.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=SSE
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx  | FileCheck %s --check-prefixes=AVX,AVX1OR2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX1OR2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX,AVX512VL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx  | FileCheck %s --check-prefixes=AVX,AVX1OR2,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX1OR2,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX,AVX512,AVX512VL
 
 define void @widen_fsub_v2f32_v4f32(ptr %a0, ptr %b0, ptr %c0) {
 ; SSE-LABEL: widen_fsub_v2f32_v4f32:
@@ -364,3 +364,69 @@ define void @widen_fsub_v2f32_v16f32(ptr %a0, ptr %b0, ptr %c0) {
   store <2 x float> %vc14, ptr %c14, align 4
   ret void
 }
+
+define <8 x float> @widen_fsub_v4f32_v8f32_const(<4 x float> %x, <4 x float> %y) {
+; SSE-LABEL: widen_fsub_v4f32_v8f32_const:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm2 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; SSE-NEXT:    subps %xmm2, %xmm0
+; SSE-NEXT:    subps %xmm2, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: widen_fsub_v4f32_v8f32_const:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm2 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; AVX-NEXT:    vsubps %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vsubps %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX-NEXT:    retq
+  %x2 = fsub <4 x float> %x, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %y2 = fsub <4 x float> %y, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %r = shufflevector <4 x float> %x2, <4 x float> %y2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x float> %r
+}
+
+define <16 x float> @widen_fsub_v4f32_v16f32_const(<4 x float> %x, <4 x float> %y, <4 x float> %z, <4 x float> %w) {
+; SSE-LABEL: widen_fsub_v4f32_v16f32_const:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm4 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; SSE-NEXT:    subps %xmm4, %xmm0
+; SSE-NEXT:    subps %xmm4, %xmm1
+; SSE-NEXT:    subps %xmm4, %xmm2
+; SSE-NEXT:    subps %xmm4, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX1OR2-LABEL: widen_fsub_v4f32_v16f32_const:
+; AVX1OR2:       # %bb.0:
+; AVX1OR2-NEXT:    vbroadcastss {{.*#+}} xmm4 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; AVX1OR2-NEXT:    vsubps %xmm4, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vsubps %xmm4, %xmm1, %xmm1
+; AVX1OR2-NEXT:    vsubps %xmm4, %xmm2, %xmm2
+; AVX1OR2-NEXT:    vsubps %xmm4, %xmm3, %xmm3
+; AVX1OR2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm1
+; AVX1OR2-NEXT:    retq
+;
+; AVX512-LABEL: widen_fsub_v4f32_v16f32_const:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm4 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; AVX512-NEXT:    vsubps %xmm4, %xmm0, %xmm0
+; AVX512-NEXT:    vsubps %xmm4, %xmm1, %xmm1
+; AVX512-NEXT:    vsubps %xmm4, %xmm2, %xmm2
+; AVX512-NEXT:    vsubps %xmm4, %xmm3, %xmm3
+; AVX512-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX512-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-NEXT:    retq
+  %x2 = fsub <4 x float> %x, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %y2 = fsub <4 x float> %y, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %z2 = fsub <4 x float> %z, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %w2 = fsub <4 x float> %w, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %r0 = shufflevector <4 x float> %x2, <4 x float> %y2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %r1 = shufflevector <4 x float> %z2, <4 x float> %w2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %r = shufflevector <8 x float> %r0, <8 x float> %r1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x float> %r
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX1: {{.*}}
+; AVX2: {{.*}}
-- 
GitLab


From d18bee2313ab8274fad386b1c0227316bf4238b2 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 26 Mar 2024 14:09:48 +0000
Subject: [PATCH 288/404] [X86] combineConcatVectorOps - concatenate
 FADD/FSUB/FMUL ops if we don't increase the number of INSERT_SUBVECTOR nodes.

FADD/FSUB/FMUL are usually less port-bound than INSERT_SUBVECTOR, so only concatenate if it reduces the instruction count and doesn't introduce extra INSERT_SUBVECTOR nodes.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 14 +++++-
 llvm/test/CodeGen/X86/widen_fadd.ll     | 59 ++++++++++++++++---------
 llvm/test/CodeGen/X86/widen_fmul.ll     | 59 ++++++++++++++++---------
 llvm/test/CodeGen/X86/widen_fsub.ll     | 59 ++++++++++++++++---------
 4 files changed, 127 insertions(+), 64 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 9acbe17d0bca..1ce742a7c19a 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -55529,7 +55529,19 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
       }
       break;
     // Due to VADD, VSUB, VMUL can executed on more ports than VINSERT and
-    // their latency are short, so here we don't replace them.
+    // their latency are short, so here we don't replace them unless we won't
+    // introduce extra VINSERT.
+    case ISD::FADD:
+    case ISD::FSUB:
+    case ISD::FMUL:
+      if (!IsSplat && (IsConcatFree(VT, Ops, 0) || IsConcatFree(VT, Ops, 1)) &&
+          (VT.is256BitVector() ||
+           (VT.is512BitVector() && Subtarget.useAVX512Regs()))) {
+        return DAG.getNode(Op0.getOpcode(), DL, VT,
+                           ConcatSubOperand(VT, Ops, 0),
+                           ConcatSubOperand(VT, Ops, 1));
+      }
+      break;
     case ISD::FDIV:
       if (!IsSplat && (VT.is256BitVector() ||
                        (VT.is512BitVector() && Subtarget.useAVX512Regs()))) {
diff --git a/llvm/test/CodeGen/X86/widen_fadd.ll b/llvm/test/CodeGen/X86/widen_fadd.ll
index 8277d532bfce..be249ddc0cca 100644
--- a/llvm/test/CodeGen/X86/widen_fadd.ll
+++ b/llvm/test/CodeGen/X86/widen_fadd.ll
@@ -373,13 +373,35 @@ define <8 x float> @widen_fadd_v4f32_v8f32_const(<4 x float> %x, <4 x float> %y)
 ; SSE-NEXT:    addps %xmm2, %xmm1
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: widen_fadd_v4f32_v8f32_const:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vbroadcastss {{.*#+}} xmm2 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; AVX-NEXT:    vaddps %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vaddps %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: widen_fadd_v4f32_v8f32_const:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: widen_fadd_v4f32_v8f32_const:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; AVX2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: widen_fadd_v4f32_v8f32_const:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512F-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vbroadcastss {{.*#+}} ymm1 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; AVX512F-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: widen_fadd_v4f32_v8f32_const:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512VL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
   %x2 = fadd <4 x float> %x, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
   %y2 = fadd <4 x float> %y, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
   %r = shufflevector <4 x float> %x2, <4 x float> %y2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -398,25 +420,23 @@ define <16 x float> @widen_fadd_v4f32_v16f32_const(<4 x float> %x, <4 x float> %
 ;
 ; AVX1OR2-LABEL: widen_fadd_v4f32_v16f32_const:
 ; AVX1OR2:       # %bb.0:
-; AVX1OR2-NEXT:    vbroadcastss {{.*#+}} xmm4 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; AVX1OR2-NEXT:    vaddps %xmm4, %xmm0, %xmm0
-; AVX1OR2-NEXT:    vaddps %xmm4, %xmm1, %xmm1
-; AVX1OR2-NEXT:    vaddps %xmm4, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; AVX1OR2-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
+; AVX1OR2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; AVX1OR2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1OR2-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm1
+; AVX1OR2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; AVX1OR2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1OR2-NEXT:    vaddps %ymm1, %ymm2, %ymm1
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512-LABEL: widen_fadd_v4f32_v16f32_const:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm4 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; AVX512-NEXT:    vaddps %xmm4, %xmm0, %xmm0
-; AVX512-NEXT:    vaddps %xmm4, %xmm1, %xmm1
-; AVX512-NEXT:    vaddps %xmm4, %xmm2, %xmm2
-; AVX512-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; AVX512-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; AVX512-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
 ; AVX512-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-NEXT:    vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
   %x2 = fadd <4 x float> %x, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
   %y2 = fadd <4 x float> %y, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
@@ -427,6 +447,3 @@ define <16 x float> @widen_fadd_v4f32_v16f32_const(<4 x float> %x, <4 x float> %
   %r = shufflevector <8 x float> %r0, <8 x float> %r1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <16 x float> %r
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX1: {{.*}}
-; AVX2: {{.*}}
diff --git a/llvm/test/CodeGen/X86/widen_fmul.ll b/llvm/test/CodeGen/X86/widen_fmul.ll
index 3adc5b48539b..9aa9d63e7fcb 100644
--- a/llvm/test/CodeGen/X86/widen_fmul.ll
+++ b/llvm/test/CodeGen/X86/widen_fmul.ll
@@ -373,13 +373,35 @@ define <8 x float> @widen_fmul_v4f32_v8f32_const(<4 x float> %x, <4 x float> %y)
 ; SSE-NEXT:    mulps %xmm2, %xmm1
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: widen_fmul_v4f32_v8f32_const:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vbroadcastss {{.*#+}} xmm2 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
-; AVX-NEXT:    vmulps %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vmulps %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: widen_fmul_v4f32_v8f32_const:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: widen_fmul_v4f32_v8f32_const:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; AVX2-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: widen_fmul_v4f32_v8f32_const:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512F-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vbroadcastss {{.*#+}} ymm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; AVX512F-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: widen_fmul_v4f32_v8f32_const:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512VL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
   %x2 = fmul <4 x float> %x, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
   %y2 = fmul <4 x float> %y, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
   %r = shufflevector <4 x float> %x2, <4 x float> %y2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -398,25 +420,23 @@ define <16 x float> @widen_fmul_v4f32_v16f32_const(<4 x float> %x, <4 x float> %
 ;
 ; AVX1OR2-LABEL: widen_fmul_v4f32_v16f32_const:
 ; AVX1OR2:       # %bb.0:
-; AVX1OR2-NEXT:    vbroadcastss {{.*#+}} xmm4 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
-; AVX1OR2-NEXT:    vmulps %xmm4, %xmm0, %xmm0
-; AVX1OR2-NEXT:    vmulps %xmm4, %xmm1, %xmm1
-; AVX1OR2-NEXT:    vmulps %xmm4, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vmulps %xmm4, %xmm3, %xmm3
+; AVX1OR2-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
+; AVX1OR2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; AVX1OR2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1OR2-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm1
+; AVX1OR2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; AVX1OR2-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1OR2-NEXT:    vmulps %ymm1, %ymm2, %ymm1
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512-LABEL: widen_fmul_v4f32_v16f32_const:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm4 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]
-; AVX512-NEXT:    vmulps %xmm4, %xmm0, %xmm0
-; AVX512-NEXT:    vmulps %xmm4, %xmm1, %xmm1
-; AVX512-NEXT:    vmulps %xmm4, %xmm2, %xmm2
-; AVX512-NEXT:    vmulps %xmm4, %xmm3, %xmm3
+; AVX512-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; AVX512-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
 ; AVX512-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-NEXT:    vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
   %x2 = fmul <4 x float> %x, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
   %y2 = fmul <4 x float> %y, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
@@ -427,6 +447,3 @@ define <16 x float> @widen_fmul_v4f32_v16f32_const(<4 x float> %x, <4 x float> %
   %r = shufflevector <8 x float> %r0, <8 x float> %r1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <16 x float> %r
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX1: {{.*}}
-; AVX2: {{.*}}
diff --git a/llvm/test/CodeGen/X86/widen_fsub.ll b/llvm/test/CodeGen/X86/widen_fsub.ll
index eb318181c698..60e54ab71abc 100644
--- a/llvm/test/CodeGen/X86/widen_fsub.ll
+++ b/llvm/test/CodeGen/X86/widen_fsub.ll
@@ -373,13 +373,35 @@ define <8 x float> @widen_fsub_v4f32_v8f32_const(<4 x float> %x, <4 x float> %y)
 ; SSE-NEXT:    subps %xmm2, %xmm1
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: widen_fsub_v4f32_v8f32_const:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vbroadcastss {{.*#+}} xmm2 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; AVX-NEXT:    vsubps %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vsubps %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: widen_fsub_v4f32_v8f32_const:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: widen_fsub_v4f32_v8f32_const:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0]
+; AVX2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: widen_fsub_v4f32_v8f32_const:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512F-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0]
+; AVX512F-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: widen_fsub_v4f32_v8f32_const:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512VL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
   %x2 = fsub <4 x float> %x, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
   %y2 = fsub <4 x float> %y, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
   %r = shufflevector <4 x float> %x2, <4 x float> %y2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -398,25 +420,23 @@ define <16 x float> @widen_fsub_v4f32_v16f32_const(<4 x float> %x, <4 x float> %
 ;
 ; AVX1OR2-LABEL: widen_fsub_v4f32_v16f32_const:
 ; AVX1OR2:       # %bb.0:
-; AVX1OR2-NEXT:    vbroadcastss {{.*#+}} xmm4 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; AVX1OR2-NEXT:    vsubps %xmm4, %xmm0, %xmm0
-; AVX1OR2-NEXT:    vsubps %xmm4, %xmm1, %xmm1
-; AVX1OR2-NEXT:    vsubps %xmm4, %xmm2, %xmm2
-; AVX1OR2-NEXT:    vsubps %xmm4, %xmm3, %xmm3
+; AVX1OR2-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
+; AVX1OR2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; AVX1OR2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1OR2-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm1
+; AVX1OR2-NEXT:    vbroadcastss {{.*#+}} ymm1 = [-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0,-2.0E+0]
+; AVX1OR2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX1OR2-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1OR2-NEXT:    vaddps %ymm1, %ymm2, %ymm1
 ; AVX1OR2-NEXT:    retq
 ;
 ; AVX512-LABEL: widen_fsub_v4f32_v16f32_const:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm4 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]
-; AVX512-NEXT:    vsubps %xmm4, %xmm0, %xmm0
-; AVX512-NEXT:    vsubps %xmm4, %xmm1, %xmm1
-; AVX512-NEXT:    vsubps %xmm4, %xmm2, %xmm2
-; AVX512-NEXT:    vsubps %xmm4, %xmm3, %xmm3
+; AVX512-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; AVX512-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
 ; AVX512-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vinsertf64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512-NEXT:    vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
   %x2 = fsub <4 x float> %x, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
   %y2 = fsub <4 x float> %y, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
@@ -427,6 +447,3 @@ define <16 x float> @widen_fsub_v4f32_v16f32_const(<4 x float> %x, <4 x float> %
   %r = shufflevector <8 x float> %r0, <8 x float> %r1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <16 x float> %r
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX1: {{.*}}
-; AVX2: {{.*}}
-- 
GitLab


From ecfffbfd3921fd47f3afc1d03a7518b34ec396b3 Mon Sep 17 00:00:00 2001
From: Nick Desaulniers <nickdesaulniers@users.noreply.github.com>
Date: Tue, 26 Mar 2024 08:27:43 -0700
Subject: [PATCH 289/404] [libc][OSUtil] refactor quick_exit to be an object
 library everywhere (#85955)

The usage of __builtin_unreachable after calls to quick_exit were distressing.
If a function is properly marked [[noreturn]] then __builtin_unreachable is not
necessary.

Looking into this further, we seem to have header only implementations for CPU
targets. The inline nature of these functions is curious; we're going to exit,
it doesn't matter if we need to pay the call of a function or not. If we just
make these functions have distinct TUs rather than be header only, we can clean
up the cmake rules for quick_exit which were different between CPU and GPU.

Remove darwin support for quick_exit. This isn't being tested, and we can bring
it back when necessary.
---
 libc/src/__support/OSUtil/CMakeLists.txt      | 27 +++++--------------
 .../__support/OSUtil/baremetal/CMakeLists.txt |  5 ++--
 .../{quick_exit.h => quick_exit.cpp}          | 13 ++++-----
 .../__support/OSUtil/darwin/CMakeLists.txt    |  1 -
 libc/src/__support/OSUtil/darwin/quick_exit.h | 26 ------------------
 libc/src/__support/OSUtil/gpu/CMakeLists.txt  |  1 -
 libc/src/__support/OSUtil/gpu/quick_exit.cpp  |  9 ++-----
 libc/src/__support/OSUtil/gpu/quick_exit.h    | 18 -------------
 .../src/__support/OSUtil/linux/CMakeLists.txt |  5 ++--
 .../linux/{quick_exit.h => quick_exit.cpp}    | 11 ++------
 libc/src/__support/OSUtil/quick_exit.h        | 15 +++--------
 libc/src/stdlib/CMakeLists.txt                |  5 ++--
 libc/src/stdlib/_Exit.cpp                     |  3 +--
 libc/src/stdlib/exit.cpp                      |  3 +--
 .../llvm-project-overlay/libc/BUILD.bazel     |  5 +---
 15 files changed, 32 insertions(+), 115 deletions(-)
 rename libc/src/__support/OSUtil/baremetal/{quick_exit.h => quick_exit.cpp} (61%)
 delete mode 100644 libc/src/__support/OSUtil/darwin/quick_exit.h
 delete mode 100644 libc/src/__support/OSUtil/gpu/quick_exit.h
 rename libc/src/__support/OSUtil/linux/{quick_exit.h => quick_exit.cpp} (78%)

diff --git a/libc/src/__support/OSUtil/CMakeLists.txt b/libc/src/__support/OSUtil/CMakeLists.txt
index ca3b3bf1263e..94d1042ccbb4 100644
--- a/libc/src/__support/OSUtil/CMakeLists.txt
+++ b/libc/src/__support/OSUtil/CMakeLists.txt
@@ -8,23 +8,10 @@ if(NOT TARGET ${target_os_util})
   return()
 endif()
 
-# The OSUtil is an object library in GPU mode.
-if(NOT LIBC_TARGET_OS_IS_GPU)
-  add_header_library(
-    osutil
-    HDRS
-      io.h
-      quick_exit.h
-      syscall.h
-    DEPENDS
-      ${target_os_util}
-  )
-else()
-  add_object_library(
-    osutil
-    ALIAS
-      ${target_os_util}
-    DEPENDS
-      ${target_os_util}
-  )
-endif()
+add_object_library(
+  osutil
+  ALIAS
+    ${target_os_util}
+  DEPENDS
+    ${target_os_util}
+)
diff --git a/libc/src/__support/OSUtil/baremetal/CMakeLists.txt b/libc/src/__support/OSUtil/baremetal/CMakeLists.txt
index 280ff87cf147..23da40326bbb 100644
--- a/libc/src/__support/OSUtil/baremetal/CMakeLists.txt
+++ b/libc/src/__support/OSUtil/baremetal/CMakeLists.txt
@@ -1,8 +1,9 @@
-add_header_library(
+add_object_library(
   baremetal_util
+  SRCS
+    quick_exit.cpp
   HDRS
     io.h
-    quick_exit.h
   DEPENDS
     libc.src.__support.common
     libc.src.__support.CPP.string_view
diff --git a/libc/src/__support/OSUtil/baremetal/quick_exit.h b/libc/src/__support/OSUtil/baremetal/quick_exit.cpp
similarity index 61%
rename from libc/src/__support/OSUtil/baremetal/quick_exit.h
rename to libc/src/__support/OSUtil/baremetal/quick_exit.cpp
index 74f9142e21b8..f0971d4db352 100644
--- a/libc/src/__support/OSUtil/baremetal/quick_exit.h
+++ b/libc/src/__support/OSUtil/baremetal/quick_exit.cpp
@@ -6,16 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_BAREMETAL_QUICK_EXIT_H
-#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_BAREMETAL_QUICK_EXIT_H
-
-namespace LIBC_NAMESPACE {
+#include "src/__support/OSUtil/quick_exit.h"
 
 // This is intended to be provided by the vendor.
-extern "C" void __llvm_libc_quick_exit(int status);
+[[noreturn]] extern "C" void __llvm_libc_quick_exit(int status);
 
-void quick_exit(int status) { __llvm_libc_quick_exit(status); }
+namespace LIBC_NAMESPACE {
 
-} // namespace LIBC_NAMESPACE
+[[noreturn]] void quick_exit(int status) { __llvm_libc_quick_exit(status); }
 
-#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_BAREMETAL_QUICK_EXIT_H
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/__support/OSUtil/darwin/CMakeLists.txt b/libc/src/__support/OSUtil/darwin/CMakeLists.txt
index a3e6b93c8e99..4241bb37684f 100644
--- a/libc/src/__support/OSUtil/darwin/CMakeLists.txt
+++ b/libc/src/__support/OSUtil/darwin/CMakeLists.txt
@@ -8,7 +8,6 @@ add_header_library(
   darwin_util
   HDRS
     io.h
-    quick_exit.h
     syscall.h
   DEPENDS
     .${LIBC_TARGET_ARCHITECTURE}.darwin_util
diff --git a/libc/src/__support/OSUtil/darwin/quick_exit.h b/libc/src/__support/OSUtil/darwin/quick_exit.h
deleted file mode 100644
index 71647f50def5..000000000000
--- a/libc/src/__support/OSUtil/darwin/quick_exit.h
+++ /dev/null
@@ -1,26 +0,0 @@
-//===--------- Darwin implementation of a quick exit function ---*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_DARWIN_QUICK_EXIT_H
-#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_DARWIN_QUICK_EXIT_H
-
-#include "syscall.h" // For internal syscall function.
-
-#include "src/__support/common.h"
-
-namespace LIBC_NAMESPACE {
-
-LIBC_INLINE void quick_exit(int status) {
-  for (;;) {
-    LIBC_NAMESPACE::syscall_impl<long>(1 /* SYS_exit */, status);
-  }
-}
-
-} // namespace LIBC_NAMESPACE
-
-#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_DARWIN_QUICK_EXIT_H
diff --git a/libc/src/__support/OSUtil/gpu/CMakeLists.txt b/libc/src/__support/OSUtil/gpu/CMakeLists.txt
index 8e892a7000c9..0c89f9223678 100644
--- a/libc/src/__support/OSUtil/gpu/CMakeLists.txt
+++ b/libc/src/__support/OSUtil/gpu/CMakeLists.txt
@@ -4,7 +4,6 @@ add_object_library(
     quick_exit.cpp
     io.cpp
   HDRS
-    quick_exit.h
     io.h
   DEPENDS
     libc.src.__support.common
diff --git a/libc/src/__support/OSUtil/gpu/quick_exit.cpp b/libc/src/__support/OSUtil/gpu/quick_exit.cpp
index 1a03be0ace67..af4795905e78 100644
--- a/libc/src/__support/OSUtil/gpu/quick_exit.cpp
+++ b/libc/src/__support/OSUtil/gpu/quick_exit.cpp
@@ -6,17 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_GPU_QUICK_EXIT_H
-#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_GPU_QUICK_EXIT_H
-
-#include "quick_exit.h"
+#include "src/__support/OSUtil/quick_exit.h"
 
 #include "src/__support/RPC/rpc_client.h"
 #include "src/__support/macros/properties/architectures.h"
 
 namespace LIBC_NAMESPACE {
 
-void quick_exit(int status) {
+[[noreturn]] void quick_exit(int status) {
   // We want to first make sure the server is listening before we exit.
   rpc::Client::Port port = rpc::client.open<RPC_EXIT>();
   port.send_and_recv([](rpc::Buffer *) {}, [](rpc::Buffer *) {});
@@ -29,5 +26,3 @@ void quick_exit(int status) {
 }
 
 } // namespace LIBC_NAMESPACE
-
-#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_GPU_QUICK_EXIT_H
diff --git a/libc/src/__support/OSUtil/gpu/quick_exit.h b/libc/src/__support/OSUtil/gpu/quick_exit.h
deleted file mode 100644
index b51385defbc0..000000000000
--- a/libc/src/__support/OSUtil/gpu/quick_exit.h
+++ /dev/null
@@ -1,18 +0,0 @@
-//===---------- GPU implementation of a quick exit function -----*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_GPU_QUICK_EXIT_H
-#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_GPU_QUICK_EXIT_H
-
-namespace LIBC_NAMESPACE {
-
-void quick_exit(int status);
-
-} // namespace LIBC_NAMESPACE
-
-#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_GPU_QUICK_EXIT_H
diff --git a/libc/src/__support/OSUtil/linux/CMakeLists.txt b/libc/src/__support/OSUtil/linux/CMakeLists.txt
index c27f9be74648..239d11570492 100644
--- a/libc/src/__support/OSUtil/linux/CMakeLists.txt
+++ b/libc/src/__support/OSUtil/linux/CMakeLists.txt
@@ -4,11 +4,12 @@ endif()
 
 add_subdirectory(${LIBC_TARGET_ARCHITECTURE})
 
-add_header_library(
+add_object_library(
   linux_util
+  SRCS
+    quick_exit.cpp
   HDRS
     io.h
-    quick_exit.h
     syscall.h
   DEPENDS
     .${LIBC_TARGET_ARCHITECTURE}.linux_${LIBC_TARGET_ARCHITECTURE}_util
diff --git a/libc/src/__support/OSUtil/linux/quick_exit.h b/libc/src/__support/OSUtil/linux/quick_exit.cpp
similarity index 78%
rename from libc/src/__support/OSUtil/linux/quick_exit.h
rename to libc/src/__support/OSUtil/linux/quick_exit.cpp
index 432395584d84..51b3231d389f 100644
--- a/libc/src/__support/OSUtil/linux/quick_exit.h
+++ b/libc/src/__support/OSUtil/linux/quick_exit.cpp
@@ -6,13 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_QUICK_EXIT_H
-#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_QUICK_EXIT_H
-
-#include "syscall.h" // For internal syscall function.
-
 #include "src/__support/common.h"
-
+#include "syscall.h"     // For internal syscall function.
 #include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE {
@@ -22,7 +17,7 @@ namespace LIBC_NAMESPACE {
 #ifdef LIBC_TARGET_ARCH_IS_X86
 __attribute__((no_stack_protector))
 #endif
-LIBC_INLINE void
+__attribute__((noreturn)) void
 quick_exit(int status) {
   for (;;) {
     LIBC_NAMESPACE::syscall_impl<long>(SYS_exit_group, status);
@@ -31,5 +26,3 @@ quick_exit(int status) {
 }
 
 } // namespace LIBC_NAMESPACE
-
-#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_QUICK_EXIT_H
diff --git a/libc/src/__support/OSUtil/quick_exit.h b/libc/src/__support/OSUtil/quick_exit.h
index 6c59c1afcda2..e445917059c3 100644
--- a/libc/src/__support/OSUtil/quick_exit.h
+++ b/libc/src/__support/OSUtil/quick_exit.h
@@ -9,17 +9,10 @@
 #ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_QUICK_EXIT_H
 #define LLVM_LIBC_SRC___SUPPORT_OSUTIL_QUICK_EXIT_H
 
-#include "src/__support/macros/properties/architectures.h"
+namespace LIBC_NAMESPACE {
 
-#if defined(LIBC_TARGET_ARCH_IS_GPU)
-#include "gpu/quick_exit.h"
-#elif defined(__APPLE__)
-#include "darwin/quick_exit.h"
-#elif defined(__linux__)
-#include "linux/quick_exit.h"
-#elif defined(__ELF__)
-// TODO: Ideally we would have LIBC_TARGET_OS_IS_BAREMETAL.
-#include "baremetal/quick_exit.h"
-#endif
+[[noreturn]] void quick_exit(int status);
+
+}
 
 #endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_QUICK_EXIT_H
diff --git a/libc/src/stdlib/CMakeLists.txt b/libc/src/stdlib/CMakeLists.txt
index a2b16e73f353..e526ba040bef 100644
--- a/libc/src/stdlib/CMakeLists.txt
+++ b/libc/src/stdlib/CMakeLists.txt
@@ -414,10 +414,11 @@ add_entrypoint_object(
   CXX_STANDARD
     20 # For constinit of the atexit callback list.
   DEPENDS
-    libc.src.__support.fixedvector
+    libc.src.__support.CPP.new
+    libc.src.__support.OSUtil.osutil
     libc.src.__support.blockstore
+    libc.src.__support.fixedvector
     libc.src.__support.threads.mutex
-    libc.src.__support.CPP.new
 )
 
 add_entrypoint_object(
diff --git a/libc/src/stdlib/_Exit.cpp b/libc/src/stdlib/_Exit.cpp
index 85684d1e9087..233af2097392 100644
--- a/libc/src/stdlib/_Exit.cpp
+++ b/libc/src/stdlib/_Exit.cpp
@@ -13,9 +13,8 @@
 
 namespace LIBC_NAMESPACE {
 
-LLVM_LIBC_FUNCTION(void, _Exit, (int status)) {
+[[noreturn]] LLVM_LIBC_FUNCTION(void, _Exit, (int status)) {
   quick_exit(status);
-  __builtin_unreachable();
 }
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/stdlib/exit.cpp b/libc/src/stdlib/exit.cpp
index e754b34e4698..ba87bffaeb54 100644
--- a/libc/src/stdlib/exit.cpp
+++ b/libc/src/stdlib/exit.cpp
@@ -14,10 +14,9 @@ extern "C" void __cxa_finalize(void *);
 
 namespace LIBC_NAMESPACE {
 
-LLVM_LIBC_FUNCTION(void, exit, (int status)) {
+[[noreturn]] LLVM_LIBC_FUNCTION(void, exit, (int status)) {
   __cxa_finalize(nullptr);
   quick_exit(status);
-  __builtin_unreachable();
 }
 
 } // namespace LIBC_NAMESPACE
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 0557c3aec0d8..57dc75a3a48e 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -1002,10 +1002,7 @@ libc_support_library(
 libc_support_library(
     name = "__support_osutil_quick_exit",
     hdrs = ["src/__support/OSUtil/quick_exit.h"],
-    textual_hdrs = [
-        "src/__support/OSUtil/linux/quick_exit.h",
-        #TODO: add support for GPU quick_exit (isn't just in a header.)
-    ],
+    srcs = ["src/__support/OSUtil/linux/quick_exit.cpp"],
     deps = [
         ":__support_osutil_syscall",
     ],
-- 
GitLab


From 87519a2830eab4b634a5816f53fe773929eed531 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Tue, 26 Mar 2024 23:28:04 +0800
Subject: [PATCH 290/404] [RISCV] Combine (mul (zext, zext)) -> (zext (mul
 (zext, zext))) (#86465)

Building on #86248, we can also narrow the width of a mul of zexts.

This is specifically legal because on RVV we always extend to the next
power of 2 width, and multiplying two N bit integers produces a maximum
value of 2\*N bits.
So as long as we keep an inner zext of 2\*N, we will have enough space
for the multiply and won't overflow.

Alive2 proof: https://alive2.llvm.org/ce/z/XteYyb
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |   4 +
 .../CodeGen/RISCV/rvv/fixed-vectors-vwmulu.ll |   8 +-
 llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll   | 128 ++++++++----------
 3 files changed, 64 insertions(+), 76 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index e6814c5f71a0..ca78648c6aa9 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -12935,6 +12935,7 @@ static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG,
 
 // add (zext, zext) -> zext (add (zext, zext))
 // sub (zext, zext) -> sext (sub (zext, zext))
+// mul (zext, zext) -> zext (mul (zext, zext))
 //
 // where the sum of the extend widths match, and the the range of the bin op
 // fits inside the width of the narrower bin op. (For profitability on rvv, we
@@ -13380,6 +13381,9 @@ static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG) {
     return DAG.getNode(AddSubOpc, DL, VT, N0, MulVal);
   }
 
+  if (SDValue V = combineBinOpOfZExt(N, DAG))
+    return V;
+
   return SDValue();
 }
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmulu.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmulu.ll
index bff7ef86c289..b97c9654ad3c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmulu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmulu.ll
@@ -391,12 +391,12 @@ define <32 x i64> @vwmulu_v32i64(ptr %x, ptr %y) {
 define <2 x i32> @vwmulu_v2i32_v2i8(ptr %x, ptr %y) {
 ; CHECK-LABEL: vwmulu_v2i32_v2i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    vle8.v v9, (a1)
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vzext.vf2 v11, v9
-; CHECK-NEXT:    vwmulu.vv v8, v10, v11
+; CHECK-NEXT:    vwmulu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v10
 ; CHECK-NEXT:    ret
   %a = load <2 x i8>, ptr %x
   %b = load <2 x i8>, ptr %y
diff --git a/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll
index 539a4bdb27ad..28fc53f37ba1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll
@@ -355,10 +355,10 @@ define <vscale x 1 x i64> @vwmul_vv_nxv1i64_nxv1i16(<vscale x 1 x i16> %va, <vsc
 define <vscale x 1 x i64> @vwmulu_vv_nxv1i64_nxv1i16(<vscale x 1 x i16> %va, <vscale x 1 x i16> %vb) {
 ; CHECK-LABEL: vwmulu_vv_nxv1i64_nxv1i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vzext.vf2 v11, v9
-; CHECK-NEXT:    vwmulu.vv v8, v10, v11
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vwmulu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v10
 ; CHECK-NEXT:    ret
   %vc = zext <vscale x 1 x i16> %va to <vscale x 1 x i64>
   %vd = zext <vscale x 1 x i16> %vb to <vscale x 1 x i64>
@@ -402,11 +402,9 @@ define <vscale x 1 x i64> @vwmulu_vx_nxv1i64_nxv1i16(<vscale x 1 x i16> %va, i16
 ; CHECK-LABEL: vwmulu_vx_nxv1i64_nxv1i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vzext.vf2 v11, v9
-; CHECK-NEXT:    vwmulu.vv v8, v10, v11
+; CHECK-NEXT:    vwmulu.vx v9, v8, a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v9
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i16> undef, i16 %b, i16 0
   %splat = shufflevector <vscale x 1 x i16> %head, <vscale x 1 x i16> undef, <vscale x 1 x i32> zeroinitializer
@@ -451,10 +449,10 @@ define <vscale x 2 x i64> @vwmul_vv_nxv2i64_nxv2i16(<vscale x 2 x i16> %va, <vsc
 define <vscale x 2 x i64> @vwmulu_vv_nxv2i64_nxv2i16(<vscale x 2 x i16> %va, <vscale x 2 x i16> %vb) {
 ; CHECK-LABEL: vwmulu_vv_nxv2i64_nxv2i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vzext.vf2 v11, v9
-; CHECK-NEXT:    vwmulu.vv v8, v10, v11
+; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vwmulu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v10
 ; CHECK-NEXT:    ret
   %vc = zext <vscale x 2 x i16> %va to <vscale x 2 x i64>
   %vd = zext <vscale x 2 x i16> %vb to <vscale x 2 x i64>
@@ -498,11 +496,9 @@ define <vscale x 2 x i64> @vwmulu_vx_nxv2i64_nxv2i16(<vscale x 2 x i16> %va, i16
 ; CHECK-LABEL: vwmulu_vx_nxv2i64_nxv2i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vzext.vf2 v10, v8
-; CHECK-NEXT:    vzext.vf2 v11, v9
-; CHECK-NEXT:    vwmulu.vv v8, v10, v11
+; CHECK-NEXT:    vwmulu.vx v10, v8, a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v10
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i16> undef, i16 %b, i16 0
   %splat = shufflevector <vscale x 2 x i16> %head, <vscale x 2 x i16> undef, <vscale x 2 x i32> zeroinitializer
@@ -547,10 +543,10 @@ define <vscale x 4 x i64> @vwmul_vv_nxv4i64_nxv4i16(<vscale x 4 x i16> %va, <vsc
 define <vscale x 4 x i64> @vwmulu_vv_nxv4i64_nxv4i16(<vscale x 4 x i16> %va, <vscale x 4 x i16> %vb) {
 ; CHECK-LABEL: vwmulu_vv_nxv4i64_nxv4i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v12, v8
-; CHECK-NEXT:    vzext.vf2 v14, v9
-; CHECK-NEXT:    vwmulu.vv v8, v12, v14
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vv v12, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v12
 ; CHECK-NEXT:    ret
   %vc = zext <vscale x 4 x i16> %va to <vscale x 4 x i64>
   %vd = zext <vscale x 4 x i16> %vb to <vscale x 4 x i64>
@@ -594,11 +590,9 @@ define <vscale x 4 x i64> @vwmulu_vx_nxv4i64_nxv4i16(<vscale x 4 x i16> %va, i16
 ; CHECK-LABEL: vwmulu_vx_nxv4i64_nxv4i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vzext.vf2 v12, v8
-; CHECK-NEXT:    vzext.vf2 v14, v9
-; CHECK-NEXT:    vwmulu.vv v8, v12, v14
+; CHECK-NEXT:    vwmulu.vx v12, v8, a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v12
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i16> undef, i16 %b, i16 0
   %splat = shufflevector <vscale x 4 x i16> %head, <vscale x 4 x i16> undef, <vscale x 4 x i32> zeroinitializer
@@ -643,10 +637,10 @@ define <vscale x 8 x i64> @vwmul_vv_nxv8i64_nxv8i16(<vscale x 8 x i16> %va, <vsc
 define <vscale x 8 x i64> @vwmulu_vv_nxv8i64_nxv8i16(<vscale x 8 x i16> %va, <vscale x 8 x i16> %vb) {
 ; CHECK-LABEL: vwmulu_vv_nxv8i64_nxv8i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vzext.vf2 v16, v8
-; CHECK-NEXT:    vzext.vf2 v20, v10
-; CHECK-NEXT:    vwmulu.vv v8, v16, v20
+; CHECK-NEXT:    vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vwmulu.vv v16, v8, v10
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v16
 ; CHECK-NEXT:    ret
   %vc = zext <vscale x 8 x i16> %va to <vscale x 8 x i64>
   %vd = zext <vscale x 8 x i16> %vb to <vscale x 8 x i64>
@@ -690,11 +684,9 @@ define <vscale x 8 x i64> @vwmulu_vx_nxv8i64_nxv8i16(<vscale x 8 x i16> %va, i16
 ; CHECK-LABEL: vwmulu_vx_nxv8i64_nxv8i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.v.x v10, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vzext.vf2 v16, v8
-; CHECK-NEXT:    vzext.vf2 v20, v10
-; CHECK-NEXT:    vwmulu.vv v8, v16, v20
+; CHECK-NEXT:    vwmulu.vx v16, v8, a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i16> undef, i16 %b, i16 0
   %splat = shufflevector <vscale x 8 x i16> %head, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
@@ -739,10 +731,10 @@ define <vscale x 1 x i64> @vwmul_vv_nxv1i64_nxv1i8(<vscale x 1 x i8> %va, <vscal
 define <vscale x 1 x i64> @vwmulu_vv_nxv1i64_nxv1i8(<vscale x 1 x i8> %va, <vscale x 1 x i8> %vb) {
 ; CHECK-LABEL: vwmulu_vv_nxv1i64_nxv1i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vzext.vf4 v10, v8
-; CHECK-NEXT:    vzext.vf4 v11, v9
-; CHECK-NEXT:    vwmulu.vv v8, v10, v11
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf8, ta, ma
+; CHECK-NEXT:    vwmulu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v10
 ; CHECK-NEXT:    ret
   %vc = zext <vscale x 1 x i8> %va to <vscale x 1 x i64>
   %vd = zext <vscale x 1 x i8> %vb to <vscale x 1 x i64>
@@ -786,11 +778,9 @@ define <vscale x 1 x i64> @vwmulu_vx_nxv1i64_nxv1i8(<vscale x 1 x i8> %va, i8 %b
 ; CHECK-LABEL: vwmulu_vx_nxv1i64_nxv1i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf8, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vzext.vf4 v10, v8
-; CHECK-NEXT:    vzext.vf4 v11, v9
-; CHECK-NEXT:    vwmulu.vv v8, v10, v11
+; CHECK-NEXT:    vwmulu.vx v9, v8, a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v9
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i8> undef, i8 %b, i8 0
   %splat = shufflevector <vscale x 1 x i8> %head, <vscale x 1 x i8> undef, <vscale x 1 x i32> zeroinitializer
@@ -835,10 +825,10 @@ define <vscale x 2 x i64> @vwmul_vv_nxv2i64_nxv2i8(<vscale x 2 x i8> %va, <vscal
 define <vscale x 2 x i64> @vwmulu_vv_nxv2i64_nxv2i8(<vscale x 2 x i8> %va, <vscale x 2 x i8> %vb) {
 ; CHECK-LABEL: vwmulu_vv_nxv2i64_nxv2i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vzext.vf4 v10, v8
-; CHECK-NEXT:    vzext.vf4 v11, v9
-; CHECK-NEXT:    vwmulu.vv v8, v10, v11
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf4, ta, ma
+; CHECK-NEXT:    vwmulu.vv v10, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v10
 ; CHECK-NEXT:    ret
   %vc = zext <vscale x 2 x i8> %va to <vscale x 2 x i64>
   %vd = zext <vscale x 2 x i8> %vb to <vscale x 2 x i64>
@@ -882,11 +872,9 @@ define <vscale x 2 x i64> @vwmulu_vx_nxv2i64_nxv2i8(<vscale x 2 x i8> %va, i8 %b
 ; CHECK-LABEL: vwmulu_vx_nxv2i64_nxv2i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vzext.vf4 v10, v8
-; CHECK-NEXT:    vzext.vf4 v11, v9
-; CHECK-NEXT:    vwmulu.vv v8, v10, v11
+; CHECK-NEXT:    vwmulu.vx v10, v8, a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v10
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i8> undef, i8 %b, i8 0
   %splat = shufflevector <vscale x 2 x i8> %head, <vscale x 2 x i8> undef, <vscale x 2 x i32> zeroinitializer
@@ -931,10 +919,10 @@ define <vscale x 4 x i64> @vwmul_vv_nxv4i64_nxv4i8(<vscale x 4 x i8> %va, <vscal
 define <vscale x 4 x i64> @vwmulu_vv_nxv4i64_nxv4i8(<vscale x 4 x i8> %va, <vscale x 4 x i8> %vb) {
 ; CHECK-LABEL: vwmulu_vv_nxv4i64_nxv4i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vzext.vf4 v12, v8
-; CHECK-NEXT:    vzext.vf4 v14, v9
-; CHECK-NEXT:    vwmulu.vv v8, v12, v14
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vwmulu.vv v12, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v12
 ; CHECK-NEXT:    ret
   %vc = zext <vscale x 4 x i8> %va to <vscale x 4 x i64>
   %vd = zext <vscale x 4 x i8> %vb to <vscale x 4 x i64>
@@ -978,11 +966,9 @@ define <vscale x 4 x i64> @vwmulu_vx_nxv4i64_nxv4i8(<vscale x 4 x i8> %va, i8 %b
 ; CHECK-LABEL: vwmulu_vx_nxv4i64_nxv4i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vzext.vf4 v12, v8
-; CHECK-NEXT:    vzext.vf4 v14, v9
-; CHECK-NEXT:    vwmulu.vv v8, v12, v14
+; CHECK-NEXT:    vwmulu.vx v12, v8, a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v12
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 4 x i8> undef, i8 %b, i8 0
   %splat = shufflevector <vscale x 4 x i8> %head, <vscale x 4 x i8> undef, <vscale x 4 x i32> zeroinitializer
@@ -1027,10 +1013,10 @@ define <vscale x 8 x i64> @vwmul_vv_nxv8i64_nxv8i8(<vscale x 8 x i8> %va, <vscal
 define <vscale x 8 x i64> @vwmulu_vv_nxv8i64_nxv8i8(<vscale x 8 x i8> %va, <vscale x 8 x i8> %vb) {
 ; CHECK-LABEL: vwmulu_vv_nxv8i64_nxv8i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vzext.vf4 v16, v8
-; CHECK-NEXT:    vzext.vf4 v20, v9
-; CHECK-NEXT:    vwmulu.vv v8, v16, v20
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vv v16, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v16
 ; CHECK-NEXT:    ret
   %vc = zext <vscale x 8 x i8> %va to <vscale x 8 x i64>
   %vd = zext <vscale x 8 x i8> %vb to <vscale x 8 x i64>
@@ -1074,11 +1060,9 @@ define <vscale x 8 x i64> @vwmulu_vx_nxv8i64_nxv8i8(<vscale x 8 x i8> %va, i8 %b
 ; CHECK-LABEL: vwmulu_vx_nxv8i64_nxv8i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vmv.v.x v9, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vzext.vf4 v16, v8
-; CHECK-NEXT:    vzext.vf4 v20, v9
-; CHECK-NEXT:    vwmulu.vv v8, v16, v20
+; CHECK-NEXT:    vwmulu.vx v16, v8, a0
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v16
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 8 x i8> undef, i8 %b, i8 0
   %splat = shufflevector <vscale x 8 x i8> %head, <vscale x 8 x i8> undef, <vscale x 8 x i32> zeroinitializer
-- 
GitLab


From 4998587e6f5f66d464ac22ad4c11fe9afd2d56ab Mon Sep 17 00:00:00 2001
From: Daniel Chen <cdchen@ca.ibm.com>
Date: Tue, 26 Mar 2024 11:29:24 -0400
Subject: [PATCH 291/404] [Flang] Support for passing procedure pointer,
 reference to a function that returns a procedure pointer to structure
 constructor. (#86533)

This PR fixes `not yet implemented: procedure pointer component in
structure constructor` as shown in the following test case.

```
  MODULE M
    TYPE :: DT
      PROCEDURE(Fun), POINTER, NOPASS :: pp1
    END TYPE

    CONTAINS

    INTEGER FUNCTION Fun(Arg)
    INTEGER :: Arg
      Fun = Arg
    END FUNCTION

  END MODULE

  PROGRAM MAIN
  USE M
  IMPLICIT NONE
  TYPE (DT) :: v2
  PROCEDURE(FUN), POINTER :: pp2
  v2 = DT(pp2)
  v2 = DT(bar())
  CONTAINS
    FUNCTION BAR() RESULT(res)
      PROCEDURE(FUN), POINTER :: res
    END
  END
  ```
---
 flang/lib/Lower/Bridge.cpp                    |  3 +-
 flang/lib/Lower/ConvertExprToHLFIR.cpp        | 21 ++++++++++++--
 ...ointer-component-structure-constructor.f90 | 29 +++++++++++++++++--
 3 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 48830dc55578..91b898eb513e 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -3490,7 +3490,8 @@ private:
       if (Fortran::evaluate::UnwrapExpr<Fortran::evaluate::NullPointer>(
               assign.rhs)) {
         // rhs is null(). rhs being null(pptr) is handled in genNull.
-        auto boxTy{Fortran::lower::getUntypedBoxProcType(&getMLIRContext())};
+        auto boxTy{
+            Fortran::lower::getUntypedBoxProcType(builder->getContext())};
         hlfir::Entity rhs(
             fir::factory::createNullBoxProc(*builder, loc, boxTy));
         builder->createStoreWithConvert(loc, rhs, lhs);
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index fe5ce4b17b25..6e57b31d022b 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -130,7 +130,8 @@ public:
     // shape is deferred and should not be loaded now to preserve
     // pointer/allocatable aspects.
     if (componentSym.Rank() == 0 ||
-        Fortran::semantics::IsAllocatableOrObjectPointer(&componentSym))
+        Fortran::semantics::IsAllocatableOrObjectPointer(&componentSym) ||
+        Fortran::semantics::IsProcedurePointer(&componentSym))
       return mlir::Value{};
 
     fir::FirOpBuilder &builder = getBuilder();
@@ -1767,8 +1768,22 @@ private:
 
       if (attrs && bitEnumContainsAny(attrs.getFlags(),
                                       fir::FortranVariableFlagsEnum::pointer)) {
-        if (Fortran::semantics::IsProcedure(sym))
-          TODO(loc, "procedure pointer component in structure constructor");
+        if (Fortran::semantics::IsProcedure(sym)) {
+          // Procedure pointer components.
+          if (Fortran::evaluate::UnwrapExpr<Fortran::evaluate::NullPointer>(
+                  expr)) {
+            auto boxTy{
+                Fortran::lower::getUntypedBoxProcType(builder.getContext())};
+            hlfir::Entity rhs(
+                fir::factory::createNullBoxProc(builder, loc, boxTy));
+            builder.createStoreWithConvert(loc, rhs, lhs);
+            continue;
+          }
+          hlfir::Entity rhs(getBase(Fortran::lower::convertExprToAddress(
+              loc, converter, expr, symMap, stmtCtx)));
+          builder.createStoreWithConvert(loc, rhs, lhs);
+          continue;
+        }
         // Pointer component construction is just a copy of the box contents.
         fir::ExtendedValue lhsExv =
             hlfir::translateToExtendedValue(loc, builder, lhs);
diff --git a/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90 b/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90
index f41c832ee5ec..7b64634d10d4 100644
--- a/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90
+++ b/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90
@@ -1,8 +1,8 @@
 ! Test passing
 !  1. NULL(),
 !  2. procedure,
-!  3. procedure pointer, (pending)
-!  4. reference to a function that returns a procedure pointer (pending)
+!  3. procedure pointer,
+!  4. reference to a function that returns a procedure pointer.
 ! to a derived type structure constructor.
 ! RUN: bbc -emit-hlfir -o - %s | FileCheck %s
 
@@ -25,10 +25,33 @@
   IMPLICIT NONE
   TYPE (DT), PARAMETER :: v1 = DT(NULL())
   TYPE (DT) :: v2
+  PROCEDURE(FUN), POINTER :: pp2
   v2 = DT(fun)
+  v2 = DT(pp2)
+  v2 = DT(bar())
+  CONTAINS
+    FUNCTION BAR() RESULT(res)
+      PROCEDURE(FUN), POINTER :: res
+    END
   END
 
-! CDHECK-LABEL:  fir.global internal @_QFECv1 constant : !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}> {
+! CHECK-LABEL:  func.func @_QQmain() attributes {fir.bindc_name = "main"} {
+! CHECK:    %[[VAL_0:.*]] = fir.alloca !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>
+! CHECK:    %[[VAL_1:.*]] = fir.alloca !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>
+! CHECK:    %[[VAL_2:.*]] = fir.alloca !fir.boxproc<(!fir.ref<i32>) -> i32> {bindc_name = "pp2", uniq_name = "_QFEpp2"}
+! CHECK:    %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFEpp2"} : (!fir.ref<!fir.boxproc<(!fir.ref<i32>) -> i32>>) -> (!fir.ref<!fir.boxproc<(!fir.ref<i32>) -> i32>>, !fir.ref<!fir.boxproc<(!fir.ref<i32>) -> i32>>)
+! CHECK:    %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>>) -> (!fir.ref<!fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>>, !fir.ref<!fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>>)
+! CHECK:    %[[VAL_23:.*]] = hlfir.designate %[[VAL_17]]#0{"pp1"}   {fortran_attrs = #fir.var_attrs<pointer>} : (!fir.ref<!fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>>) -> !fir.ref<!fir.boxproc<(!fir.ref<i32>) -> i32>>
+! CHECK:    %[[VAL_24:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<!fir.boxproc<(!fir.ref<i32>) -> i32>>
+! CHECK:    fir.store %[[VAL_24]] to %[[VAL_23]] : !fir.ref<!fir.boxproc<(!fir.ref<i32>) -> i32>>
+! CHECK:    %[[VAL_25:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>>) -> (!fir.ref<!fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>>, !fir.ref<!fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>>)
+! CHECK:    %[[VAL_31:.*]] = hlfir.designate %[[VAL_25]]#0{"pp1"}   {fortran_attrs = #fir.var_attrs<pointer>} : (!fir.ref<!fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>>) -> !fir.ref<!fir.boxproc<(!fir.ref<i32>) -> i32>>
+! CHECK:    %[[VAL_32:.*]] = fir.call @_QFPbar() fastmath<contract> : () -> !fir.boxproc<(!fir.ref<i32>) -> i32>
+! CHECK:    fir.store %[[VAL_32]] to %[[VAL_31]] : !fir.ref<!fir.boxproc<(!fir.ref<i32>) -> i32>>
+! CHECK:    return
+! CHECK:  }
+
+! CHECK-LABEL:  fir.global internal @_QFECv1 constant : !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}> {
 ! CHECK:    %[[VAL_0:.*]] = fir.undefined !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>
 ! CHECK:    %[[VAL_1:.*]] = fir.field_index pp1, !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref<i32>) -> i32>}>
 ! CHECK:    %[[VAL_2:.*]] = fir.zero_bits (!fir.ref<i32>) -> i32
-- 
GitLab


From 6e261d9c37a49b19552b439712703968ea88ee2a Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Tue, 26 Mar 2024 08:46:21 -0700
Subject: [PATCH 292/404] [flang] Accept more unrecognized !DIR$ compiler
 directives (#85829)

When encountering an unparsable !DIR$ compiler directive line, accept it
as a whole source line and emit a warning that it is unrecognizable.

Fixes https://github.com/llvm/llvm-project/issues/59107,
https://github.com/llvm/llvm-project/issues/82212, and
https://github.com/llvm/llvm-project/issues/82654.
---
 flang/include/flang/Parser/dump-parse-tree.h  |  1 +
 .../include/flang/Parser/parse-tree-visitor.h | 12 ++++++++
 flang/include/flang/Parser/parse-tree.h       |  6 ++--
 flang/lib/Parser/Fortran-parsers.cpp          | 30 +++++++++++--------
 flang/lib/Parser/unparse.cpp                  |  4 +++
 flang/test/Parser/unrecognized-dir.f90        |  4 +++
 6 files changed, 42 insertions(+), 15 deletions(-)
 create mode 100644 flang/test/Parser/unrecognized-dir.f90

diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index b2c3d9290937..06c168a5de61 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -207,6 +207,7 @@ public:
   NODE(CompilerDirective, LoopCount)
   NODE(CompilerDirective, AssumeAligned)
   NODE(CompilerDirective, NameValue)
+  NODE(CompilerDirective, Unrecognized)
   NODE(parser, ComplexLiteralConstant)
   NODE(parser, ComplexPart)
   NODE(parser, ComponentArraySpec)
diff --git a/flang/include/flang/Parser/parse-tree-visitor.h b/flang/include/flang/Parser/parse-tree-visitor.h
index 79ea29f4b7f3..81d01dbdd65c 100644
--- a/flang/include/flang/Parser/parse-tree-visitor.h
+++ b/flang/include/flang/Parser/parse-tree-visitor.h
@@ -861,6 +861,18 @@ template <typename M> void Walk(CompilerDirective &x, M &mutator) {
   }
 }
 template <typename V>
+void Walk(const CompilerDirective::Unrecognized &x, V &visitor) {
+  if (visitor.Pre(x)) {
+    visitor.Post(x);
+  }
+}
+template <typename M>
+void Walk(CompilerDirective::Unrecognized &x, M &mutator) {
+  if (mutator.Pre(x)) {
+    mutator.Post(x);
+  }
+}
+template <typename V>
 void Walk(const OmpLinearClause::WithModifier &x, V &visitor) {
   if (visitor.Pre(x)) {
     Walk(x.modifier, visitor);
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index c96abfba491d..85e8121dd125 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3298,7 +3298,8 @@ struct StmtFunctionStmt {
 // Compiler directives
 // !DIR$ IGNORE_TKR [ [(tkrdmac...)] name ]...
 // !DIR$ LOOP COUNT (n1[, n2]...)
-// !DIR$ name...
+// !DIR$ name[=value] [, name[=value]]...    = can be :
+// !DIR$ <anything else>
 struct CompilerDirective {
   UNION_CLASS_BOILERPLATE(CompilerDirective);
   struct IgnoreTKR {
@@ -3316,9 +3317,10 @@ struct CompilerDirective {
     TUPLE_CLASS_BOILERPLATE(NameValue);
     std::tuple<Name, std::optional<std::uint64_t>> t;
   };
+  struct Unrecognized {};
   CharBlock source;
   std::variant<std::list<IgnoreTKR>, LoopCount, std::list<AssumeAligned>,
-      std::list<NameValue>>
+      std::list<NameValue>, Unrecognized>
       u;
 };
 
diff --git a/flang/lib/Parser/Fortran-parsers.cpp b/flang/lib/Parser/Fortran-parsers.cpp
index fc81a477897a..fd28eea0f947 100644
--- a/flang/lib/Parser/Fortran-parsers.cpp
+++ b/flang/lib/Parser/Fortran-parsers.cpp
@@ -1261,24 +1261,28 @@ TYPE_PARSER(construct<StatOrErrmsg>("STAT =" >> statVariable) ||
 // Directives, extensions, and deprecated statements
 // !DIR$ IGNORE_TKR [ [(tkrdmac...)] name ]...
 // !DIR$ LOOP COUNT (n1[, n2]...)
-// !DIR$ name...
+// !DIR$ name[=value] [, name[=value]]...
+// !DIR$ <anything else>
 constexpr auto ignore_tkr{
-    "DIR$ IGNORE_TKR" >> optionalList(construct<CompilerDirective::IgnoreTKR>(
-                             maybe(parenthesized(many(letter))), name))};
+    "IGNORE_TKR" >> optionalList(construct<CompilerDirective::IgnoreTKR>(
+                        maybe(parenthesized(many(letter))), name))};
 constexpr auto loopCount{
-    "DIR$ LOOP COUNT" >> construct<CompilerDirective::LoopCount>(
-                             parenthesized(nonemptyList(digitString64)))};
-constexpr auto assumeAligned{"DIR$ ASSUME_ALIGNED" >>
+    "LOOP COUNT" >> construct<CompilerDirective::LoopCount>(
+                        parenthesized(nonemptyList(digitString64)))};
+constexpr auto assumeAligned{"ASSUME_ALIGNED" >>
     optionalList(construct<CompilerDirective::AssumeAligned>(
         indirect(designator), ":"_tok >> digitString64))};
-TYPE_PARSER(beginDirective >>
-    sourced(construct<CompilerDirective>(ignore_tkr) ||
-        construct<CompilerDirective>(loopCount) ||
-        construct<CompilerDirective>(assumeAligned) ||
+TYPE_PARSER(beginDirective >> "DIR$ "_tok >>
+    sourced((construct<CompilerDirective>(ignore_tkr) ||
+                construct<CompilerDirective>(loopCount) ||
+                construct<CompilerDirective>(assumeAligned) ||
+                construct<CompilerDirective>(
+                    many(construct<CompilerDirective::NameValue>(
+                        name, maybe(("="_tok || ":"_tok) >> digitString64))))) /
+            endOfStmt ||
         construct<CompilerDirective>(
-            "DIR$" >> many(construct<CompilerDirective::NameValue>(name,
-                          maybe(("="_tok || ":"_tok) >> digitString64))))) /
-        endOfStmt)
+            SkipTo<'\n'>{} >> pure<CompilerDirective::Unrecognized>()) /
+            endOfStmt))
 
 TYPE_PARSER(extension<LanguageFeature::CrayPointer>(
     "nonstandard usage: based POINTER"_port_en_US,
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index baba4863f577..c06458833f07 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -1827,6 +1827,10 @@ public:
             [&](const std::list<CompilerDirective::NameValue> &names) {
               Walk("!DIR$ ", names, " ");
             },
+            [&](const CompilerDirective::Unrecognized &) {
+              Word("!DIR$ ");
+              Word(x.source.ToString());
+            },
         },
         x.u);
     Put('\n');
diff --git a/flang/test/Parser/unrecognized-dir.f90 b/flang/test/Parser/unrecognized-dir.f90
new file mode 100644
index 000000000000..ba6fff7562e2
--- /dev/null
+++ b/flang/test/Parser/unrecognized-dir.f90
@@ -0,0 +1,4 @@
+! RUN: %flang_fc1 -fsyntax-only %s 2>&1 | FileCheck %s
+!CHECK: warning: Compiler directive was ignored
+!DIR$ Not a recognized directive
+end
-- 
GitLab


From 6d579cd1d91cdde5adfa455bda7903e737e9d5cf Mon Sep 17 00:00:00 2001
From: Cyndy Ishida <cyndy_ishida@apple.com>
Date: Tue, 26 Mar 2024 11:55:37 -0400
Subject: [PATCH 293/404] [InstallAPI] Add missing license header to file, NFC

---
 clang/lib/InstallAPI/DylibVerifier.cpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/clang/lib/InstallAPI/DylibVerifier.cpp b/clang/lib/InstallAPI/DylibVerifier.cpp
index 94b8e9cd3233..ba25e4183a9b 100644
--- a/clang/lib/InstallAPI/DylibVerifier.cpp
+++ b/clang/lib/InstallAPI/DylibVerifier.cpp
@@ -1,3 +1,11 @@
+//===- DylibVerifier.cpp ----------------------------------------*- C++--*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
 #include "clang/InstallAPI/DylibVerifier.h"
 #include "clang/InstallAPI/FrontendRecords.h"
 #include "clang/InstallAPI/InstallAPIDiagnostic.h"
-- 
GitLab


From f50f0caf87d762ca1c4a0122bd29d2fc9c667113 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Tue, 26 Mar 2024 08:59:22 -0700
Subject: [PATCH 294/404] [flang] Fix crash in name resolution (#85835)

ConvertToObjectEntity() returns true for use- and host-associated object
symbols, too. Ensure in this case that the symbol really is a
non-associated object.

Fixes https://github.com/llvm/llvm-project/issues/85776.
---
 flang/lib/Semantics/resolve-names.cpp | 18 +++++++++---------
 flang/test/Semantics/resolve61.f90    |  9 +++++++++
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index f89323f3e54a..73450c49d5fe 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -4646,23 +4646,23 @@ bool DeclarationVisitor::Pre(const parser::OldParameterStmt &x) {
 bool DeclarationVisitor::Pre(const parser::NamedConstantDef &x) {
   auto &name{std::get<parser::NamedConstant>(x.t).v};
   auto &symbol{HandleAttributeStmt(Attr::PARAMETER, name)};
-  if (!ConvertToObjectEntity(symbol) ||
-      symbol.test(Symbol::Flag::CrayPointer) ||
+  ConvertToObjectEntity(symbol);
+  auto *details{symbol.detailsIf<ObjectEntityDetails>()};
+  if (!details || symbol.test(Symbol::Flag::CrayPointer) ||
       symbol.test(Symbol::Flag::CrayPointee)) {
     SayWithDecl(
         name, symbol, "PARAMETER attribute not allowed on '%s'"_err_en_US);
     return false;
   }
   const auto &expr{std::get<parser::ConstantExpr>(x.t)};
-  auto &details{symbol.get<ObjectEntityDetails>()};
-  if (details.init() || symbol.test(Symbol::Flag::InDataStmt)) {
+  if (details->init() || symbol.test(Symbol::Flag::InDataStmt)) {
     Say(name, "Named constant '%s' already has a value"_err_en_US);
   }
   if (inOldStyleParameterStmt_) {
     // non-standard extension PARAMETER statement (no parentheses)
     Walk(expr);
     auto folded{EvaluateExpr(expr)};
-    if (details.type()) {
+    if (details->type()) {
       SayWithDecl(name, symbol,
           "Alternative style PARAMETER '%s' must not already have an explicit type"_err_en_US);
     } else if (folded) {
@@ -4674,9 +4674,9 @@ bool DeclarationVisitor::Pre(const parser::NamedConstantDef &x) {
           } else if (auto shape{ToArraySpec(
                          GetFoldingContext(), evaluate::GetShape(*folded))}) {
             // The type of the named constant is assumed from the expression.
-            details.set_type(*type);
-            details.set_init(std::move(*folded));
-            details.set_shape(std::move(*shape));
+            details->set_type(*type);
+            details->set_init(std::move(*folded));
+            details->set_shape(std::move(*shape));
           } else {
             Say(at, "The expression must have constant shape"_err_en_US);
           }
@@ -4693,7 +4693,7 @@ bool DeclarationVisitor::Pre(const parser::NamedConstantDef &x) {
     Walk(expr);
     if (auto converted{EvaluateNonPointerInitializer(
             symbol, expr, expr.thing.value().source)}) {
-      details.set_init(std::move(*converted));
+      details->set_init(std::move(*converted));
     }
   }
   return false;
diff --git a/flang/test/Semantics/resolve61.f90 b/flang/test/Semantics/resolve61.f90
index 32bf9091a856..2a1f584ffaf0 100644
--- a/flang/test/Semantics/resolve61.f90
+++ b/flang/test/Semantics/resolve61.f90
@@ -126,3 +126,12 @@ subroutine p13
     pointer(ip, x) ! ok, local declaration
   end
 end
+
+subroutine p14
+  real :: r
+  block
+    asynchronous :: r
+    !ERROR: PARAMETER attribute not allowed on 'r'
+    parameter (r = 1.0)
+  end block
+end
-- 
GitLab


From 2f63718f8567413a1c596bda803663eb58d6da5a Mon Sep 17 00:00:00 2001
From: Jason Molenda <jmolenda@apple.com>
Date: Tue, 26 Mar 2024 09:07:15 -0700
Subject: [PATCH 295/404] [lldb] Don't clear a Module's UnwindTable when adding
 a SymbolFile (#86603)

Fixing a crash in lldb when `symbols.auto-download` setting is enabled.
When doing a backtrace, this feature has lldb search for a SymbolFile
for stack frames when we are backtracing, and add them either
synchoronously or asynchronously, depending on the specific setting
used.

Module::SetSymbolFileFileSpec clears the Module's UnwindTable, once we
find a new SymbolFile. We may be adding a source of unwind information
that we did not have when lldb was working only with the executable
binary.

What happens in practice is that we're using a reference to the Module's
UnwindTable, and then the other thread getting the SymbolFile clears it
and now the first thread is referring to freed memory and we can crash.
When built with address sanitizer, it crashes much more reliably.

Given that unwind information used for exception handling -- eh_frame,
compact unwind -- is present in executable binaries, the only thing
we're likely to *add* would be DWARF's `debug_frame` if that was also
available. The actual value of re-creating the UnwindTable when we have
added a SymbolFile is not large.

I also tried fixing this by changing the Module to have a shared_ptr to
the UnwindTable, so we could have two different UnwindTable's in use
simultaneously for a brief period. This would be fine TODAY, but it
introduces a very subtle bug that someone will have a heck of a time
figuring out in the future.

In the end, I believe the safest approach is to sacrifice the possible
marginal gain of reconstructing the UnwindTable once a SymbolFile has
been added, to sidestep this whole problem area.

Also, in `Module::GetUnwindTable()`, call `DownloadSymbolFileAsync`
before we create the UnwindTable for the first time, in case the symbol
file is fetched synchronously, we will have it for that possible
marginal gain.
---
 lldb/source/Core/Module.cpp | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/lldb/source/Core/Module.cpp b/lldb/source/Core/Module.cpp
index 8ffa35518b3c..a520523a9652 100644
--- a/lldb/source/Core/Module.cpp
+++ b/lldb/source/Core/Module.cpp
@@ -1239,9 +1239,9 @@ void Module::SectionFileAddressesChanged() {
 
 UnwindTable &Module::GetUnwindTable() {
   if (!m_unwind_table) {
-    m_unwind_table.emplace(*this);
     if (!m_symfile_spec)
       SymbolLocator::DownloadSymbolFileAsync(GetUUID());
+    m_unwind_table.emplace(*this);
   }
   return *m_unwind_table;
 }
@@ -1359,15 +1359,10 @@ void Module::SetSymbolFileFileSpec(const FileSpec &file) {
         // one
         obj_file->ClearSymtab();
 
-        // Clear the unwind table too, as that may also be affected by the
-        // symbol file information.
-        m_unwind_table.reset();
-
         // The symbol file might be a directory bundle ("/tmp/a.out.dSYM")
         // instead of a full path to the symbol file within the bundle
         // ("/tmp/a.out.dSYM/Contents/Resources/DWARF/a.out"). So we need to
         // check this
-
         if (FileSystem::Instance().IsDirectory(file)) {
           std::string new_path(file.GetPath());
           std::string old_path(obj_file->GetFileSpec().GetPath());
-- 
GitLab


From 44d037cc258dcf179d2c48c93996bb406ecd0fae Mon Sep 17 00:00:00 2001
From: Abhin P Jose <abhinjose@live.com>
Date: Tue, 26 Mar 2024 17:10:54 +0100
Subject: [PATCH 296/404] Change type of DiagnosticHandlerTy (#86504)

Changing type of DiagnosticHandlerTy due to adding
-Wcast-function-type-mismatch to -Wextra
group(https://github.com/llvm/llvm-project/pull/86131#issuecomment-2018014179).
Changed the reference argument DiagnosticInfo to a pointer and edited
the test cases failing due to this change. Added another small change
where Gtest api was throwing an warning due varargs argument not being
passed.
---
 llvm/include/llvm/IR/DiagnosticHandler.h  | 4 ++--
 llvm/unittests/Linker/LinkModulesTest.cpp | 2 +-
 llvm/unittests/Support/ThreadPool.cpp     | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/include/llvm/IR/DiagnosticHandler.h b/llvm/include/llvm/IR/DiagnosticHandler.h
index db7d7444f75f..1f0544b9fa9f 100644
--- a/llvm/include/llvm/IR/DiagnosticHandler.h
+++ b/llvm/include/llvm/IR/DiagnosticHandler.h
@@ -28,7 +28,7 @@ struct DiagnosticHandler {
       : DiagnosticContext(DiagContext) {}
   virtual ~DiagnosticHandler() = default;
 
-  using DiagnosticHandlerTy = void (*)(const DiagnosticInfo &DI, void *Context);
+  using DiagnosticHandlerTy = void (*)(const DiagnosticInfo *DI, void *Context);
 
   /// DiagHandlerCallback is settable from the C API and base implementation
   /// of DiagnosticHandler will call it from handleDiagnostics(). Any derived
@@ -42,7 +42,7 @@ struct DiagnosticHandler {
   /// with a prefix based on the severity.
   virtual bool handleDiagnostics(const DiagnosticInfo &DI) {
     if (DiagHandlerCallback) {
-      DiagHandlerCallback(DI, DiagnosticContext);
+      DiagHandlerCallback(&DI, DiagnosticContext);
       return true;
     }
     return false;
diff --git a/llvm/unittests/Linker/LinkModulesTest.cpp b/llvm/unittests/Linker/LinkModulesTest.cpp
index 182ce73178c1..884e20e89c5c 100644
--- a/llvm/unittests/Linker/LinkModulesTest.cpp
+++ b/llvm/unittests/Linker/LinkModulesTest.cpp
@@ -72,7 +72,7 @@ protected:
   BasicBlock *ExitBB;
 };
 
-static void expectNoDiags(const DiagnosticInfo &DI, void *C) {
+static void expectNoDiags(const DiagnosticInfo *DI, void *C) {
   llvm_unreachable("expectNoDiags called!");
 }
 
diff --git a/llvm/unittests/Support/ThreadPool.cpp b/llvm/unittests/Support/ThreadPool.cpp
index d74c625d1229..381b4fc2a26b 100644
--- a/llvm/unittests/Support/ThreadPool.cpp
+++ b/llvm/unittests/Support/ThreadPool.cpp
@@ -126,7 +126,7 @@ using ThreadPoolImpls = ::testing::Types<
 #endif
     SingleThreadExecutor>;
 
-TYPED_TEST_SUITE(ThreadPoolTest, ThreadPoolImpls);
+TYPED_TEST_SUITE(ThreadPoolTest, ThreadPoolImpls, );
 
 #define CHECK_UNSUPPORTED()                                                    \
   do {                                                                         \
-- 
GitLab


From 375ddd677c08fa32ed188ba20097c1cb0949d846 Mon Sep 17 00:00:00 2001
From: Alfie Richards <alfie.richards@arm.com>
Date: Tue, 26 Mar 2024 16:13:41 +0000
Subject: [PATCH 297/404] [ARM][MC] Add GNU Alias for ldrexd, ldaexd, stlexd,
 and strexd instructions (#86507)

These aliases were supported previously there was a regression at some point.

This adds back the alternate forms and tidies up this section of code a little.

See https://github.com/llvm/llvm-project/pull/83436#issuecomment-2010213714 for the initial report regarding this change.
---
 .../lib/Target/ARM/AsmParser/ARMAsmParser.cpp | 57 ++++++++++++-------
 llvm/test/MC/ARM/basic-arm-instructions.s     |  8 +++
 .../ARM/load-store-acquire-release-v8-thumb.s |  8 +++
 .../MC/ARM/load-store-acquire-release-v8.s    | 17 +++++-
 4 files changed, 67 insertions(+), 23 deletions(-)

diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index 2ad576ab1a9c..d63e53df284b 100644
--- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -6865,8 +6865,9 @@ static void applyMnemonicAliases(StringRef &Mnemonic,
                                  const FeatureBitset &Features,
                                  unsigned VariantID);
 
-// The GNU assembler has aliases of ldrd and strd with the second register
-// omitted. We don't have a way to do that in tablegen, so fix it up here.
+// The GNU assembler has aliases of ldrd, strd, ldrexd, strexd, ldaexd, and
+// stlexd with the second register omitted. We don't have a way to do that in
+// tablegen, so fix it up here.
 //
 // We have to be careful to not emit an invalid Rt2 here, because the rest of
 // the assembly parser could then generate confusing diagnostics refering to
@@ -6876,13 +6877,19 @@ static void applyMnemonicAliases(StringRef &Mnemonic,
 void ARMAsmParser::fixupGNULDRDAlias(StringRef Mnemonic,
                                      OperandVector &Operands,
                                      unsigned MnemonicOpsEndInd) {
-  if (Mnemonic != "ldrd" && Mnemonic != "strd")
+  if (Mnemonic != "ldrd" && Mnemonic != "strd" && Mnemonic != "ldrexd" &&
+      Mnemonic != "strexd" && Mnemonic != "ldaexd" && Mnemonic != "stlexd")
     return;
-  if (Operands.size() < MnemonicOpsEndInd + 2)
+
+  unsigned IdX = Mnemonic == "strexd" || Mnemonic == "stlexd"
+                     ? MnemonicOpsEndInd + 1
+                     : MnemonicOpsEndInd;
+
+  if (Operands.size() < IdX + 2)
     return;
 
-  ARMOperand &Op2 = static_cast<ARMOperand &>(*Operands[MnemonicOpsEndInd]);
-  ARMOperand &Op3 = static_cast<ARMOperand &>(*Operands[MnemonicOpsEndInd + 1]);
+  ARMOperand &Op2 = static_cast<ARMOperand &>(*Operands[IdX]);
+  ARMOperand &Op3 = static_cast<ARMOperand &>(*Operands[IdX + 1]);
 
   if (!Op2.isReg())
     return;
@@ -6907,7 +6914,7 @@ void ARMAsmParser::fixupGNULDRDAlias(StringRef Mnemonic,
     return;
 
   Operands.insert(
-      Operands.begin() + MnemonicOpsEndInd + 1,
+      Operands.begin() + IdX + 1,
       ARMOperand::CreateReg(PairedReg, Op2.getStartLoc(), Op2.getEndLoc()));
 }
 
@@ -7336,6 +7343,9 @@ bool ARMAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
       static_cast<ARMOperand &>(*Operands[MnemonicOpsEndInd]).isImm())
     removeCondCode(Operands, MnemonicOpsEndInd);
 
+  // GNU Assembler extension (compatibility).
+  fixupGNULDRDAlias(Mnemonic, Operands, MnemonicOpsEndInd);
+
   // Adjust operands of ldrexd/strexd to MCK_GPRPair.
   // ldrexd/strexd require even/odd GPR pair. To enforce this constraint,
   // a single GPRPair reg operand is used in the .td file to replace the two
@@ -7343,40 +7353,43 @@ bool ARMAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
   // automatically
   // expressed as a GPRPair, so we have to manually merge them.
   // FIXME: We would really like to be able to tablegen'erate this.
-  if (!isThumb() && Operands.size() > MnemonicOpsEndInd + 1 &&
+  bool IsLoad = (Mnemonic == "ldrexd" || Mnemonic == "ldaexd");
+  if (!isThumb() && Operands.size() > MnemonicOpsEndInd + 1 + (!IsLoad) &&
       (Mnemonic == "ldrexd" || Mnemonic == "strexd" || Mnemonic == "ldaexd" ||
        Mnemonic == "stlexd")) {
-    bool isLoad = (Mnemonic == "ldrexd" || Mnemonic == "ldaexd");
-    unsigned Idx = isLoad ? MnemonicOpsEndInd : MnemonicOpsEndInd + 1;
+    unsigned Idx = IsLoad ? MnemonicOpsEndInd : MnemonicOpsEndInd + 1;
     ARMOperand &Op1 = static_cast<ARMOperand &>(*Operands[Idx]);
     ARMOperand &Op2 = static_cast<ARMOperand &>(*Operands[Idx + 1]);
 
     const MCRegisterClass &MRC = MRI->getRegClass(ARM::GPRRegClassID);
-    // Adjust only if Op1 and Op2 are GPRs.
-    if (Op1.isReg() && Op2.isReg() && MRC.contains(Op1.getReg()) &&
-        MRC.contains(Op2.getReg())) {
+    // Adjust only if Op1 is a GPR.
+    if (Op1.isReg() && MRC.contains(Op1.getReg())) {
       unsigned Reg1 = Op1.getReg();
-      unsigned Reg2 = Op2.getReg();
       unsigned Rt = MRI->getEncodingValue(Reg1);
+      unsigned Reg2 = Op2.getReg();
       unsigned Rt2 = MRI->getEncodingValue(Reg2);
-
-      // Rt2 must be Rt + 1 and Rt must be even.
-      if (Rt + 1 != Rt2 || (Rt & 1)) {
+      // Rt2 must be Rt + 1.
+      if (Rt + 1 != Rt2)
         return Error(Op2.getStartLoc(),
-                     isLoad ? "destination operands must be sequential"
+                     IsLoad ? "destination operands must be sequential"
                             : "source operands must be sequential");
-      }
+
+      // Rt must be even
+      if (Rt & 1)
+        return Error(
+            Op1.getStartLoc(),
+            IsLoad ? "destination operands must start start at an even register"
+                   : "source operands must start start at an even register");
+
       unsigned NewReg = MRI->getMatchingSuperReg(
           Reg1, ARM::gsub_0, &(MRI->getRegClass(ARM::GPRPairRegClassID)));
+
       Operands[Idx] =
           ARMOperand::CreateReg(NewReg, Op1.getStartLoc(), Op2.getEndLoc());
       Operands.erase(Operands.begin() + Idx + 1);
     }
   }
 
-  // GNU Assembler extension (compatibility).
-  fixupGNULDRDAlias(Mnemonic, Operands, MnemonicOpsEndInd);
-
   // FIXME: As said above, this is all a pretty gross hack.  This instruction
   // does not fit with other "subs" and tblgen.
   // Adjust operands of B9.3.19 SUBS PC, LR, #imm (Thumb2) system instruction
diff --git a/llvm/test/MC/ARM/basic-arm-instructions.s b/llvm/test/MC/ARM/basic-arm-instructions.s
index 84a7cf52fa30..9f3a5cd4afa7 100644
--- a/llvm/test/MC/ARM/basic-arm-instructions.s
+++ b/llvm/test/MC/ARM/basic-arm-instructions.s
@@ -1202,6 +1202,10 @@ Lforward:
 @ CHECK: ldrex	r1, [r7]                @ encoding: [0x9f,0x1f,0x97,0xe1]
 @ CHECK: ldrexd	r6, r7, [r8]            @ encoding: [0x9f,0x6f,0xb8,0xe1]
 
+@ GNU alias
+        ldrexd  r6, [r8]
+@ CHECK: ldrexd	r6, r7, [r8]            @ encoding: [0x9f,0x6f,0xb8,0xe1]
+
 @------------------------------------------------------------------------------
 @ LDRHT
 @------------------------------------------------------------------------------
@@ -2904,6 +2908,10 @@ Lforward:
 @ CHECK: strex	r2, r1, [r7]            @ encoding: [0x91,0x2f,0x87,0xe1]
 @ CHECK: strexd	r6, r2, r3, [r8]        @ encoding: [0x92,0x6f,0xa8,0xe1]
 
+@ GNU alias
+        strexd  r6, r2, [r8]
+@ CHECK: strexd	r6, r2, r3, [r8]        @ encoding: [0x92,0x6f,0xa8,0xe1]
+
 @------------------------------------------------------------------------------
 @ STR
 @------------------------------------------------------------------------------
diff --git a/llvm/test/MC/ARM/load-store-acquire-release-v8-thumb.s b/llvm/test/MC/ARM/load-store-acquire-release-v8-thumb.s
index be8d3c324e68..b802bfdb28f7 100644
--- a/llvm/test/MC/ARM/load-store-acquire-release-v8-thumb.s
+++ b/llvm/test/MC/ARM/load-store-acquire-release-v8-thumb.s
@@ -14,6 +14,10 @@
 @ CHECK-V7: error: instruction requires: acquire/release
 @ CHECK-V7: error: instruction requires: acquire/release
 
+@ GNU alias
+        ldaexd  r6, [r8]
+@ CHECK:  ldaexd	r6, r7, [r8]            @ encoding: [0xd8,0xe8,0xff,0x67]
+
         stlexb  r1, r3, [r4]
         stlexh  r4, r2, [r5]
         stlex  r2, r1, [r7]
@@ -27,6 +31,10 @@
 @ CHECK-V7: error: instruction requires: acquire/release
 @ CHECK-V7: error: instruction requires: acquire/release
 
+@ GNU alias
+        stlexd  r6, r2, [r8]
+@ CHECK: stlexd r6, r2, r3, [r8]        @ encoding: [0xc8,0xe8,0xf6,0x23]
+
          lda r5, [r6]
          ldab r5, [r6]
          ldah r12, [r9]
diff --git a/llvm/test/MC/ARM/load-store-acquire-release-v8.s b/llvm/test/MC/ARM/load-store-acquire-release-v8.s
index 273519e050b1..edfe14c93e4e 100644
--- a/llvm/test/MC/ARM/load-store-acquire-release-v8.s
+++ b/llvm/test/MC/ARM/load-store-acquire-release-v8.s
@@ -1,4 +1,5 @@
-@ RUN: llvm-mc -triple=armv8 -show-encoding < %s | FileCheck %s
+@ RUN: not llvm-mc -triple=armv8 -show-encoding < %s 2> %t | FileCheck %s
+@ RUN: FileCheck %s < %t --check-prefix=CHECK-ERROR
 @ RUN: not llvm-mc -triple=armv7 -show-encoding < %s 2>&1 | FileCheck %s --check-prefix=CHECK-V7
         ldaexb  r3, [r4]
         ldaexh  r2, [r5]
@@ -14,6 +15,13 @@
 @ CHECK-V7: instruction requires: acquire/release
 @ CHECK-V7: instruction requires: acquire/release
 
+        ldaexd  r2, r4, [r8]
+@ CHECK-ERROR: error: destination operands must be sequential
+
+@ GNU alias
+        ldaexd  r6, [r8]
+@ CHECK:  ldaexd	r6, r7, [r8]            @ encoding: [0x9f,0x6e,0xb8,0xe1]
+
         stlexb  r1, r3, [r4]
         stlexh  r4, r2, [r5]
         stlex  r2, r1, [r7]
@@ -27,6 +35,13 @@
 @ CHECK-V7: instruction requires: acquire/release
 @ CHECK-V7: instruction requires: acquire/release
 
+        stlexd  r6, r2, r4, [r8]
+@ CHECK-ERROR: error: source operands must be sequential
+
+@ GNU alias
+        stlexd  r6, r2, [r8]
+@ CHECK: stlexd r6, r2, r3, [r8]        @ encoding: [0x92,0x6e,0xa8,0xe1]
+
          lda r5, [r6]
          ldab r5, [r6]
          ldah r12, [r9]
-- 
GitLab


From 348f9e73d7175f67750dc40b15cf1fc888bf60b5 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Tue, 26 Mar 2024 09:12:16 -0700
Subject: [PATCH 298/404] [lldb] Fix warning: comparison of unsigned expression
 in >= 0 is always true
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp:1195:15: warning:
comparison of unsigned expression in ‘>= 0’ is always true
 1195 |   if (weekday >= 0 && weekday < 7)
      |       ~~~~~~~~^~~~
---
 lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
index bcb04fae15bd..d2d50152c07c 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
@@ -1192,7 +1192,7 @@ bool lldb_private::formatters::LibcxxChronoWeekdaySummaryProvider(
     return false;
 
   const unsigned weekday = ptr_sp->GetValueAsUnsigned(0);
-  if (weekday >= 0 && weekday < 7)
+  if (weekday < 7)
     stream << "weekday=" << weekdays[weekday];
   else
     stream.Printf("weekday=%u", weekday);
-- 
GitLab


From 3140d138e4ae9bba35dd23ce2eb0d1e088596f0a Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 26 Mar 2024 15:52:13 +0000
Subject: [PATCH 299/404] [X86] extractelement-load.ll - use X86 instead of X32
 check prefix. NFC

X32 should be used for gnux32 triples
---
 llvm/test/CodeGen/X86/extractelement-load.ll | 282 +++++++++----------
 1 file changed, 141 insertions(+), 141 deletions(-)

diff --git a/llvm/test/CodeGen/X86/extractelement-load.ll b/llvm/test/CodeGen/X86/extractelement-load.ll
index 9d573ef2a8fa..9a5f296f4516 100644
--- a/llvm/test/CodeGen/X86/extractelement-load.ll
+++ b/llvm/test/CodeGen/X86/extractelement-load.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X32-SSE2
+; RUN: llc < %s -mtriple=i686-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X86-SSE2
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=X64,X64-SSSE3
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx  | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX2
@@ -7,12 +7,12 @@
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define i32 @t(ptr %val) nounwind  {
-; X32-SSE2-LABEL: t:
-; X32-SSE2:       # %bb.0:
-; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = mem[2,3,2,3]
-; X32-SSE2-NEXT:    movd %xmm0, %eax
-; X32-SSE2-NEXT:    retl
+; X86-SSE2-LABEL: t:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = mem[2,3,2,3]
+; X86-SSE2-NEXT:    movd %xmm0, %eax
+; X86-SSE2-NEXT:    retl
 ;
 ; X64-SSSE3-LABEL: t:
 ; X64-SSSE3:       # %bb.0:
@@ -33,9 +33,9 @@ define i32 @t(ptr %val) nounwind  {
 ; Case where extractelement of load ends up as undef.
 ; (Making sure this doesn't crash.)
 define i32 @t2(ptr %xp) {
-; X32-SSE2-LABEL: t2:
-; X32-SSE2:       # %bb.0:
-; X32-SSE2-NEXT:    retl
+; X86-SSE2-LABEL: t2:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    retl
 ;
 ; X64-LABEL: t2:
 ; X64:       # %bb.0:
@@ -51,12 +51,12 @@ define i32 @t2(ptr %xp) {
 ; narrow load.
 
 define void @t3(ptr %a0) {
-; X32-SSE2-LABEL: t3:
-; X32-SSE2:       # %bb.0: # %bb
-; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE2-NEXT:    movups (%eax), %xmm0
-; X32-SSE2-NEXT:    movhps %xmm0, (%eax)
-; X32-SSE2-NEXT:    retl
+; X86-SSE2-LABEL: t3:
+; X86-SSE2:       # %bb.0: # %bb
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movups (%eax), %xmm0
+; X86-SSE2-NEXT:    movhps %xmm0, (%eax)
+; X86-SSE2-NEXT:    retl
 ;
 ; X64-SSSE3-LABEL: t3:
 ; X64-SSSE3:       # %bb.0: # %bb
@@ -81,14 +81,14 @@ bb:
 ; This is testing for an assertion - the extraction was assuming that the undef
 ; second shuffle operand was a post-bitcast type instead of a pre-bitcast type.
 define i64 @t4(ptr %a) {
-; X32-SSE2-LABEL: t4:
-; X32-SSE2:       # %bb.0:
-; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE2-NEXT:    movdqa (%eax), %xmm0
-; X32-SSE2-NEXT:    movd %xmm0, %eax
-; X32-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X32-SSE2-NEXT:    movd %xmm0, %edx
-; X32-SSE2-NEXT:    retl
+; X86-SSE2-LABEL: t4:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movdqa (%eax), %xmm0
+; X86-SSE2-NEXT:    movd %xmm0, %eax
+; X86-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X86-SSE2-NEXT:    movd %xmm0, %edx
+; X86-SSE2-NEXT:    retl
 ;
 ; X64-LABEL: t4:
 ; X64:       # %bb.0:
@@ -103,13 +103,13 @@ define i64 @t4(ptr %a) {
 
 ; Don't extract from a volatile.
 define void @t5(ptr%a0, ptr%a1) {
-; X32-SSE2-LABEL: t5:
-; X32-SSE2:       # %bb.0:
-; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE2-NEXT:    movaps (%ecx), %xmm0
-; X32-SSE2-NEXT:    movhps %xmm0, (%eax)
-; X32-SSE2-NEXT:    retl
+; X86-SSE2-LABEL: t5:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movaps (%ecx), %xmm0
+; X86-SSE2-NEXT:    movhps %xmm0, (%eax)
+; X86-SSE2-NEXT:    retl
 ;
 ; X64-SSSE3-LABEL: t5:
 ; X64-SSSE3:       # %bb.0:
@@ -130,24 +130,24 @@ define void @t5(ptr%a0, ptr%a1) {
 
 ; Check for multiuse.
 define float @t6(ptr%a0) {
-; X32-SSE2-LABEL: t6:
-; X32-SSE2:       # %bb.0:
-; X32-SSE2-NEXT:    pushl %eax
-; X32-SSE2-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE2-NEXT:    movaps (%eax), %xmm0
-; X32-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X32-SSE2-NEXT:    xorps %xmm1, %xmm1
-; X32-SSE2-NEXT:    cmpeqss %xmm0, %xmm1
-; X32-SSE2-NEXT:    movss {{.*#+}} xmm2 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]
-; X32-SSE2-NEXT:    andps %xmm1, %xmm2
-; X32-SSE2-NEXT:    andnps %xmm0, %xmm1
-; X32-SSE2-NEXT:    orps %xmm2, %xmm1
-; X32-SSE2-NEXT:    movss %xmm1, (%esp)
-; X32-SSE2-NEXT:    flds (%esp)
-; X32-SSE2-NEXT:    popl %eax
-; X32-SSE2-NEXT:    .cfi_def_cfa_offset 4
-; X32-SSE2-NEXT:    retl
+; X86-SSE2-LABEL: t6:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %eax
+; X86-SSE2-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movaps (%eax), %xmm0
+; X86-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X86-SSE2-NEXT:    xorps %xmm1, %xmm1
+; X86-SSE2-NEXT:    cmpeqss %xmm0, %xmm1
+; X86-SSE2-NEXT:    movss {{.*#+}} xmm2 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X86-SSE2-NEXT:    andps %xmm1, %xmm2
+; X86-SSE2-NEXT:    andnps %xmm0, %xmm1
+; X86-SSE2-NEXT:    orps %xmm2, %xmm1
+; X86-SSE2-NEXT:    movss %xmm1, (%esp)
+; X86-SSE2-NEXT:    flds (%esp)
+; X86-SSE2-NEXT:    popl %eax
+; X86-SSE2-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE2-NEXT:    retl
 ;
 ; X64-SSSE3-LABEL: t6:
 ; X64-SSSE3:       # %bb.0:
@@ -184,20 +184,20 @@ define float @t6(ptr%a0) {
 }
 
 define void @PR43971(ptr%a0, ptr%a1) {
-; X32-SSE2-LABEL: PR43971:
-; X32-SSE2:       # %bb.0: # %entry
-; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE2-NEXT:    movaps 16(%ecx), %xmm0
-; X32-SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; X32-SSE2-NEXT:    xorps %xmm1, %xmm1
-; X32-SSE2-NEXT:    cmpltss %xmm0, %xmm1
-; X32-SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; X32-SSE2-NEXT:    andps %xmm1, %xmm2
-; X32-SSE2-NEXT:    andnps %xmm0, %xmm1
-; X32-SSE2-NEXT:    orps %xmm2, %xmm1
-; X32-SSE2-NEXT:    movss %xmm1, (%eax)
-; X32-SSE2-NEXT:    retl
+; X86-SSE2-LABEL: PR43971:
+; X86-SSE2:       # %bb.0: # %entry
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movaps 16(%ecx), %xmm0
+; X86-SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; X86-SSE2-NEXT:    xorps %xmm1, %xmm1
+; X86-SSE2-NEXT:    cmpltss %xmm0, %xmm1
+; X86-SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; X86-SSE2-NEXT:    andps %xmm1, %xmm2
+; X86-SSE2-NEXT:    andnps %xmm0, %xmm1
+; X86-SSE2-NEXT:    orps %xmm2, %xmm1
+; X86-SSE2-NEXT:    movss %xmm1, (%eax)
+; X86-SSE2-NEXT:    retl
 ;
 ; X64-SSSE3-LABEL: PR43971:
 ; X64-SSSE3:       # %bb.0: # %entry
@@ -231,22 +231,22 @@ entry:
 }
 
 define float @PR43971_1(ptr%a0) nounwind {
-; X32-SSE2-LABEL: PR43971_1:
-; X32-SSE2:       # %bb.0: # %entry
-; X32-SSE2-NEXT:    pushl %eax
-; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE2-NEXT:    movaps (%eax), %xmm0
-; X32-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X32-SSE2-NEXT:    xorps %xmm1, %xmm1
-; X32-SSE2-NEXT:    cmpeqss %xmm0, %xmm1
-; X32-SSE2-NEXT:    movss {{.*#+}} xmm2 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]
-; X32-SSE2-NEXT:    andps %xmm1, %xmm2
-; X32-SSE2-NEXT:    andnps %xmm0, %xmm1
-; X32-SSE2-NEXT:    orps %xmm2, %xmm1
-; X32-SSE2-NEXT:    movss %xmm1, (%esp)
-; X32-SSE2-NEXT:    flds (%esp)
-; X32-SSE2-NEXT:    popl %eax
-; X32-SSE2-NEXT:    retl
+; X86-SSE2-LABEL: PR43971_1:
+; X86-SSE2:       # %bb.0: # %entry
+; X86-SSE2-NEXT:    pushl %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movaps (%eax), %xmm0
+; X86-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X86-SSE2-NEXT:    xorps %xmm1, %xmm1
+; X86-SSE2-NEXT:    cmpeqss %xmm0, %xmm1
+; X86-SSE2-NEXT:    movss {{.*#+}} xmm2 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; X86-SSE2-NEXT:    andps %xmm1, %xmm2
+; X86-SSE2-NEXT:    andnps %xmm0, %xmm1
+; X86-SSE2-NEXT:    orps %xmm2, %xmm1
+; X86-SSE2-NEXT:    movss %xmm1, (%esp)
+; X86-SSE2-NEXT:    flds (%esp)
+; X86-SSE2-NEXT:    popl %eax
+; X86-SSE2-NEXT:    retl
 ;
 ; X64-SSSE3-LABEL: PR43971_1:
 ; X64-SSSE3:       # %bb.0: # %entry
@@ -285,15 +285,15 @@ entry:
 
 ; Test for bad extractions from a VBROADCAST_LOAD of the <2 x i16> non-uniform constant bitcast as <4 x i32>.
 define void @subextract_broadcast_load_constant(ptr nocapture %0, ptr nocapture %1, ptr nocapture %2) nounwind {
-; X32-SSE2-LABEL: subextract_broadcast_load_constant:
-; X32-SSE2:       # %bb.0:
-; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-SSE2-NEXT:    movl $-1583308898, (%edx) # imm = 0xA1A09F9E
-; X32-SSE2-NEXT:    movw $-24674, (%ecx) # imm = 0x9F9E
-; X32-SSE2-NEXT:    movw $-24160, (%eax) # imm = 0xA1A0
-; X32-SSE2-NEXT:    retl
+; X86-SSE2-LABEL: subextract_broadcast_load_constant:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-SSE2-NEXT:    movl $-1583308898, (%edx) # imm = 0xA1A09F9E
+; X86-SSE2-NEXT:    movw $-24674, (%ecx) # imm = 0x9F9E
+; X86-SSE2-NEXT:    movw $-24160, (%eax) # imm = 0xA1A0
+; X86-SSE2-NEXT:    retl
 ;
 ; X64-LABEL: subextract_broadcast_load_constant:
 ; X64:       # %bb.0:
@@ -319,15 +319,15 @@ define void @subextract_broadcast_load_constant(ptr nocapture %0, ptr nocapture
 ; A scalar load is favored over a XMM->GPR register transfer in this example.
 
 define i32 @multi_use_load_scalarization(ptr %p) nounwind {
-; X32-SSE2-LABEL: multi_use_load_scalarization:
-; X32-SSE2:       # %bb.0:
-; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE2-NEXT:    movl (%ecx), %eax
-; X32-SSE2-NEXT:    movdqu (%ecx), %xmm0
-; X32-SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; X32-SSE2-NEXT:    psubd %xmm1, %xmm0
-; X32-SSE2-NEXT:    movdqa %xmm0, (%ecx)
-; X32-SSE2-NEXT:    retl
+; X86-SSE2-LABEL: multi_use_load_scalarization:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movl (%ecx), %eax
+; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; X86-SSE2-NEXT:    psubd %xmm1, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, (%ecx)
+; X86-SSE2-NEXT:    retl
 ;
 ; X64-SSSE3-LABEL: multi_use_load_scalarization:
 ; X64-SSSE3:       # %bb.0:
@@ -354,15 +354,15 @@ define i32 @multi_use_load_scalarization(ptr %p) nounwind {
 }
 
 define i32 @multi_use_volatile_load_scalarization(ptr %p) nounwind {
-; X32-SSE2-LABEL: multi_use_volatile_load_scalarization:
-; X32-SSE2:       # %bb.0:
-; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE2-NEXT:    movdqu (%ecx), %xmm0
-; X32-SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; X32-SSE2-NEXT:    movd %xmm0, %eax
-; X32-SSE2-NEXT:    psubd %xmm1, %xmm0
-; X32-SSE2-NEXT:    movdqa %xmm0, (%ecx)
-; X32-SSE2-NEXT:    retl
+; X86-SSE2-LABEL: multi_use_volatile_load_scalarization:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; X86-SSE2-NEXT:    movd %xmm0, %eax
+; X86-SSE2-NEXT:    psubd %xmm1, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, (%ecx)
+; X86-SSE2-NEXT:    retl
 ;
 ; X64-SSSE3-LABEL: multi_use_volatile_load_scalarization:
 ; X64-SSSE3:       # %bb.0:
@@ -398,41 +398,41 @@ define i32 @multi_use_volatile_load_scalarization(ptr %p) nounwind {
 @zero = internal unnamed_addr global <8 x i32> zeroinitializer, align 32
 
 define i32 @main() nounwind {
-; X32-SSE2-LABEL: main:
-; X32-SSE2:       # %bb.0:
-; X32-SSE2-NEXT:    pushl %ebp
-; X32-SSE2-NEXT:    movl %esp, %ebp
-; X32-SSE2-NEXT:    pushl %esi
-; X32-SSE2-NEXT:    andl $-32, %esp
-; X32-SSE2-NEXT:    subl $64, %esp
-; X32-SSE2-NEXT:    movdqa zero, %xmm0
-; X32-SSE2-NEXT:    movaps n1+16, %xmm1
-; X32-SSE2-NEXT:    movaps n1, %xmm2
-; X32-SSE2-NEXT:    movaps %xmm2, zero
-; X32-SSE2-NEXT:    movaps %xmm1, zero+16
-; X32-SSE2-NEXT:    movaps {{.*#+}} xmm1 = [2,2,2,2]
-; X32-SSE2-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
-; X32-SSE2-NEXT:    movaps %xmm1, (%esp)
-; X32-SSE2-NEXT:    movdqa (%esp), %xmm1
-; X32-SSE2-NEXT:    movaps {{[0-9]+}}(%esp), %xmm2
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; X32-SSE2-NEXT:    movd %xmm2, %eax
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; X32-SSE2-NEXT:    movd %xmm2, %ecx
-; X32-SSE2-NEXT:    xorl %edx, %edx
-; X32-SSE2-NEXT:    divl %ecx
-; X32-SSE2-NEXT:    movl %eax, %ecx
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X32-SSE2-NEXT:    movd %xmm0, %eax
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; X32-SSE2-NEXT:    movd %xmm0, %esi
-; X32-SSE2-NEXT:    xorl %edx, %edx
-; X32-SSE2-NEXT:    divl %esi
-; X32-SSE2-NEXT:    addl %ecx, %eax
-; X32-SSE2-NEXT:    leal -4(%ebp), %esp
-; X32-SSE2-NEXT:    popl %esi
-; X32-SSE2-NEXT:    popl %ebp
-; X32-SSE2-NEXT:    retl
+; X86-SSE2-LABEL: main:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    pushl %esi
+; X86-SSE2-NEXT:    andl $-32, %esp
+; X86-SSE2-NEXT:    subl $64, %esp
+; X86-SSE2-NEXT:    movdqa zero, %xmm0
+; X86-SSE2-NEXT:    movaps n1+16, %xmm1
+; X86-SSE2-NEXT:    movaps n1, %xmm2
+; X86-SSE2-NEXT:    movaps %xmm2, zero
+; X86-SSE2-NEXT:    movaps %xmm1, zero+16
+; X86-SSE2-NEXT:    movaps {{.*#+}} xmm1 = [2,2,2,2]
+; X86-SSE2-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    movaps %xmm1, (%esp)
+; X86-SSE2-NEXT:    movdqa (%esp), %xmm1
+; X86-SSE2-NEXT:    movaps {{[0-9]+}}(%esp), %xmm2
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; X86-SSE2-NEXT:    movd %xmm2, %eax
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; X86-SSE2-NEXT:    movd %xmm2, %ecx
+; X86-SSE2-NEXT:    xorl %edx, %edx
+; X86-SSE2-NEXT:    divl %ecx
+; X86-SSE2-NEXT:    movl %eax, %ecx
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X86-SSE2-NEXT:    movd %xmm0, %eax
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X86-SSE2-NEXT:    movd %xmm0, %esi
+; X86-SSE2-NEXT:    xorl %edx, %edx
+; X86-SSE2-NEXT:    divl %esi
+; X86-SSE2-NEXT:    addl %ecx, %eax
+; X86-SSE2-NEXT:    leal -4(%ebp), %esp
+; X86-SSE2-NEXT:    popl %esi
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
 ;
 ; X64-SSSE3-LABEL: main:
 ; X64-SSSE3:       # %bb.0:
-- 
GitLab


From c8b85add2ec1916f3aa5139dfd258e42a946a468 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 26 Mar 2024 15:56:21 +0000
Subject: [PATCH 300/404] [X86] extractelement-load.ll - add test case for
 #85419

---
 llvm/test/CodeGen/X86/extractelement-load.ll | 41 ++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/llvm/test/CodeGen/X86/extractelement-load.ll b/llvm/test/CodeGen/X86/extractelement-load.ll
index 9a5f296f4516..e3e1cdcd7f56 100644
--- a/llvm/test/CodeGen/X86/extractelement-load.ll
+++ b/llvm/test/CodeGen/X86/extractelement-load.ll
@@ -283,6 +283,47 @@ entry:
   ret float %cond
 }
 
+define i32 @PR85419(ptr %p0) {
+; X86-SSE2-LABEL: PR85419:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movdqa (%eax), %xmm0
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE2-NEXT:    movd %xmm1, %ecx
+; X86-SSE2-NEXT:    xorl %edx, %edx
+; X86-SSE2-NEXT:    orl (%eax), %ecx
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; X86-SSE2-NEXT:    movd %xmm0, %eax
+; X86-SSE2-NEXT:    cmovel %edx, %eax
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSSE3-LABEL: PR85419:
+; X64-SSSE3:       # %bb.0:
+; X64-SSSE3-NEXT:    xorl %ecx, %ecx
+; X64-SSSE3-NEXT:    cmpq $0, (%rdi)
+; X64-SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = mem[2,3,2,3]
+; X64-SSSE3-NEXT:    movd %xmm0, %eax
+; X64-SSSE3-NEXT:    cmovel %ecx, %eax
+; X64-SSSE3-NEXT:    retq
+;
+; X64-AVX-LABEL: PR85419:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    xorl %eax, %eax
+; X64-AVX-NEXT:    cmpq $0, (%rdi)
+; X64-AVX-NEXT:    je .LBB8_2
+; X64-AVX-NEXT:  # %bb.1:
+; X64-AVX-NEXT:    movl 8(%rdi), %eax
+; X64-AVX-NEXT:  .LBB8_2:
+; X64-AVX-NEXT:    retq
+  %load = load <2 x i64>, ptr %p0, align 16
+  %vecext.i = extractelement <2 x i64> %load, i64 0
+  %cmp = icmp eq i64 %vecext.i, 0
+  %.cast = bitcast <2 x i64> %load to <4 x i32>
+  %vecext.i2 = extractelement <4 x i32> %.cast, i64 2
+  %retval.0 = select i1 %cmp, i32 0, i32 %vecext.i2
+  ret i32 %retval.0
+}
+
 ; Test for bad extractions from a VBROADCAST_LOAD of the <2 x i16> non-uniform constant bitcast as <4 x i32>.
 define void @subextract_broadcast_load_constant(ptr nocapture %0, ptr nocapture %1, ptr nocapture %2) nounwind {
 ; X86-SSE2-LABEL: subextract_broadcast_load_constant:
-- 
GitLab


From 3ada883f7c96e099e1a665c091751bff5f16690e Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Tue, 26 Mar 2024 09:21:16 -0700
Subject: [PATCH 301/404] [flang][runtime] Runtime support for REDUCE()
 (#86214)

Supports the REDUCE() transformational intrinsic function of Fortran
(see F'2023 16.9.173) in a manner similar to the existing support for
SUM(), PRODUCT(), &c. There are APIs for total reductions to scalar
results, and APIs for partial reductions that reduce the rank of the
argument by one.

This implementation requires more functions than other reductions
because the various possible types of the user-supplied OPERATION=
function need to be elaborated.

Once the basic API in reduce.h has been approved, later patches will
implement lowering.

REDUCE() is primarily for completeness, not portability; only one other
Fortran compiler implements this F'2018 feature today, and only some
types work correctly with it.
---
 flang/include/flang/Runtime/reduce.h    | 257 ++++++++++++
 flang/include/flang/Runtime/reduction.h |   2 +
 flang/lib/Semantics/check-call.cpp      |   3 +
 flang/runtime/CMakeLists.txt            |   1 +
 flang/runtime/complex-reduction.c       |  22 +
 flang/runtime/complex-reduction.h       |  45 ++
 flang/runtime/io-api.cpp                |  36 +-
 flang/runtime/reduce.cpp                | 526 ++++++++++++++++++++++++
 flang/runtime/reduction-templates.h     |  55 +--
 flang/runtime/tools.cpp                 |  29 ++
 flang/runtime/tools.h                   |  16 +-
 flang/unittests/Runtime/Reduction.cpp   |  37 ++
 12 files changed, 972 insertions(+), 57 deletions(-)
 create mode 100644 flang/include/flang/Runtime/reduce.h
 create mode 100644 flang/runtime/reduce.cpp

diff --git a/flang/include/flang/Runtime/reduce.h b/flang/include/flang/Runtime/reduce.h
new file mode 100644
index 000000000000..975aa6dea305
--- /dev/null
+++ b/flang/include/flang/Runtime/reduce.h
@@ -0,0 +1,257 @@
+//===-- include/flang/Runtime/reduce.h --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Defines the API for implementations of the transformational intrinsic
+// function REDUCE(); see F'2023 16.9.173.
+//
+// Similar to the definition of the APIs for SUM(), &c., in reduction.h,
+// there are typed functions here like ReduceInteger4() for total reductions
+// to scalars and void functions like ReduceInteger4Dim() for partial
+// reductions to smaller arrays.
+
+#ifndef FORTRAN_RUNTIME_REDUCE_H_
+#define FORTRAN_RUNTIME_REDUCE_H_
+
+#include "flang/Common/float128.h"
+#include "flang/Common/uint128.h"
+#include "flang/Runtime/cpp-type.h"
+#include "flang/Runtime/entry-names.h"
+#include <complex>
+#include <cstdint>
+
+namespace Fortran::runtime {
+
+class Descriptor;
+
+template <typename T> using ReductionOperation = T (*)(const T *, const T *);
+template <typename CHAR>
+using ReductionCharOperation = void (*)(CHAR *hiddenResult,
+    std::size_t resultLen, const CHAR *x, const CHAR *y, std::size_t xLen,
+    std::size_t yLen);
+using ReductionDerivedTypeOperation = void (*)(
+    void *hiddenResult, const void *x, const void *y);
+
+extern "C" {
+
+std::int8_t RTDECL(ReduceInteger1)(const Descriptor &,
+    ReductionOperation<std::int8_t>, const char *source, int line, int dim = 0,
+    const Descriptor *mask = nullptr, const std::int8_t *identity = nullptr,
+    bool ordered = true);
+void RTDECL(ReduceInteger1Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::int8_t>, const char *source, int line, int dim,
+    const Descriptor *mask = nullptr, const std::int8_t *identity = nullptr,
+    bool ordered = true);
+std::int16_t RTDECL(ReduceInteger2)(const Descriptor &,
+    ReductionOperation<std::int16_t>, const char *source, int line, int dim = 0,
+    const Descriptor *mask = nullptr, const std::int16_t *identity = nullptr,
+    bool ordered = true);
+void RTDECL(ReduceInteger2Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::int16_t>, const char *source, int line, int dim,
+    const Descriptor *mask = nullptr, const std::int16_t *identity = nullptr,
+    bool ordered = true);
+std::int32_t RTDECL(ReduceInteger4)(const Descriptor &,
+    ReductionOperation<std::int32_t>, const char *source, int line, int dim = 0,
+    const Descriptor *mask = nullptr, const std::int32_t *identity = nullptr,
+    bool ordered = true);
+void RTDECL(ReduceInteger4Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::int32_t>, const char *source, int line, int dim,
+    const Descriptor *mask = nullptr, const std::int32_t *identity = nullptr,
+    bool ordered = true);
+std::int64_t RTDECL(ReduceInteger8)(const Descriptor &,
+    ReductionOperation<std::int64_t>, const char *source, int line, int dim = 0,
+    const Descriptor *mask = nullptr, const std::int64_t *identity = nullptr,
+    bool ordered = true);
+void RTDECL(ReduceInteger8Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::int64_t>, const char *source, int line, int dim,
+    const Descriptor *mask = nullptr, const std::int64_t *identity = nullptr,
+    bool ordered = true);
+#ifdef __SIZEOF_INT128__
+common::int128_t RTDECL(ReduceInteger16)(const Descriptor &,
+    ReductionOperation<common::int128_t>, const char *source, int line,
+    int dim = 0, const Descriptor *mask = nullptr,
+    const common::int128_t *identity = nullptr, bool ordered = true);
+void RTDECL(ReduceInteger16Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<common::int128_t>, const char *source, int line, int dim,
+    const Descriptor *mask = nullptr,
+    const common::int128_t *identity = nullptr, bool ordered = true);
+#endif
+
+// REAL/COMPLEX(2 & 3) return 32-bit float results for the caller to downconvert
+float RTDECL(ReduceReal2)(const Descriptor &, ReductionOperation<float>,
+    const char *source, int line, int dim = 0, const Descriptor *mask = nullptr,
+    const float *identity = nullptr, bool ordered = true);
+void RTDECL(ReduceReal2Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<float>, const char *source, int line, int dim,
+    const Descriptor *mask = nullptr, const float *identity = nullptr,
+    bool ordered = true);
+float RTDECL(ReduceReal3)(const Descriptor &, ReductionOperation<float>,
+    const char *source, int line, int dim = 0, const Descriptor *mask = nullptr,
+    const float *identity = nullptr, bool ordered = true);
+void RTDECL(ReduceReal3Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<float>, const char *source, int line, int dim,
+    const Descriptor *mask = nullptr, const float *identity = nullptr,
+    bool ordered = true);
+float RTDECL(ReduceReal4)(const Descriptor &, ReductionOperation<float>,
+    const char *source, int line, int dim = 0, const Descriptor *mask = nullptr,
+    const float *identity = nullptr, bool ordered = true);
+void RTDECL(ReduceReal4Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<float>, const char *source, int line, int dim,
+    const Descriptor *mask = nullptr, const float *identity = nullptr,
+    bool ordered = true);
+double RTDECL(ReduceReal8)(const Descriptor &, ReductionOperation<double>,
+    const char *source, int line, int dim = 0, const Descriptor *mask = nullptr,
+    const double *identity = nullptr, bool ordered = true);
+void RTDECL(ReduceReal8Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<double>, const char *source, int line, int dim,
+    const Descriptor *mask = nullptr, const double *identity = nullptr,
+    bool ordered = true);
+#if LDBL_MANT_DIG == 64
+long double RTDECL(ReduceReal10)(const Descriptor &,
+    ReductionOperation<long double>, const char *source, int line, int dim = 0,
+    const Descriptor *mask = nullptr, const long double *identity = nullptr,
+    bool ordered = true);
+void RTDECL(ReduceReal10Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<long double>, const char *source, int line, int dim,
+    const Descriptor *mask = nullptr, const long double *identity = nullptr,
+    bool ordered = true);
+#endif
+#if LDBL_MANT_DIG == 113 || HAS_FLOAT128
+CppFloat128Type RTDECL(ReduceReal16)(const Descriptor &,
+    ReductionOperation<CppFloat128Type>, const char *source, int line,
+    int dim = 0, const Descriptor *mask = nullptr,
+    const CppFloat128Type *identity = nullptr, bool ordered = true);
+void RTDECL(ReduceReal16Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<CppFloat128Type>, const char *source, int line, int dim,
+    const Descriptor *mask = nullptr, const CppFloat128Type *identity = nullptr,
+    bool ordered = true);
+#endif
+
+void RTDECL(CppReduceComplex2)(std::complex<float> &, const Descriptor &,
+    ReductionOperation<std::complex<float>>, const char *source, int line,
+    int dim = 0, const Descriptor *mask = nullptr,
+    const std::complex<float> *identity = nullptr, bool ordered = true);
+void RTDECL(CppReduceComplex2Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::complex<float>>, const char *source, int line,
+    int dim, const Descriptor *mask = nullptr,
+    const std::complex<float> *identity = nullptr, bool ordered = true);
+void RTDECL(CppReduceComplex3)(std::complex<float> &, const Descriptor &,
+    ReductionOperation<std::complex<float>>, const char *source, int line,
+    int dim = 0, const Descriptor *mask = nullptr,
+    const std::complex<float> *identity = nullptr, bool ordered = true);
+void RTDECL(CppReduceComplex3Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::complex<float>>, const char *source, int line,
+    int dim, const Descriptor *mask = nullptr,
+    const std::complex<float> *identity = nullptr, bool ordered = true);
+void RTDECL(CppReduceComplex4)(std::complex<float> &, const Descriptor &,
+    ReductionOperation<std::complex<float>>, const char *source, int line,
+    int dim = 0, const Descriptor *mask = nullptr,
+    const std::complex<float> *identity = nullptr, bool ordered = true);
+void RTDECL(CppReduceComplex4Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::complex<float>>, const char *source, int line,
+    int dim, const Descriptor *mask = nullptr,
+    const std::complex<float> *identity = nullptr, bool ordered = true);
+void RTDECL(CppReduceComplex8)(std::complex<double> &, const Descriptor &,
+    ReductionOperation<std::complex<double>>, const char *source, int line,
+    int dim = 0, const Descriptor *mask = nullptr,
+    const std::complex<double> *identity = nullptr, bool ordered = true);
+void RTDECL(CppReduceComplex8Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::complex<double>>, const char *source, int line,
+    int dim, const Descriptor *mask = nullptr,
+    const std::complex<double> *identity = nullptr, bool ordered = true);
+#if LDBL_MANT_DIG == 64
+void RTDECL(CppReduceComplex10)(std::complex<long double> &, const Descriptor &,
+    ReductionOperation<std::complex<long double>>, const char *source, int line,
+    int dim = 0, const Descriptor *mask = nullptr,
+    const std::complex<long double> *identity = nullptr, bool ordered = true);
+void RTDECL(CppReduceComplex10Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::complex<long double>>, const char *source, int line,
+    int dim, const Descriptor *mask = nullptr,
+    const std::complex<long double> *identity = nullptr, bool ordered = true);
+#endif
+#if LDBL_MANT_DIG == 113 || HAS_FLOAT128
+void RTDECL(CppReduceComplex16)(std::complex<CppFloat128Type> &,
+    const Descriptor &, ReductionOperation<std::complex<CppFloat128Type>>,
+    const char *source, int line, int dim = 0, const Descriptor *mask = nullptr,
+    const std::complex<CppFloat128Type> *identity = nullptr,
+    bool ordered = true);
+void RTDECL(CppReduceComplex16Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::complex<CppFloat128Type>>, const char *source,
+    int line, int dim, const Descriptor *mask = nullptr,
+    const std::complex<CppFloat128Type> *identity = nullptr,
+    bool ordered = true);
+#endif
+
+bool RTDECL(ReduceLogical1)(const Descriptor &, ReductionOperation<std::int8_t>,
+    const char *source, int line, int dim = 0, const Descriptor *mask = nullptr,
+    const std::int8_t *identity = nullptr, bool ordered = true);
+void RTDECL(ReduceLogical1Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::int8_t>, const char *source, int line, int dim,
+    const Descriptor *mask = nullptr, const std::int8_t *identity = nullptr,
+    bool ordered = true);
+bool RTDECL(ReduceLogical2)(const Descriptor &,
+    ReductionOperation<std::int16_t>, const char *source, int line, int dim = 0,
+    const Descriptor *mask = nullptr, const std::int16_t *identity = nullptr,
+    bool ordered = true);
+void RTDECL(ReduceLogical2Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::int16_t>, const char *source, int line, int dim,
+    const Descriptor *mask = nullptr, const std::int16_t *identity = nullptr,
+    bool ordered = true);
+bool RTDECL(ReduceLogical4)(const Descriptor &,
+    ReductionOperation<std::int32_t>, const char *source, int line, int dim = 0,
+    const Descriptor *mask = nullptr, const std::int32_t *identity = nullptr,
+    bool ordered = true);
+void RTDECL(ReduceLogical4Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::int32_t>, const char *source, int line, int dim,
+    const Descriptor *mask = nullptr, const std::int32_t *identity = nullptr,
+    bool ordered = true);
+bool RTDECL(ReduceLogical8)(const Descriptor &,
+    ReductionOperation<std::int64_t>, const char *source, int line, int dim = 0,
+    const Descriptor *mask = nullptr, const std::int64_t *identity = nullptr,
+    bool ordered = true);
+void RTDECL(ReduceLogical8Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::int64_t>, const char *source, int line, int dim,
+    const Descriptor *mask = nullptr, const std::int64_t *identity = nullptr,
+    bool ordered = true);
+
+void RTDECL(ReduceChar1)(char *result, const Descriptor &array,
+    ReductionCharOperation<char>, const char *source, int line, int dim = 0,
+    const Descriptor *mask = nullptr, const char *identity = nullptr,
+    bool ordered = true);
+void RTDECL(ReduceCharacter1Dim)(Descriptor &result, const Descriptor &array,
+    ReductionCharOperation<char>, const char *source, int line, int dim,
+    const Descriptor *mask = nullptr, const char *identity = nullptr,
+    bool ordered = true);
+void RTDECL(ReduceChar2)(char16_t *result, const Descriptor &array,
+    ReductionCharOperation<char16_t>, const char *source, int line, int dim = 0,
+    const Descriptor *mask = nullptr, const char16_t *identity = nullptr,
+    bool ordered = true);
+void RTDECL(ReduceCharacter2Dim)(Descriptor &result, const Descriptor &array,
+    ReductionCharOperation<char16_t>, const char *source, int line, int dim,
+    const Descriptor *mask = nullptr, const char16_t *identity = nullptr,
+    bool ordered = true);
+void RTDECL(ReduceChar4)(char32_t *result, const Descriptor &array,
+    ReductionCharOperation<char32_t>, const char *source, int line, int dim = 0,
+    const Descriptor *mask = nullptr, const char32_t *identity = nullptr,
+    bool ordered = true);
+void RTDECL(ReduceCharacter4Dim)(Descriptor &result, const Descriptor &array,
+    ReductionCharOperation<char32_t>, const char *source, int line, int dim,
+    const Descriptor *mask = nullptr, const char32_t *identity = nullptr,
+    bool ordered = true);
+
+void RTDECL(ReduceDerivedType)(char *result, const Descriptor &array,
+    ReductionDerivedTypeOperation, const char *source, int line, int dim = 0,
+    const Descriptor *mask = nullptr, const char *identity = nullptr,
+    bool ordered = true);
+void RTDECL(ReduceDerivedTypeDim)(Descriptor &result, const Descriptor &array,
+    ReductionDerivedTypeOperation, const char *source, int line, int dim,
+    const Descriptor *mask = nullptr, const char *identity = nullptr,
+    bool ordered = true);
+
+} // extern "C"
+} // namespace Fortran::runtime
+#endif // FORTRAN_RUNTIME_REDUCE_H_
diff --git a/flang/include/flang/Runtime/reduction.h b/flang/include/flang/Runtime/reduction.h
index 5b6077658575..97986c12e8a1 100644
--- a/flang/include/flang/Runtime/reduction.h
+++ b/flang/include/flang/Runtime/reduction.h
@@ -89,9 +89,11 @@ void RTDECL(CppSumComplex4)(std::complex<float> &, const Descriptor &,
 void RTDECL(CppSumComplex8)(std::complex<double> &, const Descriptor &,
     const char *source, int line, int dim = 0,
     const Descriptor *mask = nullptr);
+#if LDBL_MANT_DIG == 64
 void RTDECL(CppSumComplex10)(std::complex<long double> &, const Descriptor &,
     const char *source, int line, int dim = 0,
     const Descriptor *mask = nullptr);
+#endif
 #if LDBL_MANT_DIG == 113 || HAS_FLOAT128
 void RTDECL(CppSumComplex16)(std::complex<CppFloat128Type> &,
     const Descriptor &, const char *source, int line, int dim = 0,
diff --git a/flang/lib/Semantics/check-call.cpp b/flang/lib/Semantics/check-call.cpp
index d625f8c2f7fc..51a16ee155fa 100644
--- a/flang/lib/Semantics/check-call.cpp
+++ b/flang/lib/Semantics/check-call.cpp
@@ -1588,6 +1588,9 @@ static void CheckReduce(
       procChars->dummyArguments.size() != 2 || !procChars->functionResult) {
     messages.Say(
         "OPERATION= argument of REDUCE() must be a pure function of two data arguments"_err_en_US);
+  } else if (procChars->attrs.test(characteristics::Procedure::Attr::BindC)) {
+    messages.Say(
+        "A BIND(C) OPERATION= argument of REDUCE() is not supported"_err_en_US);
   } else if (!result || result->Rank() != 0) {
     messages.Say(
         "OPERATION= argument of REDUCE() must be a scalar function"_err_en_US);
diff --git a/flang/runtime/CMakeLists.txt b/flang/runtime/CMakeLists.txt
index 335ef3707274..facd14432b3e 100644
--- a/flang/runtime/CMakeLists.txt
+++ b/flang/runtime/CMakeLists.txt
@@ -153,6 +153,7 @@ set(sources
   pseudo-unit.cpp
   ragged.cpp
   random.cpp
+  reduce.cpp
   reduction.cpp
   stat.cpp
   stop.cpp
diff --git a/flang/runtime/complex-reduction.c b/flang/runtime/complex-reduction.c
index c91d12539911..7654de8080a1 100644
--- a/flang/runtime/complex-reduction.c
+++ b/flang/runtime/complex-reduction.c
@@ -155,3 +155,25 @@ ADAPT_REDUCTION(DotProductComplex10, long_double_Complex_t,
 ADAPT_REDUCTION(DotProductComplex16, CFloat128ComplexType, CppComplexFloat128,
     CMPLXF128, DOT_PRODUCT_ARGS, DOT_PRODUCT_ARG_NAMES)
 #endif
+
+/* REDUCE() */
+#define RARGS REDUCE_ARGS(float_Complex_t)
+ADAPT_REDUCTION(ReduceComplex4, float_Complex_t, CppComplexFloat, CMPLXF, RARGS,
+    REDUCE_ARG_NAMES)
+#undef RARGS
+#define RARGS REDUCE_ARGS(double_Complex_t)
+ADAPT_REDUCTION(ReduceComplex8, double_Complex_t, CppComplexDouble, CMPLX,
+    RARGS, REDUCE_ARG_NAMES)
+#undef RARGS
+#if LDBL_MANT_DIG == 64
+#define RARGS REDUCE_ARGS(long_double_Complex_t)
+ADAPT_REDUCTION(ReduceComplex10, long_double_Complex_t, CppComplexLongDouble,
+    CMPLXL, RARGS, REDUCE_ARG_NAMES)
+#undef RARGS
+#endif
+#if LDBL_MANT_DIG == 113 || HAS_FLOAT128
+#define RARGS REDUCE_ARGS(CFloat128ComplexType)
+ADAPT_REDUCTION(ReduceComplex16, CFloat128ComplexType, CppComplexFloat128,
+    CMPLXF128, RARGS, REDUCE_ARG_NAMES)
+#undef RARGS
+#endif
diff --git a/flang/runtime/complex-reduction.h b/flang/runtime/complex-reduction.h
index 1d37b235d519..98b20d1e592b 100644
--- a/flang/runtime/complex-reduction.h
+++ b/flang/runtime/complex-reduction.h
@@ -69,4 +69,49 @@ long_double_Complex_t RTNAME(DotProductComplex10)(DOT_PRODUCT_ARGS);
 CFloat128ComplexType RTNAME(DotProductComplex16)(DOT_PRODUCT_ARGS);
 #endif
 
+#define REDUCE_ARGS(T) \
+  T##_op operation, const struct CppDescriptor *x, \
+      const struct CppDescriptor *y, const char *source, int line, \
+      int dim /*=0*/, const struct CppDescriptor *mask /*=NULL*/, \
+      const T *identity /*=NULL*/, _Bool ordered /*=true*/
+#define REDUCE_ARG_NAMES \
+  operation, x, y, source, line, dim, mask, identity, ordered
+
+typedef float_Complex_t (*float_Complex_t_op)(
+    const float_Complex_t *, const float_Complex_t *);
+typedef double_Complex_t (*double_Complex_t_op)(
+    const double_Complex_t *, const double_Complex_t *);
+typedef long_double_Complex_t (*long_double_Complex_t_op)(
+    const long_double_Complex_t *, const long_double_Complex_t *);
+
+float_Complex_t RTNAME(ReduceComplex2)(REDUCE_ARGS(float_Complex_t));
+float_Complex_t RTNAME(ReduceComplex3)(REDUCE_ARGS(float_Complex_t));
+float_Complex_t RTNAME(ReduceComplex4)(REDUCE_ARGS(float_Complex_t));
+double_Complex_t RTNAME(ReduceComplex8)(REDUCE_ARGS(double_Complex_t));
+long_double_Complex_t RTNAME(ReduceComplex10)(
+    REDUCE_ARGS(long_double_Complex_t));
+#if LDBL_MANT_DIG == 113 || HAS_FLOAT128
+typedef CFloat128ComplexType (*CFloat128ComplexType_op)(
+    const CFloat128ComplexType *, const CFloat128ComplexType *);
+CFloat128ComplexType RTNAME(ReduceComplex16)(REDUCE_ARGS(CFloat128ComplexType));
+#endif
+
+#define REDUCE_DIM_ARGS(T) \
+  struct CppDescriptor *result, T##_op operation, \
+      const struct CppDescriptor *x, const struct CppDescriptor *y, \
+      const char *source, int line, int dim, \
+      const struct CppDescriptor *mask /*=NULL*/, const T *identity /*=NULL*/, \
+      _Bool ordered /*=true*/
+#define REDUCE_DIM_ARG_NAMES \
+  result, operation, x, y, source, line, dim, mask, identity, ordered
+
+void RTNAME(ReduceComplex2Dim)(REDUCE_DIM_ARGS(float_Complex_t));
+void RTNAME(ReduceComplex3Dim)(REDUCE_DIM_ARGS(float_Complex_t));
+void RTNAME(ReduceComplex4Dim)(REDUCE_DIM_ARGS(float_Complex_t));
+void RTNAME(ReduceComplex8Dim)(REDUCE_DIM_ARGS(double_Complex_t));
+void RTNAME(ReduceComplex10Dim)(REDUCE_DIM_ARGS(long_double_Complex_t));
+#if LDBL_MANT_DIG == 113 || HAS_FLOAT128
+void RTNAME(ReduceComplex16Dim)(REDUCE_DIM_ARGS(CFloat128ComplexType));
+#endif
+
 #endif // FORTRAN_RUNTIME_COMPLEX_REDUCTION_H_
diff --git a/flang/runtime/io-api.cpp b/flang/runtime/io-api.cpp
index 0f259f4715bf..3a86c9fa7375 100644
--- a/flang/runtime/io-api.cpp
+++ b/flang/runtime/io-api.cpp
@@ -1147,7 +1147,7 @@ bool IONAME(OutputInteger8)(Cookie cookie, std::int8_t n) {
   if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputInteger8")) {
     return false;
   }
-  StaticDescriptor staticDescriptor;
+  StaticDescriptor<0> staticDescriptor;
   Descriptor &descriptor{staticDescriptor.descriptor()};
   descriptor.Establish(
       TypeCategory::Integer, 1, reinterpret_cast<void *>(&n), 0);
@@ -1158,7 +1158,7 @@ bool IONAME(OutputInteger16)(Cookie cookie, std::int16_t n) {
   if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputInteger16")) {
     return false;
   }
-  StaticDescriptor staticDescriptor;
+  StaticDescriptor<0> staticDescriptor;
   Descriptor &descriptor{staticDescriptor.descriptor()};
   descriptor.Establish(
       TypeCategory::Integer, 2, reinterpret_cast<void *>(&n), 0);
@@ -1170,7 +1170,7 @@ bool IODEF(OutputInteger32)(Cookie cookie, std::int32_t n) {
   if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputInteger32")) {
     return false;
   }
-  StaticDescriptor staticDescriptor;
+  StaticDescriptor<0> staticDescriptor;
   Descriptor &descriptor{staticDescriptor.descriptor()};
   descriptor.Establish(
       TypeCategory::Integer, 4, reinterpret_cast<void *>(&n), 0);
@@ -1182,7 +1182,7 @@ bool IONAME(OutputInteger64)(Cookie cookie, std::int64_t n) {
   if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputInteger64")) {
     return false;
   }
-  StaticDescriptor staticDescriptor;
+  StaticDescriptor<0> staticDescriptor;
   Descriptor &descriptor{staticDescriptor.descriptor()};
   descriptor.Establish(
       TypeCategory::Integer, 8, reinterpret_cast<void *>(&n), 0);
@@ -1194,7 +1194,7 @@ bool IONAME(OutputInteger128)(Cookie cookie, common::int128_t n) {
   if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputInteger128")) {
     return false;
   }
-  StaticDescriptor staticDescriptor;
+  StaticDescriptor<0> staticDescriptor;
   Descriptor &descriptor{staticDescriptor.descriptor()};
   descriptor.Establish(
       TypeCategory::Integer, 16, reinterpret_cast<void *>(&n), 0);
@@ -1206,7 +1206,7 @@ bool IONAME(InputInteger)(Cookie cookie, std::int64_t &n, int kind) {
   if (!cookie->CheckFormattedStmtType<Direction::Input>("InputInteger")) {
     return false;
   }
-  StaticDescriptor staticDescriptor;
+  StaticDescriptor<0> staticDescriptor;
   Descriptor &descriptor{staticDescriptor.descriptor()};
   descriptor.Establish(
       TypeCategory::Integer, kind, reinterpret_cast<void *>(&n), 0);
@@ -1217,7 +1217,7 @@ bool IONAME(OutputReal32)(Cookie cookie, float x) {
   if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputReal32")) {
     return false;
   }
-  StaticDescriptor staticDescriptor;
+  StaticDescriptor<0> staticDescriptor;
   Descriptor &descriptor{staticDescriptor.descriptor()};
   descriptor.Establish(TypeCategory::Real, 4, reinterpret_cast<void *>(&x), 0);
   return descr::DescriptorIO<Direction::Output>(*cookie, descriptor);
@@ -1227,7 +1227,7 @@ bool IONAME(OutputReal64)(Cookie cookie, double x) {
   if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputReal64")) {
     return false;
   }
-  StaticDescriptor staticDescriptor;
+  StaticDescriptor<0> staticDescriptor;
   Descriptor &descriptor{staticDescriptor.descriptor()};
   descriptor.Establish(TypeCategory::Real, 8, reinterpret_cast<void *>(&x), 0);
   return descr::DescriptorIO<Direction::Output>(*cookie, descriptor);
@@ -1237,7 +1237,7 @@ bool IONAME(InputReal32)(Cookie cookie, float &x) {
   if (!cookie->CheckFormattedStmtType<Direction::Input>("InputReal32")) {
     return false;
   }
-  StaticDescriptor staticDescriptor;
+  StaticDescriptor<0> staticDescriptor;
   Descriptor &descriptor{staticDescriptor.descriptor()};
   descriptor.Establish(TypeCategory::Real, 4, reinterpret_cast<void *>(&x), 0);
   return descr::DescriptorIO<Direction::Input>(*cookie, descriptor);
@@ -1247,7 +1247,7 @@ bool IONAME(InputReal64)(Cookie cookie, double &x) {
   if (!cookie->CheckFormattedStmtType<Direction::Input>("InputReal64")) {
     return false;
   }
-  StaticDescriptor staticDescriptor;
+  StaticDescriptor<0> staticDescriptor;
   Descriptor &descriptor{staticDescriptor.descriptor()};
   descriptor.Establish(TypeCategory::Real, 8, reinterpret_cast<void *>(&x), 0);
   return descr::DescriptorIO<Direction::Input>(*cookie, descriptor);
@@ -1258,7 +1258,7 @@ bool IONAME(OutputComplex32)(Cookie cookie, float r, float i) {
     return false;
   }
   float z[2]{r, i};
-  StaticDescriptor staticDescriptor;
+  StaticDescriptor<0> staticDescriptor;
   Descriptor &descriptor{staticDescriptor.descriptor()};
   descriptor.Establish(
       TypeCategory::Complex, 4, reinterpret_cast<void *>(&z), 0);
@@ -1270,7 +1270,7 @@ bool IONAME(OutputComplex64)(Cookie cookie, double r, double i) {
     return false;
   }
   double z[2]{r, i};
-  StaticDescriptor staticDescriptor;
+  StaticDescriptor<0> staticDescriptor;
   Descriptor &descriptor{staticDescriptor.descriptor()};
   descriptor.Establish(
       TypeCategory::Complex, 8, reinterpret_cast<void *>(&z), 0);
@@ -1281,7 +1281,7 @@ bool IONAME(InputComplex32)(Cookie cookie, float z[2]) {
   if (!cookie->CheckFormattedStmtType<Direction::Input>("InputComplex32")) {
     return false;
   }
-  StaticDescriptor staticDescriptor;
+  StaticDescriptor<0> staticDescriptor;
   Descriptor &descriptor{staticDescriptor.descriptor()};
   descriptor.Establish(
       TypeCategory::Complex, 4, reinterpret_cast<void *>(z), 0);
@@ -1292,7 +1292,7 @@ bool IONAME(InputComplex64)(Cookie cookie, double z[2]) {
   if (!cookie->CheckFormattedStmtType<Direction::Input>("InputComplex64")) {
     return false;
   }
-  StaticDescriptor staticDescriptor;
+  StaticDescriptor<0> staticDescriptor;
   Descriptor &descriptor{staticDescriptor.descriptor()};
   descriptor.Establish(
       TypeCategory::Complex, 8, reinterpret_cast<void *>(z), 0);
@@ -1304,7 +1304,7 @@ bool IONAME(OutputCharacter)(
   if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputCharacter")) {
     return false;
   }
-  StaticDescriptor staticDescriptor;
+  StaticDescriptor<0> staticDescriptor;
   Descriptor &descriptor{staticDescriptor.descriptor()};
   descriptor.Establish(
       kind, length, reinterpret_cast<void *>(const_cast<char *>(x)), 0);
@@ -1320,7 +1320,7 @@ bool IONAME(InputCharacter)(
   if (!cookie->CheckFormattedStmtType<Direction::Input>("InputCharacter")) {
     return false;
   }
-  StaticDescriptor staticDescriptor;
+  StaticDescriptor<0> staticDescriptor;
   Descriptor &descriptor{staticDescriptor.descriptor()};
   descriptor.Establish(kind, length, reinterpret_cast<void *>(x), 0);
   return descr::DescriptorIO<Direction::Input>(*cookie, descriptor);
@@ -1334,7 +1334,7 @@ bool IONAME(OutputLogical)(Cookie cookie, bool truth) {
   if (!cookie->CheckFormattedStmtType<Direction::Output>("OutputLogical")) {
     return false;
   }
-  StaticDescriptor staticDescriptor;
+  StaticDescriptor<0> staticDescriptor;
   Descriptor &descriptor{staticDescriptor.descriptor()};
   descriptor.Establish(
       TypeCategory::Logical, sizeof truth, reinterpret_cast<void *>(&truth), 0);
@@ -1345,7 +1345,7 @@ bool IONAME(InputLogical)(Cookie cookie, bool &truth) {
   if (!cookie->CheckFormattedStmtType<Direction::Input>("InputLogical")) {
     return false;
   }
-  StaticDescriptor staticDescriptor;
+  StaticDescriptor<0> staticDescriptor;
   Descriptor &descriptor{staticDescriptor.descriptor()};
   descriptor.Establish(
       TypeCategory::Logical, sizeof truth, reinterpret_cast<void *>(&truth), 0);
diff --git a/flang/runtime/reduce.cpp b/flang/runtime/reduce.cpp
new file mode 100644
index 000000000000..f8a5221a1ebf
--- /dev/null
+++ b/flang/runtime/reduce.cpp
@@ -0,0 +1,526 @@
+//===-- runtime/reduce.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REDUCE() implementation
+
+#include "flang/Runtime/reduce.h"
+#include "reduction-templates.h"
+#include "terminator.h"
+#include "tools.h"
+#include "flang/Runtime/descriptor.h"
+
+namespace Fortran::runtime {
+
+template <typename T> class ReduceAccumulator {
+public:
+  RT_API_ATTRS ReduceAccumulator(const Descriptor &array,
+      ReductionOperation<T> operation, const T *identity,
+      Terminator &terminator)
+      : array_{array}, operation_{operation}, identity_{identity},
+        terminator_{terminator} {}
+  RT_API_ATTRS void Reinitialize() { result_.reset(); }
+  template <typename A>
+  RT_API_ATTRS bool AccumulateAt(const SubscriptValue at[]) {
+    const auto *operand{array_.Element<A>(at)};
+    if (result_) {
+      result_ = operation_(&*result_, operand);
+    } else {
+      result_ = *operand;
+    }
+    return true;
+  }
+  template <typename A>
+  RT_API_ATTRS void GetResult(A *to, int /*zeroBasedDim*/ = -1) {
+    if (result_) {
+      *to = *result_;
+    } else if (identity_) {
+      *to = *identity_;
+    } else {
+      terminator_.Crash("REDUCE() without IDENTITY= has no result");
+    }
+  }
+
+private:
+  const Descriptor &array_;
+  common::optional<T> result_;
+  ReductionOperation<T> operation_;
+  const T *identity_{nullptr};
+  Terminator &terminator_;
+};
+
+template <typename T, typename OP, bool hasLength>
+class BufferedReduceAccumulator {
+public:
+  RT_API_ATTRS BufferedReduceAccumulator(const Descriptor &array, OP operation,
+      const T *identity, Terminator &terminator)
+      : array_{array}, operation_{operation}, identity_{identity},
+        terminator_{terminator} {}
+  RT_API_ATTRS void Reinitialize() { activeTemp_ = -1; }
+  template <typename A>
+  RT_API_ATTRS bool AccumulateAt(const SubscriptValue at[]) {
+    const auto *operand{array_.Element<A>(at)};
+    if (activeTemp_ >= 0) {
+      if constexpr (hasLength) {
+        operation_(&*temp_[1 - activeTemp_], length_, &*temp_[activeTemp_],
+            operand, length_, length_);
+      } else {
+        operation_(&*temp_[1 - activeTemp_], &*temp_[activeTemp_], operand);
+      }
+      activeTemp_ = 1 - activeTemp_;
+    } else {
+      activeTemp_ = 0;
+      std::memcpy(&*temp_[activeTemp_], operand, elementBytes_);
+    }
+    return true;
+  }
+  template <typename A>
+  RT_API_ATTRS void GetResult(A *to, int /*zeroBasedDim*/ = -1) {
+    if (activeTemp_ >= 0) {
+      std::memcpy(to, &*temp_[activeTemp_], elementBytes_);
+    } else if (identity_) {
+      std::memcpy(to, identity_, elementBytes_);
+    } else {
+      terminator_.Crash("REDUCE() without IDENTITY= has no result");
+    }
+  }
+
+private:
+  const Descriptor &array_;
+  OP operation_;
+  const T *identity_{nullptr};
+  Terminator &terminator_;
+  std::size_t elementBytes_{array_.ElementBytes()};
+  OwningPtr<T> temp_[2]{SizedNew<T>{terminator_}(elementBytes_),
+      SizedNew<T>{terminator_}(elementBytes_)};
+  int activeTemp_{-1};
+  std::size_t length_{elementBytes_ / sizeof(T)};
+};
+
+extern "C" {
+RT_EXT_API_GROUP_BEGIN
+
+std::int8_t RTDEF(ReduceInteger1)(const Descriptor &array,
+    ReductionOperation<std::int8_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const std::int8_t *identity,
+    bool ordered) {
+  Terminator terminator{source, line};
+  return GetTotalReduction<TypeCategory::Integer, 1>(array, source, line, dim,
+      mask,
+      ReduceAccumulator<std::int8_t>{array, operation, identity, terminator},
+      "REDUCE");
+}
+void RTDEF(ReduceInteger1Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::int8_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const std::int8_t *identity,
+    bool ordered) {
+  Terminator terminator{source, line};
+  using Accumulator = ReduceAccumulator<std::int8_t>;
+  Accumulator accumulator{array, operation, identity, terminator};
+  PartialReduction<Accumulator, TypeCategory::Integer, 1>(result, array,
+      array.ElementBytes(), dim, mask, terminator, "REDUCE", accumulator);
+}
+std::int16_t RTDEF(ReduceInteger2)(const Descriptor &array,
+    ReductionOperation<std::int16_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const std::int16_t *identity,
+    bool ordered) {
+  Terminator terminator{source, line};
+  return GetTotalReduction<TypeCategory::Integer, 2>(array, source, line, dim,
+      mask,
+      ReduceAccumulator<std::int16_t>{array, operation, identity, terminator},
+      "REDUCE");
+}
+void RTDEF(ReduceInteger2Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::int16_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const std::int16_t *identity,
+    bool ordered) {
+  Terminator terminator{source, line};
+  using Accumulator = ReduceAccumulator<std::int16_t>;
+  Accumulator accumulator{array, operation, identity, terminator};
+  PartialReduction<Accumulator, TypeCategory::Integer, 2>(result, array,
+      array.ElementBytes(), dim, mask, terminator, "REDUCE", accumulator);
+}
+std::int32_t RTDEF(ReduceInteger4)(const Descriptor &array,
+    ReductionOperation<std::int32_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const std::int32_t *identity,
+    bool ordered) {
+  Terminator terminator{source, line};
+  return GetTotalReduction<TypeCategory::Integer, 4>(array, source, line, dim,
+      mask,
+      ReduceAccumulator<std::int32_t>{array, operation, identity, terminator},
+      "REDUCE");
+}
+void RTDEF(ReduceInteger4Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::int32_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const std::int32_t *identity,
+    bool ordered) {
+  Terminator terminator{source, line};
+  using Accumulator = ReduceAccumulator<std::int32_t>;
+  Accumulator accumulator{array, operation, identity, terminator};
+  PartialReduction<Accumulator, TypeCategory::Integer, 4>(result, array,
+      array.ElementBytes(), dim, mask, terminator, "REDUCE", accumulator);
+}
+std::int64_t RTDEF(ReduceInteger8)(const Descriptor &array,
+    ReductionOperation<std::int64_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const std::int64_t *identity,
+    bool ordered) {
+  Terminator terminator{source, line};
+  return GetTotalReduction<TypeCategory::Integer, 8>(array, source, line, dim,
+      mask,
+      ReduceAccumulator<std::int64_t>{array, operation, identity, terminator},
+      "REDUCE");
+}
+void RTDEF(ReduceInteger8Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::int64_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const std::int64_t *identity,
+    bool ordered) {
+  Terminator terminator{source, line};
+  using Accumulator = ReduceAccumulator<std::int64_t>;
+  Accumulator accumulator{array, operation, identity, terminator};
+  PartialReduction<Accumulator, TypeCategory::Integer, 8>(result, array,
+      array.ElementBytes(), dim, mask, terminator, "REDUCE", accumulator);
+}
+#ifdef __SIZEOF_INT128__
+common::int128_t RTDEF(ReduceInteger16)(const Descriptor &array,
+    ReductionOperation<common::int128_t> operation, const char *source,
+    int line, int dim, const Descriptor *mask, const common::int128_t *identity,
+    bool ordered) {
+  Terminator terminator{source, line};
+  return GetTotalReduction<TypeCategory::Integer, 16>(array, source, line, dim,
+      mask,
+      ReduceAccumulator<common::int128_t>{
+          array, operation, identity, terminator},
+      "REDUCE");
+}
+void RTDEF(ReduceInteger16Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<common::int128_t> operation, const char *source,
+    int line, int dim, const Descriptor *mask, const common::int128_t *identity,
+    bool ordered) {
+  Terminator terminator{source, line};
+  using Accumulator = ReduceAccumulator<common::int128_t>;
+  Accumulator accumulator{array, operation, identity, terminator};
+  PartialReduction<Accumulator, TypeCategory::Integer, 16>(result, array,
+      array.ElementBytes(), dim, mask, terminator, "REDUCE", accumulator);
+}
+#endif
+
+// TODO: real/complex(2 & 3)
+float RTDEF(ReduceReal4)(const Descriptor &array,
+    ReductionOperation<float> operation, const char *source, int line, int dim,
+    const Descriptor *mask, const float *identity, bool ordered) {
+  Terminator terminator{source, line};
+  return GetTotalReduction<TypeCategory::Real, 4>(array, source, line, dim,
+      mask, ReduceAccumulator<float>{array, operation, identity, terminator},
+      "REDUCE");
+}
+void RTDEF(ReduceReal4Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<float> operation, const char *source, int line, int dim,
+    const Descriptor *mask, const float *identity, bool ordered) {
+  Terminator terminator{source, line};
+  using Accumulator = ReduceAccumulator<float>;
+  Accumulator accumulator{array, operation, identity, terminator};
+  PartialReduction<Accumulator, TypeCategory::Real, 4>(result, array,
+      array.ElementBytes(), dim, mask, terminator, "REDUCE", accumulator);
+}
+double RTDEF(ReduceReal8)(const Descriptor &array,
+    ReductionOperation<double> operation, const char *source, int line, int dim,
+    const Descriptor *mask, const double *identity, bool ordered) {
+  Terminator terminator{source, line};
+  return GetTotalReduction<TypeCategory::Real, 8>(array, source, line, dim,
+      mask, ReduceAccumulator<double>{array, operation, identity, terminator},
+      "REDUCE");
+}
+void RTDEF(ReduceReal8Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<double> operation, const char *source, int line, int dim,
+    const Descriptor *mask, const double *identity, bool ordered) {
+  Terminator terminator{source, line};
+  using Accumulator = ReduceAccumulator<double>;
+  Accumulator accumulator{array, operation, identity, terminator};
+  PartialReduction<Accumulator, TypeCategory::Real, 8>(result, array,
+      array.ElementBytes(), dim, mask, terminator, "REDUCE", accumulator);
+}
+#if LDBL_MANT_DIG == 64
+long double RTDEF(ReduceReal10)(const Descriptor &array,
+    ReductionOperation<long double> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const long double *identity,
+    bool ordered) {
+  Terminator terminator{source, line};
+  return GetTotalReduction<TypeCategory::Real, 10>(array, source, line, dim,
+      mask,
+      ReduceAccumulator<long double>{array, operation, identity, terminator},
+      "REDUCE");
+}
+void RTDEF(ReduceReal10Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<long double> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const long double *identity,
+    bool ordered) {
+  Terminator terminator{source, line};
+  using Accumulator = ReduceAccumulator<long double>;
+  Accumulator accumulator{array, operation, identity, terminator};
+  PartialReduction<Accumulator, TypeCategory::Real, 10>(result, array,
+      array.ElementBytes(), dim, mask, terminator, "REDUCE", accumulator);
+}
+#endif
+#if LDBL_MANT_DIG == 113 || HAS_FLOAT128
+CppFloat128Type RTDEF(ReduceReal16)(const Descriptor &array,
+    ReductionOperation<CppFloat128Type> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const CppFloat128Type *identity,
+    bool ordered) {
+  Terminator terminator{source, line};
+  return GetTotalReduction<TypeCategory::Real, 16>(array, source, line, dim,
+      mask,
+      ReduceAccumulator<CppFloat128Type>{
+          array, operation, identity, terminator},
+      "REDUCE");
+}
+void RTDEF(ReduceReal16Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<CppFloat128Type> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const CppFloat128Type *identity,
+    bool ordered) {
+  Terminator terminator{source, line};
+  using Accumulator = ReduceAccumulator<CppFloat128Type>;
+  Accumulator accumulator{array, operation, identity, terminator};
+  PartialReduction<Accumulator, TypeCategory::Real, 16>(result, array,
+      array.ElementBytes(), dim, mask, terminator, "REDUCE", accumulator);
+}
+#endif
+
+void RTDEF(CppReduceComplex4)(std::complex<float> &result,
+    const Descriptor &array, ReductionOperation<std::complex<float>> operation,
+    const char *source, int line, int dim, const Descriptor *mask,
+    const std::complex<float> *identity, bool ordered) {
+  Terminator terminator{source, line};
+  result = GetTotalReduction<TypeCategory::Complex, 4>(array, source, line, dim,
+      mask,
+      ReduceAccumulator<std::complex<float>>{
+          array, operation, identity, terminator},
+      "REDUCE");
+}
+void RTDEF(CppReduceComplex4Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::complex<float>> operation, const char *source,
+    int line, int dim, const Descriptor *mask,
+    const std::complex<float> *identity, bool ordered) {
+  Terminator terminator{source, line};
+  using Accumulator = ReduceAccumulator<std::complex<float>>;
+  Accumulator accumulator{array, operation, identity, terminator};
+  PartialReduction<Accumulator, TypeCategory::Complex, 4>(result, array,
+      array.ElementBytes(), dim, mask, terminator, "REDUCE", accumulator);
+}
+void RTDEF(CppReduceComplex8)(std::complex<double> &result,
+    const Descriptor &array, ReductionOperation<std::complex<double>> operation,
+    const char *source, int line, int dim, const Descriptor *mask,
+    const std::complex<double> *identity, bool ordered) {
+  Terminator terminator{source, line};
+  result = GetTotalReduction<TypeCategory::Complex, 8>(array, source, line, dim,
+      mask,
+      ReduceAccumulator<std::complex<double>>{
+          array, operation, identity, terminator},
+      "REDUCE");
+}
+void RTDEF(CppReduceComplex8Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::complex<double>> operation, const char *source,
+    int line, int dim, const Descriptor *mask,
+    const std::complex<double> *identity, bool ordered) {
+  Terminator terminator{source, line};
+  using Accumulator = ReduceAccumulator<std::complex<double>>;
+  Accumulator accumulator{array, operation, identity, terminator};
+  PartialReduction<Accumulator, TypeCategory::Complex, 8>(result, array,
+      array.ElementBytes(), dim, mask, terminator, "REDUCE", accumulator);
+}
+#if LDBL_MANT_DIG == 64
+void RTDEF(CppReduceComplex10)(std::complex<long double> &result,
+    const Descriptor &array,
+    ReductionOperation<std::complex<long double>> operation, const char *source,
+    int line, int dim, const Descriptor *mask,
+    const std::complex<long double> *identity, bool ordered) {
+  Terminator terminator{source, line};
+  result = GetTotalReduction<TypeCategory::Complex, 10>(array, source, line,
+      dim, mask,
+      ReduceAccumulator<std::complex<long double>>{
+          array, operation, identity, terminator},
+      "REDUCE");
+}
+void RTDEF(CppReduceComplex10Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::complex<long double>> operation, const char *source,
+    int line, int dim, const Descriptor *mask,
+    const std::complex<long double> *identity, bool ordered) {
+  Terminator terminator{source, line};
+  using Accumulator = ReduceAccumulator<std::complex<long double>>;
+  Accumulator accumulator{array, operation, identity, terminator};
+  PartialReduction<Accumulator, TypeCategory::Complex, 10>(result, array,
+      array.ElementBytes(), dim, mask, terminator, "REDUCE", accumulator);
+}
+#endif
+#if LDBL_MANT_DIG == 113 || HAS_FLOAT128
+void RTDEF(CppReduceComplex16)(std::complex<CppFloat128Type> &result,
+    const Descriptor &array,
+    ReductionOperation<std::complex<CppFloat128Type>> operation,
+    const char *source, int line, int dim, const Descriptor *mask,
+    const std::complex<CppFloat128Type> *identity, bool ordered) {
+  Terminator terminator{source, line};
+  result = GetTotalReduction<TypeCategory::Complex, 16>(array, source, line,
+      dim, mask,
+      ReduceAccumulator<std::complex<CppFloat128Type>>{
+          array, operation, identity, terminator},
+      "REDUCE");
+}
+void RTDEF(CppReduceComplex16Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::complex<CppFloat128Type>> operation,
+    const char *source, int line, int dim, const Descriptor *mask,
+    const std::complex<CppFloat128Type> *identity, bool ordered) {
+  Terminator terminator{source, line};
+  using Accumulator = ReduceAccumulator<std::complex<CppFloat128Type>>;
+  Accumulator accumulator{array, operation, identity, terminator};
+  PartialReduction<Accumulator, TypeCategory::Complex, 16>(result, array,
+      array.ElementBytes(), dim, mask, terminator, "REDUCE", accumulator);
+}
+#endif
+
+bool RTDEF(ReduceLogical1)(const Descriptor &array,
+    ReductionOperation<std::int8_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const std::int8_t *identity,
+    bool ordered) {
+  return RTNAME(ReduceInteger1)(
+             array, operation, source, line, dim, mask, identity, ordered) != 0;
+}
+void RTDEF(ReduceLogical1Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::int8_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const std::int8_t *identity,
+    bool ordered) {
+  RTNAME(ReduceInteger1Dim)
+  (result, array, operation, source, line, dim, mask, identity, ordered);
+}
+bool RTDEF(ReduceLogical2)(const Descriptor &array,
+    ReductionOperation<std::int16_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const std::int16_t *identity,
+    bool ordered) {
+  return RTNAME(ReduceInteger2)(
+             array, operation, source, line, dim, mask, identity, ordered) != 0;
+}
+void RTDEF(ReduceLogical2Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::int16_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const std::int16_t *identity,
+    bool ordered) {
+  RTNAME(ReduceInteger2Dim)
+  (result, array, operation, source, line, dim, mask, identity, ordered);
+}
+bool RTDEF(ReduceLogical4)(const Descriptor &array,
+    ReductionOperation<std::int32_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const std::int32_t *identity,
+    bool ordered) {
+  return RTNAME(ReduceInteger4)(
+             array, operation, source, line, dim, mask, identity, ordered) != 0;
+}
+void RTDEF(ReduceLogical4Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::int32_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const std::int32_t *identity,
+    bool ordered) {
+  RTNAME(ReduceInteger4Dim)
+  (result, array, operation, source, line, dim, mask, identity, ordered);
+}
+bool RTDEF(ReduceLogical8)(const Descriptor &array,
+    ReductionOperation<std::int64_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const std::int64_t *identity,
+    bool ordered) {
+  return RTNAME(ReduceInteger8)(
+             array, operation, source, line, dim, mask, identity, ordered) != 0;
+}
+void RTDEF(ReduceLogical8Dim)(Descriptor &result, const Descriptor &array,
+    ReductionOperation<std::int64_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const std::int64_t *identity,
+    bool ordered) {
+  RTNAME(ReduceInteger8Dim)
+  (result, array, operation, source, line, dim, mask, identity, ordered);
+}
+
+void RTDEF(ReduceChar1)(char *result, const Descriptor &array,
+    ReductionCharOperation<char> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const char *identity, bool ordered) {
+  Terminator terminator{source, line};
+  BufferedReduceAccumulator<char, ReductionCharOperation<char>,
+      /*hasLength=*/true>
+      accumulator{array, operation, identity, terminator};
+  DoTotalReduction<char>(array, dim, mask, accumulator, "REDUCE", terminator);
+  accumulator.GetResult(result);
+}
+void RTDEF(ReduceCharacter1Dim)(Descriptor &result, const Descriptor &array,
+    ReductionCharOperation<char> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const char *identity, bool ordered) {
+  Terminator terminator{source, line};
+  using Accumulator = BufferedReduceAccumulator<char,
+      ReductionCharOperation<char>, /*hasLength=*/true>;
+  Accumulator accumulator{array, operation, identity, terminator};
+  PartialReduction<Accumulator, TypeCategory::Character, 1>(result, array,
+      array.ElementBytes(), dim, mask, terminator, "REDUCE", accumulator);
+}
+void RTDEF(ReduceChar2)(char16_t *result, const Descriptor &array,
+    ReductionCharOperation<char16_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const char16_t *identity, bool ordered) {
+  Terminator terminator{source, line};
+  BufferedReduceAccumulator<char16_t, ReductionCharOperation<char16_t>,
+      /*hasLength=*/true>
+      accumulator{array, operation, identity, terminator};
+  DoTotalReduction<char16_t>(
+      array, dim, mask, accumulator, "REDUCE", terminator);
+  accumulator.GetResult(result);
+}
+void RTDEF(ReduceCharacter2Dim)(Descriptor &result, const Descriptor &array,
+    ReductionCharOperation<char16_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const char16_t *identity, bool ordered) {
+  Terminator terminator{source, line};
+  using Accumulator = BufferedReduceAccumulator<char16_t,
+      ReductionCharOperation<char16_t>, /*hasLength=*/true>;
+  Accumulator accumulator{array, operation, identity, terminator};
+  PartialReduction<Accumulator, TypeCategory::Character, 2>(result, array,
+      array.ElementBytes(), dim, mask, terminator, "REDUCE", accumulator);
+}
+void RTDEF(ReduceChar4)(char32_t *result, const Descriptor &array,
+    ReductionCharOperation<char32_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const char32_t *identity, bool ordered) {
+  Terminator terminator{source, line};
+  BufferedReduceAccumulator<char32_t, ReductionCharOperation<char32_t>,
+      /*hasLength=*/true>
+      accumulator{array, operation, identity, terminator};
+  DoTotalReduction<char32_t>(
+      array, dim, mask, accumulator, "REDUCE", terminator);
+  accumulator.GetResult(result);
+}
+void RTDEF(ReduceCharacter4Dim)(Descriptor &result, const Descriptor &array,
+    ReductionCharOperation<char32_t> operation, const char *source, int line,
+    int dim, const Descriptor *mask, const char32_t *identity, bool ordered) {
+  Terminator terminator{source, line};
+  using Accumulator = BufferedReduceAccumulator<char32_t,
+      ReductionCharOperation<char32_t>, /*hasLength=*/true>;
+  Accumulator accumulator{array, operation, identity, terminator};
+  PartialReduction<Accumulator, TypeCategory::Character, 4>(result, array,
+      array.ElementBytes(), dim, mask, terminator, "REDUCE", accumulator);
+}
+
+void RTDEF(ReduceDerivedType)(char *result, const Descriptor &array,
+    ReductionDerivedTypeOperation operation, const char *source, int line,
+    int dim, const Descriptor *mask, const char *identity, bool ordered) {
+  Terminator terminator{source, line};
+  BufferedReduceAccumulator<char, ReductionDerivedTypeOperation,
+      /*hasLength=*/false>
+      accumulator{array, operation, identity, terminator};
+  DoTotalReduction<char>(array, dim, mask, accumulator, "REDUCE", terminator);
+  accumulator.GetResult(result);
+}
+void RTDEF(ReduceDerivedTypeDim)(Descriptor &result, const Descriptor &array,
+    ReductionDerivedTypeOperation operation, const char *source, int line,
+    int dim, const Descriptor *mask, const char *identity, bool ordered) {
+  Terminator terminator{source, line};
+  using Accumulator = BufferedReduceAccumulator<char,
+      ReductionDerivedTypeOperation, /*hasLength=*/false>;
+  Accumulator accumulator{array, operation, identity, terminator};
+  PartialReduction<Accumulator, TypeCategory::Derived, 0>(result, array,
+      array.ElementBytes(), dim, mask, terminator, "REDUCE", accumulator);
+}
+
+RT_EXT_API_GROUP_END
+} // extern "C"
+} // namespace Fortran::runtime
diff --git a/flang/runtime/reduction-templates.h b/flang/runtime/reduction-templates.h
index 5b793deb2a12..f8e6f6095509 100644
--- a/flang/runtime/reduction-templates.h
+++ b/flang/runtime/reduction-templates.h
@@ -53,9 +53,9 @@ inline RT_API_ATTRS void DoTotalReduction(const Descriptor &x, int dim,
   x.GetLowerBounds(xAt);
   if (mask) {
     CheckConformability(x, *mask, terminator, intrinsic, "ARRAY", "MASK");
-    SubscriptValue maskAt[maxRank];
-    mask->GetLowerBounds(maskAt);
     if (mask->rank() > 0) {
+      SubscriptValue maskAt[maxRank];
+      mask->GetLowerBounds(maskAt);
       for (auto elements{x.Elements()}; elements--;
            x.IncrementSubscripts(xAt), mask->IncrementSubscripts(maskAt)) {
         if (IsLogicalElementTrue(*mask, maskAt)) {
@@ -65,7 +65,7 @@ inline RT_API_ATTRS void DoTotalReduction(const Descriptor &x, int dim,
         }
       }
       return;
-    } else if (!IsLogicalElementTrue(*mask, maskAt)) {
+    } else if (!IsLogicalScalarTrue(*mask)) {
       // scalar MASK=.FALSE.: return identity value
       return;
     }
@@ -86,13 +86,22 @@ inline RT_API_ATTRS CppTypeFor<CAT, KIND> GetTotalReduction(const Descriptor &x,
   RUNTIME_CHECK(terminator, TypeCode(CAT, KIND) == x.type());
   using CppType = CppTypeFor<CAT, KIND>;
   DoTotalReduction<CppType>(x, dim, mask, accumulator, intrinsic, terminator);
-  CppType result;
+  if constexpr (std::is_void_v<CppType>) {
+    // Result is returned from accumulator, as in REDUCE() for derived type
 #ifdef _MSC_VER // work around MSVC spurious error
-  accumulator.GetResult(&result);
+    accumulator.GetResult();
 #else
-  accumulator.template GetResult(&result);
+    accumulator.template GetResult();
 #endif
-  return result;
+  } else {
+    CppType result;
+#ifdef _MSC_VER // work around MSVC spurious error
+    accumulator.GetResult(&result);
+#else
+    accumulator.template GetResult(&result);
+#endif
+    return result;
+  }
 }
 
 // For reductions on a dimension, e.g. SUM(array,DIM=2) where the shape
@@ -164,35 +173,6 @@ inline RT_API_ATTRS void ReduceDimMaskToScalar(const Descriptor &x,
 #endif
 }
 
-// Utility: establishes & allocates the result array for a partial
-// reduction (i.e., one with DIM=).
-static RT_API_ATTRS void CreatePartialReductionResult(Descriptor &result,
-    const Descriptor &x, std::size_t resultElementSize, int dim,
-    Terminator &terminator, const char *intrinsic, TypeCode typeCode) {
-  int xRank{x.rank()};
-  if (dim < 1 || dim > xRank) {
-    terminator.Crash(
-        "%s: bad DIM=%d for ARRAY with rank %d", intrinsic, dim, xRank);
-  }
-  int zeroBasedDim{dim - 1};
-  SubscriptValue resultExtent[maxRank];
-  for (int j{0}; j < zeroBasedDim; ++j) {
-    resultExtent[j] = x.GetDimension(j).Extent();
-  }
-  for (int j{zeroBasedDim + 1}; j < xRank; ++j) {
-    resultExtent[j - 1] = x.GetDimension(j).Extent();
-  }
-  result.Establish(typeCode, resultElementSize, nullptr, xRank - 1,
-      resultExtent, CFI_attribute_allocatable);
-  for (int j{0}; j + 1 < xRank; ++j) {
-    result.GetDimension(j).SetBounds(1, resultExtent[j]);
-  }
-  if (int stat{result.Allocate()}) {
-    terminator.Crash(
-        "%s: could not allocate memory for result; STAT=%d", intrinsic, stat);
-  }
-}
-
 // Partial reductions with DIM=
 
 template <typename ACCUMULATOR, TypeCategory CAT, int KIND>
@@ -208,7 +188,6 @@ inline RT_API_ATTRS void PartialReduction(Descriptor &result,
   using CppType = CppTypeFor<CAT, KIND>;
   if (mask) {
     CheckConformability(x, *mask, terminator, intrinsic, "ARRAY", "MASK");
-    SubscriptValue maskAt[maxRank]; // contents unused
     if (mask->rank() > 0) {
       for (auto n{result.Elements()}; n-- > 0; result.IncrementSubscripts(at)) {
         accumulator.Reinitialize();
@@ -216,7 +195,7 @@ inline RT_API_ATTRS void PartialReduction(Descriptor &result,
             x, dim - 1, at, *mask, result.Element<CppType>(at), accumulator);
       }
       return;
-    } else if (!IsLogicalElementTrue(*mask, maskAt)) {
+    } else if (!IsLogicalScalarTrue(*mask)) {
       // scalar MASK=.FALSE.
       accumulator.Reinitialize();
       for (auto n{result.Elements()}; n-- > 0; result.IncrementSubscripts(at)) {
diff --git a/flang/runtime/tools.cpp b/flang/runtime/tools.cpp
index 71022c7a8c17..73d6c2cf7e1d 100644
--- a/flang/runtime/tools.cpp
+++ b/flang/runtime/tools.cpp
@@ -238,5 +238,34 @@ template <int KIND> struct FitsInIntegerKind {
   }
 };
 
+// Utility: establishes & allocates the result array for a partial
+// reduction (i.e., one with DIM=).
+RT_API_ATTRS void CreatePartialReductionResult(Descriptor &result,
+    const Descriptor &x, std::size_t resultElementSize, int dim,
+    Terminator &terminator, const char *intrinsic, TypeCode typeCode) {
+  int xRank{x.rank()};
+  if (dim < 1 || dim > xRank) {
+    terminator.Crash(
+        "%s: bad DIM=%d for ARRAY with rank %d", intrinsic, dim, xRank);
+  }
+  int zeroBasedDim{dim - 1};
+  SubscriptValue resultExtent[maxRank];
+  for (int j{0}; j < zeroBasedDim; ++j) {
+    resultExtent[j] = x.GetDimension(j).Extent();
+  }
+  for (int j{zeroBasedDim + 1}; j < xRank; ++j) {
+    resultExtent[j - 1] = x.GetDimension(j).Extent();
+  }
+  result.Establish(typeCode, resultElementSize, nullptr, xRank - 1,
+      resultExtent, CFI_attribute_allocatable);
+  for (int j{0}; j + 1 < xRank; ++j) {
+    result.GetDimension(j).SetBounds(1, resultExtent[j]);
+  }
+  if (int stat{result.Allocate()}) {
+    terminator.Crash(
+        "%s: could not allocate memory for result; STAT=%d", intrinsic, stat);
+  }
+}
+
 RT_OFFLOAD_API_GROUP_END
 } // namespace Fortran::runtime
diff --git a/flang/runtime/tools.h b/flang/runtime/tools.h
index c70a1b438e33..5d7d99c08179 100644
--- a/flang/runtime/tools.h
+++ b/flang/runtime/tools.h
@@ -62,7 +62,7 @@ RT_API_ATTRS int IdentifyValue(
 RT_API_ATTRS void ToFortranDefaultCharacter(
     char *to, std::size_t toLength, const char *from);
 
-// Utility for dealing with elemental LOGICAL arguments
+// Utilities for dealing with elemental LOGICAL arguments
 inline RT_API_ATTRS bool IsLogicalElementTrue(
     const Descriptor &logical, const SubscriptValue at[]) {
   // A LOGICAL value is false if and only if all of its bytes are zero.
@@ -74,6 +74,16 @@ inline RT_API_ATTRS bool IsLogicalElementTrue(
   }
   return false;
 }
+inline RT_API_ATTRS bool IsLogicalScalarTrue(const Descriptor &logical) {
+  // A LOGICAL value is false if and only if all of its bytes are zero.
+  const char *p{logical.OffsetElement<char>()};
+  for (std::size_t j{logical.ElementBytes()}; j-- > 0; ++p) {
+    if (*p) {
+      return true;
+    }
+  }
+  return false;
+}
 
 // Check array conformability; a scalar 'x' conforms.  Crashes on error.
 RT_API_ATTRS void CheckConformability(const Descriptor &to, const Descriptor &x,
@@ -511,5 +521,9 @@ RT_API_ATTRS void CopyAndPad(
   }
 }
 
+RT_API_ATTRS void CreatePartialReductionResult(Descriptor &result,
+    const Descriptor &x, std::size_t resultElementSize, int dim, Terminator &,
+    const char *intrinsic, TypeCode);
+
 } // namespace Fortran::runtime
 #endif // FORTRAN_RUNTIME_TOOLS_H_
diff --git a/flang/unittests/Runtime/Reduction.cpp b/flang/unittests/Runtime/Reduction.cpp
index b17988bc1769..b2661e78abdf 100644
--- a/flang/unittests/Runtime/Reduction.cpp
+++ b/flang/unittests/Runtime/Reduction.cpp
@@ -13,6 +13,7 @@
 #include "flang/Runtime/allocatable.h"
 #include "flang/Runtime/cpp-type.h"
 #include "flang/Runtime/descriptor.h"
+#include "flang/Runtime/reduce.h"
 #include "flang/Runtime/type-code.h"
 #include <cstdint>
 #include <cstring>
@@ -634,3 +635,39 @@ TEST(Reductions, ExtremaReal16) {
   EXPECT_EQ(RTNAME(MaxvalReal16)(*maxArray, __FILE__, __LINE__), -1.0);
 }
 #endif // LDBL_MANT_DIG == 113 || HAS_FLOAT128
+
+static std::int32_t IAdd(const std::int32_t *x, const std::int32_t *y) {
+  return *x + *y;
+}
+
+static std::int32_t IMultiply(const std::int32_t *x, const std::int32_t *y) {
+  return *x * *y;
+}
+
+TEST(Reductions, ReduceInt4) {
+  auto intVector{MakeArray<TypeCategory::Integer, 4>(
+      std::vector<int>{4}, std::vector<std::int32_t>{1, 2, 3, 4})};
+  EXPECT_EQ(RTNAME(ReduceInteger4)(*intVector, IAdd, __FILE__, __LINE__), 10);
+  EXPECT_EQ(
+      RTNAME(ReduceInteger4)(*intVector, IMultiply, __FILE__, __LINE__), 24);
+}
+TEST(Reductions, ReduceInt4Dim) {
+  auto intMatrix{MakeArray<TypeCategory::Integer, 4>(
+      std::vector<int>{2, 2}, std::vector<std::int32_t>{1, 2, 3, 4})};
+  StaticDescriptor<1, true> statDesc;
+  Descriptor &sums{statDesc.descriptor()};
+  RTNAME(ReduceInteger4Dim)(sums, *intMatrix, IAdd, __FILE__, __LINE__, 1);
+  EXPECT_EQ(sums.rank(), 1);
+  EXPECT_EQ(sums.GetDimension(0).LowerBound(), 1);
+  EXPECT_EQ(sums.GetDimension(0).Extent(), 2);
+  EXPECT_EQ(*sums.ZeroBasedIndexedElement<std::int32_t>(0), 3);
+  EXPECT_EQ(*sums.ZeroBasedIndexedElement<std::int32_t>(1), 7);
+  sums.Destroy();
+  RTNAME(ReduceInteger4Dim)(sums, *intMatrix, IAdd, __FILE__, __LINE__, 2);
+  EXPECT_EQ(sums.rank(), 1);
+  EXPECT_EQ(sums.GetDimension(0).LowerBound(), 1);
+  EXPECT_EQ(sums.GetDimension(0).Extent(), 2);
+  EXPECT_EQ(*sums.ZeroBasedIndexedElement<std::int32_t>(0), 4);
+  EXPECT_EQ(*sums.ZeroBasedIndexedElement<std::int32_t>(1), 6);
+  sums.Destroy();
+}
-- 
GitLab


From 5a0382ce7ab3577630bdbb396e9ddaed8749101e Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Tue, 26 Mar 2024 09:40:48 -0700
Subject: [PATCH 302/404] [flang] Catch non-constant targets for procedure
 pointer initialization (#86338)

Detect attempts to use non-constant targets, including internal
procedures, as initializers for procedure pointers, including components
of structure components being used as initializers.
---
 flang/lib/Evaluate/check-expression.cpp | 10 +++++--
 flang/test/Semantics/structconst09.f90  | 37 +++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 3 deletions(-)
 create mode 100644 flang/test/Semantics/structconst09.f90

diff --git a/flang/lib/Evaluate/check-expression.cpp b/flang/lib/Evaluate/check-expression.cpp
index 0e7d97900328..7d721399072c 100644
--- a/flang/lib/Evaluate/check-expression.cpp
+++ b/flang/lib/Evaluate/check-expression.cpp
@@ -358,10 +358,14 @@ bool IsInitialProcedureTarget(const semantics::Symbol &symbol) {
   const auto &ultimate{symbol.GetUltimate()};
   return common::visit(
       common::visitors{
-          [](const semantics::SubprogramDetails &subp) {
-            return !subp.isDummy();
+          [&](const semantics::SubprogramDetails &subp) {
+            return !subp.isDummy() && !subp.stmtFunction() &&
+                symbol.owner().kind() != semantics::Scope::Kind::MainProgram &&
+                symbol.owner().kind() != semantics::Scope::Kind::Subprogram;
+          },
+          [](const semantics::SubprogramNameDetails &x) {
+            return x.kind() != semantics::SubprogramKind::Internal;
           },
-          [](const semantics::SubprogramNameDetails &) { return true; },
           [&](const semantics::ProcEntityDetails &proc) {
             return !semantics::IsPointer(ultimate) && !proc.isDummy();
           },
diff --git a/flang/test/Semantics/structconst09.f90 b/flang/test/Semantics/structconst09.f90
new file mode 100644
index 000000000000..c129f1160685
--- /dev/null
+++ b/flang/test/Semantics/structconst09.f90
@@ -0,0 +1,37 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
+! Structure constructors with bad pointer targets
+module m
+  real, target, save :: x
+  type t
+    real, pointer :: rp => x
+    procedure(f), pointer, nopass :: pp => f
+  end type
+ contains
+  real function f()
+    f = 0.
+  end
+  subroutine test(da, dp)
+    real, target :: y, da
+    procedure(f) dp
+    procedure(f), pointer :: lpp
+    external ext
+    type(t) :: a1 = t() ! ok
+    type(t) :: a2 = t(rp=x) ! ok
+    type(t) :: a3 = t(pp=f) ! ok
+    type(t) :: a4 = t(pp=ext) ! ok
+    !ERROR: Must be a constant value
+    type(t) :: a5 = t(rp=y)
+    !ERROR: Must be a constant value
+    type(t) :: a6 = t(rp=da)
+    !ERROR: Must be a constant value
+    type(t) :: a7 = t(pp=lpp)
+    !ERROR: Must be a constant value
+    type(t) :: a8 = t(pp=internal)
+    !ERROR: Must be a constant value
+    type(t) :: a9 = t(pp=dp)
+   contains
+    real function internal()
+      internal = 666.
+    end
+  end
+end
-- 
GitLab


From b06913103fe7d3abeb474369eb69e7dfab7f8893 Mon Sep 17 00:00:00 2001
From: Zahira Ammarguellat <zahira.ammarguellat@intel.com>
Date: Tue, 26 Mar 2024 09:43:38 -0700
Subject: [PATCH 303/404] Fix failure in buildbot (#86661)

Sanitizer failure in
https://lab.llvm.org/buildbot/#/builders/19/builds/25788
---
 clang/lib/AST/TypePrinter.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index d9504f9dcb38..d0ba6066fa01 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -2303,10 +2303,10 @@ printTo(raw_ostream &OS, ArrayRef<TA> Args, const PrintingPolicy &Policy,
     } else {
       if (!FirstArg)
         OS << Comma;
-        // Tries to print the argument with location info if exists.
-        printArgument(Arg, Policy, ArgOS,
-                      TemplateParameterList::shouldIncludeTypeForArgument(
-                          Policy, TPL, ParmIndex));
+      // Tries to print the argument with location info if exists.
+      printArgument(Arg, Policy, ArgOS,
+                    TemplateParameterList::shouldIncludeTypeForArgument(
+                        Policy, TPL, ParmIndex));
     }
     StringRef ArgString = ArgOS.str();
 
-- 
GitLab


From 5cf1e2e2ec7da58403182265f2a7d2d5c1c87854 Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com>
Date: Tue, 26 Mar 2024 12:46:11 -0400
Subject: [PATCH 304/404] [DXIL] Implement log intrinsic Lowering (#86569)

Completes #86192
`DXIL.td` - add log2 to dxilop lowering
`DXILIntrinsicExpansion.cpp` - add log and log10 to log2 expansions
---
 llvm/lib/Target/DirectX/DXIL.td               |  3 ++
 .../Target/DirectX/DXILIntrinsicExpansion.cpp | 36 +++++++++++++++++--
 llvm/test/CodeGen/DirectX/log-vec.ll          | 30 ++++++++++++++++
 llvm/test/CodeGen/DirectX/log.ll              | 25 +++++++++++++
 llvm/test/CodeGen/DirectX/log10.ll            | 25 +++++++++++++
 llvm/test/CodeGen/DirectX/log2.ll             | 20 +++++++++++
 llvm/test/CodeGen/DirectX/log2_error.ll       | 10 ++++++
 7 files changed, 147 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/DirectX/log-vec.ll
 create mode 100644 llvm/test/CodeGen/DirectX/log.ll
 create mode 100644 llvm/test/CodeGen/DirectX/log10.ll
 create mode 100644 llvm/test/CodeGen/DirectX/log2.ll
 create mode 100644 llvm/test/CodeGen/DirectX/log2_error.ll

diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index b4bc9eda4f50..1fd6f3ed044e 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -274,6 +274,9 @@ def Frac : DXILOpMapping<22, unary, int_dx_frac,
                          "Returns a fraction from 0 to 1 that represents the "
                          "decimal part of the input.",
                          [llvm_halforfloat_ty, LLVMMatchType<0>]>;
+def Log2 : DXILOpMapping<23, unary, int_log2,
+                         "Returns the base-2 logarithm of the specified value.",
+                         [llvm_halforfloat_ty, LLVMMatchType<0>]>;
 def Sqrt : DXILOpMapping<24, unary, int_sqrt,
                          "Returns the square root of the specified floating-point"
                          "value, per component.",
diff --git a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
index b46564702c7a..3cc375edabde 100644
--- a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
+++ b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
@@ -35,6 +35,8 @@ static bool isIntrinsicExpansion(Function &F) {
   switch (F.getIntrinsicID()) {
   case Intrinsic::abs:
   case Intrinsic::exp:
+  case Intrinsic::log:
+  case Intrinsic::log10:
   case Intrinsic::dx_any:
   case Intrinsic::dx_clamp:
   case Intrinsic::dx_uclamp:
@@ -108,8 +110,8 @@ static bool expandExpIntrinsic(CallInst *Orig) {
       Ty->isVectorTy() ? ConstantVector::getSplat(
                              ElementCount::getFixed(
                                  cast<FixedVectorType>(Ty)->getNumElements()),
-                             ConstantFP::get(EltTy, numbers::log2e))
-                       : ConstantFP::get(EltTy, numbers::log2e);
+                             ConstantFP::get(EltTy, numbers::log2ef))
+                       : ConstantFP::get(EltTy, numbers::log2ef);
   Value *NewX = Builder.CreateFMul(Log2eConst, X);
   auto *Exp2Call =
       Builder.CreateIntrinsic(Ty, Intrinsic::exp2, {NewX}, nullptr, "dx.exp2");
@@ -169,6 +171,32 @@ static bool expandLerpIntrinsic(CallInst *Orig) {
   return true;
 }
 
+static bool expandLogIntrinsic(CallInst *Orig,
+                               float LogConstVal = numbers::ln2f) {
+  Value *X = Orig->getOperand(0);
+  IRBuilder<> Builder(Orig->getParent());
+  Builder.SetInsertPoint(Orig);
+  Type *Ty = X->getType();
+  Type *EltTy = Ty->getScalarType();
+  Constant *Ln2Const =
+      Ty->isVectorTy() ? ConstantVector::getSplat(
+                             ElementCount::getFixed(
+                                 cast<FixedVectorType>(Ty)->getNumElements()),
+                             ConstantFP::get(EltTy, LogConstVal))
+                       : ConstantFP::get(EltTy, LogConstVal);
+  auto *Log2Call =
+      Builder.CreateIntrinsic(Ty, Intrinsic::log2, {X}, nullptr, "elt.log2");
+  Log2Call->setTailCall(Orig->isTailCall());
+  Log2Call->setAttributes(Orig->getAttributes());
+  auto *Result = Builder.CreateFMul(Ln2Const, Log2Call);
+  Orig->replaceAllUsesWith(Result);
+  Orig->eraseFromParent();
+  return true;
+}
+static bool expandLog10Intrinsic(CallInst *Orig) {
+  return expandLogIntrinsic(Orig, numbers::ln2f / numbers::ln10f);
+}
+
 static bool expandRcpIntrinsic(CallInst *Orig) {
   Value *X = Orig->getOperand(0);
   IRBuilder<> Builder(Orig->getParent());
@@ -238,6 +266,10 @@ static bool expandIntrinsic(Function &F, CallInst *Orig) {
     return expandAbs(Orig);
   case Intrinsic::exp:
     return expandExpIntrinsic(Orig);
+  case Intrinsic::log:
+    return expandLogIntrinsic(Orig);
+  case Intrinsic::log10:
+    return expandLog10Intrinsic(Orig);
   case Intrinsic::dx_any:
     return expandAnyIntrinsic(Orig);
   case Intrinsic::dx_uclamp:
diff --git a/llvm/test/CodeGen/DirectX/log-vec.ll b/llvm/test/CodeGen/DirectX/log-vec.ll
new file mode 100644
index 000000000000..4768fdd94b02
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/log-vec.ll
@@ -0,0 +1,30 @@
+; RUN: opt -S  -dxil-intrinsic-expansion  < %s | FileCheck %s
+
+; Make sure dxil operation function calls for log are generated for float and half.
+
+; CHECK-LABEL: log_float4
+; CHECK: call <4 x float> @llvm.log2.v4f32(<4 x float>  %{{.*}})
+; CHECK: fmul <4 x float> <float 0x3FE62E4300000000, float 0x3FE62E4300000000, float 0x3FE62E4300000000, float 0x3FE62E4300000000>,  %{{.*}}
+define noundef <4 x float> @log_float4(<4 x float> noundef %p0) {
+entry:
+  %p0.addr = alloca <4 x float>, align 16
+  store <4 x float> %p0, ptr %p0.addr, align 16
+  %0 = load <4 x float>, ptr %p0.addr, align 16
+  %elt.log = call <4 x float> @llvm.log.v4f32(<4 x float> %0)
+  ret <4 x float> %elt.log
+}
+
+; CHECK-LABEL: log10_float4
+; CHECK: call <4 x float> @llvm.log2.v4f32(<4 x float>  %{{.*}})
+; CHECK: fmul <4 x float> <float 0x3FD3441340000000, float 0x3FD3441340000000, float 0x3FD3441340000000, float 0x3FD3441340000000>,  %{{.*}}
+define noundef <4 x float> @log10_float4(<4 x float> noundef %p0) {
+entry:
+  %p0.addr = alloca <4 x float>, align 16
+  store <4 x float> %p0, ptr %p0.addr, align 16
+  %0 = load <4 x float>, ptr %p0.addr, align 16
+  %elt.log10 = call <4 x float> @llvm.log10.v4f32(<4 x float> %0)
+  ret <4 x float> %elt.log10
+}
+
+declare <4 x float> @llvm.log.v4f32(<4 x float>)
+declare <4 x float> @llvm.log10.v4f32(<4 x float>)
diff --git a/llvm/test/CodeGen/DirectX/log.ll b/llvm/test/CodeGen/DirectX/log.ll
new file mode 100644
index 000000000000..172c3bfed3b7
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/log.ll
@@ -0,0 +1,25 @@
+; RUN: opt -S  -dxil-intrinsic-expansion  < %s | FileCheck %s --check-prefixes=CHECK,EXPCHECK
+; RUN: opt -S  -dxil-op-lower  < %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK
+
+; Make sure dxil operation function calls for log are generated.
+
+define noundef float @log_float(float noundef %a) #0 {
+entry:
+; DOPCHECK: call float @dx.op.unary.f32(i32 23, float %{{.*}})
+; EXPCHECK: call float @llvm.log2.f32(float %a)
+; CHECK: fmul float 0x3FE62E4300000000, %{{.*}}
+  %elt.log = call float @llvm.log.f32(float %a)
+  ret float %elt.log
+}
+
+define noundef half @log_half(half noundef %a) #0 {
+entry:
+; DOPCHECK: call half @dx.op.unary.f16(i32 23, half %{{.*}})
+; EXPCHECK: call half @llvm.log2.f16(half %a)
+; CHECK: fmul half 0xH398C, %{{.*}}
+  %elt.log = call half @llvm.log.f16(half %a)
+  ret half %elt.log
+}
+
+declare half @llvm.log.f16(half)
+declare float @llvm.log.f32(float)
diff --git a/llvm/test/CodeGen/DirectX/log10.ll b/llvm/test/CodeGen/DirectX/log10.ll
new file mode 100644
index 000000000000..d4f827a0d1af
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/log10.ll
@@ -0,0 +1,25 @@
+; RUN: opt -S  -dxil-intrinsic-expansion  < %s | FileCheck %s --check-prefixes=CHECK,EXPCHECK
+; RUN: opt -S  -dxil-op-lower  < %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK
+
+; Make sure dxil operation function calls for log10 are generated.
+
+define noundef float @log10_float(float noundef %a) #0 {
+entry:
+; DOPCHECK: call float @dx.op.unary.f32(i32 23, float %{{.*}})
+; EXPCHECK: call float @llvm.log2.f32(float %a)
+; CHECK: fmul float 0x3FD3441340000000, %{{.*}}
+  %elt.log10 = call float @llvm.log10.f32(float %a)
+  ret float %elt.log10
+}
+
+define noundef half @log10_half(half noundef %a) #0 {
+entry:
+; DOPCHECK: call half @dx.op.unary.f16(i32 23, half %{{.*}})
+; EXPCHECK: call half @llvm.log2.f16(half %a)
+; CHECK: fmul half 0xH34D1, %{{.*}}
+  %elt.log10 = call half @llvm.log10.f16(half %a)
+  ret half %elt.log10
+}
+
+declare half @llvm.log10.f16(half)
+declare float @llvm.log10.f32(float)
diff --git a/llvm/test/CodeGen/DirectX/log2.ll b/llvm/test/CodeGen/DirectX/log2.ll
new file mode 100644
index 000000000000..2164d4db9396
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/log2.ll
@@ -0,0 +1,20 @@
+; RUN: opt -S -dxil-op-lower < %s | FileCheck %s
+
+; Make sure dxil operation function calls for log2 are generated for float and half.
+
+define noundef float @log2_float(float noundef %a) #0 {
+entry:
+; CHECK:call float @dx.op.unary.f32(i32 23, float %{{.*}})
+  %elt.log2 = call float @llvm.log2.f32(float %a)
+  ret float %elt.log2
+}
+
+define noundef half @log2_half(half noundef %a) #0 {
+entry:
+; CHECK:call half @dx.op.unary.f16(i32 23, half %{{.*}})
+  %elt.log2 = call half @llvm.log2.f16(half %a)
+  ret half %elt.log2
+}
+
+declare half @llvm.log2.f16(half)
+declare float @llvm.log2.f32(float)
diff --git a/llvm/test/CodeGen/DirectX/log2_error.ll b/llvm/test/CodeGen/DirectX/log2_error.ll
new file mode 100644
index 000000000000..a26f6e8c3117
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/log2_error.ll
@@ -0,0 +1,10 @@
+; RUN: not opt -S -dxil-op-lower %s 2>&1 | FileCheck %s
+
+; DXIL operation log2 does not support double overload type
+; CHECK: LLVM ERROR: Invalid Overload Type
+
+define noundef double @log2_double(double noundef %a) {
+entry:
+  %elt.log2 = call double @llvm.log2.f64(double %a)
+  ret double %elt.log2
+}
-- 
GitLab


From 7860f970666f46184ad740db48a69882d62e64fc Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin@nvidia.com>
Date: Tue, 26 Mar 2024 09:47:10 -0700
Subject: [PATCH 305/404] [flang][runtime] Use cuda::std::variant in the CUDA
 build. (#86615)

Added `FLANG_LIBCUDACXX_PATH` CMake variable to specify
installation of header-only libcudacxx library.
If it is specified, the `<cuda/std/variant>` is used to provide
implementation of `std::variant`.
---
 flang/include/flang/Common/idioms.h   |  2 +-
 flang/include/flang/Common/template.h |  2 +-
 flang/include/flang/Common/unwrap.h   |  2 +-
 flang/include/flang/Common/variant.h  | 30 +++++++++++++++++++++++++++
 flang/include/flang/Common/visit.h    |  2 +-
 flang/runtime/CMakeLists.txt          |  8 +++++++
 flang/runtime/io-stmt.h               |  2 +-
 flang/runtime/unit.cpp                |  6 ------
 flang/runtime/unit.h                  |  8 +------
 9 files changed, 44 insertions(+), 18 deletions(-)
 create mode 100644 flang/include/flang/Common/variant.h

diff --git a/flang/include/flang/Common/idioms.h b/flang/include/flang/Common/idioms.h
index 231fbd8601a2..99f383ec75b9 100644
--- a/flang/include/flang/Common/idioms.h
+++ b/flang/include/flang/Common/idioms.h
@@ -24,6 +24,7 @@
 #endif
 
 #include "enum-class.h"
+#include "variant.h"
 #include "visit.h"
 #include <array>
 #include <functional>
@@ -33,7 +34,6 @@
 #include <string>
 #include <tuple>
 #include <type_traits>
-#include <variant>
 
 #if __GNUC__ == 7
 // Avoid a deduction bug in GNU 7.x headers by forcing the answer.
diff --git a/flang/include/flang/Common/template.h b/flang/include/flang/Common/template.h
index 2ab3b8bce1df..51d09fb42ce3 100644
--- a/flang/include/flang/Common/template.h
+++ b/flang/include/flang/Common/template.h
@@ -9,12 +9,12 @@
 #ifndef FORTRAN_COMMON_TEMPLATE_H_
 #define FORTRAN_COMMON_TEMPLATE_H_
 
+#include "variant.h"
 #include "flang/Common/idioms.h"
 #include <functional>
 #include <optional>
 #include <tuple>
 #include <type_traits>
-#include <variant>
 #include <vector>
 
 // Utility templates for metaprogramming and for composing the
diff --git a/flang/include/flang/Common/unwrap.h b/flang/include/flang/Common/unwrap.h
index edb343d77b53..84582174e4b3 100644
--- a/flang/include/flang/Common/unwrap.h
+++ b/flang/include/flang/Common/unwrap.h
@@ -12,11 +12,11 @@
 #include "indirection.h"
 #include "reference-counted.h"
 #include "reference.h"
+#include "variant.h"
 #include "visit.h"
 #include <memory>
 #include <optional>
 #include <type_traits>
-#include <variant>
 
 // Given a nest of variants, optionals, &/or pointers, Unwrap<>() isolates
 // a packaged value of a specific type if it is present and returns a pointer
diff --git a/flang/include/flang/Common/variant.h b/flang/include/flang/Common/variant.h
new file mode 100644
index 000000000000..1af85876afac
--- /dev/null
+++ b/flang/include/flang/Common/variant.h
@@ -0,0 +1,30 @@
+//===-- include/flang/Common/variant.h --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// A single way to expose C++ variant class in files that can be used
+// in F18 runtime build. With inclusion of this file std::variant
+// and the related names become available, though, they may correspond
+// to alternative definitions (e.g. from cuda::std namespace).
+
+#ifndef FORTRAN_COMMON_VARIANT_H
+#define FORTRAN_COMMON_VARIANT_H
+
+#if RT_USE_LIBCUDACXX
+#include <cuda/std/variant>
+namespace std {
+using cuda::std::get;
+using cuda::std::monostate;
+using cuda::std::variant;
+using cuda::std::variant_size_v;
+using cuda::std::visit;
+} // namespace std
+#else // !RT_USE_LIBCUDACXX
+#include <variant>
+#endif // !RT_USE_LIBCUDACXX
+
+#endif // FORTRAN_COMMON_VARIANT_H
diff --git a/flang/include/flang/Common/visit.h b/flang/include/flang/Common/visit.h
index 54f8ca70b313..4d0897301e01 100644
--- a/flang/include/flang/Common/visit.h
+++ b/flang/include/flang/Common/visit.h
@@ -21,9 +21,9 @@
 #ifndef FORTRAN_COMMON_VISIT_H_
 #define FORTRAN_COMMON_VISIT_H_
 
+#include "variant.h"
 #include "flang/Common/api-attrs.h"
 #include <type_traits>
-#include <variant>
 
 namespace Fortran::common {
 namespace log2visit {
diff --git a/flang/runtime/CMakeLists.txt b/flang/runtime/CMakeLists.txt
index facd14432b3e..c0e4cff698e3 100644
--- a/flang/runtime/CMakeLists.txt
+++ b/flang/runtime/CMakeLists.txt
@@ -174,6 +174,7 @@ set(sources
 option(FLANG_EXPERIMENTAL_CUDA_RUNTIME
   "Compile Fortran runtime as CUDA sources (experimental)" OFF
   )
+set(FLANG_LIBCUDACXX_PATH "" CACHE PATH "Path to libcu++ package installation")
 
 # List of files that are buildable for all devices.
 set(supported_files
@@ -260,6 +261,13 @@ if (FLANG_EXPERIMENTAL_CUDA_RUNTIME)
   set_source_files_properties(${supported_files} PROPERTIES COMPILE_OPTIONS
     "${CUDA_COMPILE_OPTIONS}"
     )
+
+  if (EXISTS "${FLANG_LIBCUDACXX_PATH}/include")
+    # When using libcudacxx headers files, we have to use them
+    # for all files of F18 runtime.
+    include_directories(AFTER ${FLANG_LIBCUDACXX_PATH}/include)
+    add_compile_definitions(RT_USE_LIBCUDACXX=1)
+  endif()
 endif()
 
 set(FLANG_EXPERIMENTAL_OMP_OFFLOAD_BUILD "off" CACHE STRING
diff --git a/flang/runtime/io-stmt.h b/flang/runtime/io-stmt.h
index e0dafa9c763c..8b5752311de5 100644
--- a/flang/runtime/io-stmt.h
+++ b/flang/runtime/io-stmt.h
@@ -21,9 +21,9 @@
 #include "flang/Common/visit.h"
 #include "flang/Runtime/descriptor.h"
 #include "flang/Runtime/io-api.h"
+#include <flang/Common/variant.h>
 #include <functional>
 #include <type_traits>
-#include <variant>
 
 namespace Fortran::runtime::io {
 
diff --git a/flang/runtime/unit.cpp b/flang/runtime/unit.cpp
index b5aa307eade8..6c648d3bd834 100644
--- a/flang/runtime/unit.cpp
+++ b/flang/runtime/unit.cpp
@@ -482,10 +482,7 @@ bool ExternalFileUnit::SetDirectRec(
 
 void ExternalFileUnit::EndIoStatement() {
   io_.reset();
-  RT_DIAG_PUSH
-  RT_DIAG_DISABLE_CALL_HOST_FROM_DEVICE_WARN
   u_.emplace<std::monostate>();
-  RT_DIAG_POP
   lock_.Drop();
 }
 
@@ -752,10 +749,7 @@ std::int32_t ExternalFileUnit::ReadHeaderOrFooter(std::int64_t frameOffset) {
 
 void ChildIo::EndIoStatement() {
   io_.reset();
-  RT_DIAG_PUSH
-  RT_DIAG_DISABLE_CALL_HOST_FROM_DEVICE_WARN
   u_.emplace<std::monostate>();
-  RT_DIAG_POP
 }
 
 Iostat ChildIo::CheckFormattingAndDirection(
diff --git a/flang/runtime/unit.h b/flang/runtime/unit.h
index 8b7db5cbc90b..a6ee5971a165 100644
--- a/flang/runtime/unit.h
+++ b/flang/runtime/unit.h
@@ -25,7 +25,7 @@
 #include "flang/Runtime/memory.h"
 #include <cstdlib>
 #include <cstring>
-#include <variant>
+#include <flang/Common/variant.h>
 
 namespace Fortran::runtime::io {
 
@@ -152,10 +152,7 @@ public:
 #else
     lock_.Take();
 #endif
-    RT_DIAG_PUSH
-    RT_DIAG_DISABLE_CALL_HOST_FROM_DEVICE_WARN
     A &state{u_.emplace<A>(std::forward<X>(xs)...)};
-    RT_DIAG_POP
     if constexpr (!std::is_same_v<A, OpenStatementState>) {
       state.mutableModes() = ConnectionState::modes;
     }
@@ -265,10 +262,7 @@ public:
 
   template <typename A, typename... X>
   RT_API_ATTRS IoStatementState &BeginIoStatement(X &&...xs) {
-    RT_DIAG_PUSH
-    RT_DIAG_DISABLE_CALL_HOST_FROM_DEVICE_WARN
     A &state{u_.emplace<A>(std::forward<X>(xs)...)};
-    RT_DIAG_POP
     io_.emplace(state);
     return *io_;
   }
-- 
GitLab


From e75989e93063d1ac2626c3478c28ca364e04ef28 Mon Sep 17 00:00:00 2001
From: smanna12 <soumi.manna@intel.com>
Date: Tue, 26 Mar 2024 11:48:37 -0500
Subject: [PATCH 306/404] [NFC][Clang] Fix potential dereferencing of nullptr
 (#85944)

This patch replaces getAs<> with castAs<> to resolve potential static
analyzer bugs for

1. Dereferencing a pointer issue with nullptr FPT when calling
ResolveExceptionSpec() in
    checkEscapingByref(clang::VarDecl *, clang::Sema &).

3. Dereferencing a pointer issue with nullptr ElementTy->getAs() when
calling getElementType() in
clang::Sema::SemaBuiltinFPClassification(clang::CallExpr *, unsigned
int).

4. Dereferencing a pointer issue with nullptr ConvType->getAs() when
calling getKeyword() in
clang::Sema::ActOnConversionDeclarator(clang::CXXConversionDecl *).
---
 clang/lib/Sema/Sema.cpp         | 2 +-
 clang/lib/Sema/SemaChecking.cpp | 2 +-
 clang/lib/Sema/SemaDeclCXX.cpp  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index b4a47898cd17..b55f433a8be7 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -2207,7 +2207,7 @@ static void checkEscapingByref(VarDecl *VD, Sema &S) {
   // block copy/destroy functions. Resolve it here.
   if (const CXXRecordDecl *RD = T->getAsCXXRecordDecl())
     if (CXXDestructorDecl *DD = RD->getDestructor()) {
-      auto *FPT = DD->getType()->getAs<FunctionProtoType>();
+      auto *FPT = DD->getType()->castAs<FunctionProtoType>();
       S.ResolveExceptionSpec(Loc, FPT);
     }
 }
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index c725ec0ef055..b4e87b618032 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -9709,7 +9709,7 @@ bool Sema::SemaBuiltinFPClassification(CallExpr *TheCall, unsigned NumArgs,
   // vector argument can be supported in all of them.
   if (ElementTy->isVectorType() && IsFPClass) {
     VectorResultTy = GetSignedVectorType(ElementTy);
-    ElementTy = ElementTy->getAs<VectorType>()->getElementType();
+    ElementTy = ElementTy->castAs<VectorType>()->getElementType();
   }
 
   // This operation requires a non-_Complex floating-point number.
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 7070ea00cff2..c63f2bdc5f83 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -11353,7 +11353,7 @@ Decl *Sema::ActOnConversionDeclarator(CXXConversionDecl *Conversion) {
     if (ConvType->isUndeducedAutoType()) {
       Diag(Conversion->getTypeSpecStartLoc(), diag::err_auto_not_allowed)
           << getReturnTypeLoc(Conversion).getSourceRange()
-          << llvm::to_underlying(ConvType->getAs<AutoType>()->getKeyword())
+          << llvm::to_underlying(ConvType->castAs<AutoType>()->getKeyword())
           << /* in declaration of conversion function template= */ 24;
     }
 
-- 
GitLab


From 8f01ecaeb8e537511718c4df123fb92633d9f73d Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Tue, 26 Mar 2024 09:50:37 -0700
Subject: [PATCH 307/404] [flang] Special-case handling of INTRINSIC in
 type-decl-stmt (#86518)

Fortran allows the INTRINSIC attribute to be specified with a distinct
attribute statement, and also as part of the attribute list of a
type-declaration-stmt. This is an odd case (especially as the declared
type is mandated to be ignored if it doesn't match the type of the
intrinsic function) that can lead to odd error messages and crashes,
since the rest of name resolution expects that intrinsics with explicit
declarations will have been declared with INTRINSIC attribute
statements. Resolve by handling an "inline" INTRINSIC attribute as a
special case while processing a type-declaration-stmt, so that

  real, intrinsic :: acos, asin, atan

is processed exactly as if it had been

  intrinsic acos, asin, atan; real acos, asin, atan

Fixes https://github.com/llvm/llvm-project/issues/86382.
---
 flang/include/flang/Parser/tools.h    |  1 +
 flang/lib/Parser/tools.cpp            |  4 ++
 flang/lib/Semantics/resolve-names.cpp | 88 ++++++++++++++++-----------
 flang/test/Semantics/init01.f90       |  2 +
 flang/test/Semantics/resolve81.f90    |  1 +
 5 files changed, 61 insertions(+), 35 deletions(-)

diff --git a/flang/include/flang/Parser/tools.h b/flang/include/flang/Parser/tools.h
index 1e347fab6461..f1ead11734fa 100644
--- a/flang/include/flang/Parser/tools.h
+++ b/flang/include/flang/Parser/tools.h
@@ -40,6 +40,7 @@ const Name &GetFirstName(const ProcedureDesignator &);
 const Name &GetFirstName(const Call &);
 const Name &GetFirstName(const FunctionReference &);
 const Name &GetFirstName(const Variable &);
+const Name &GetFirstName(const EntityDecl &);
 
 // When a parse tree node is an instance of a specific type wrapped in
 // layers of packaging, return a pointer to that object.
diff --git a/flang/lib/Parser/tools.cpp b/flang/lib/Parser/tools.cpp
index 899fb0f069a9..6e5f1ed2fc66 100644
--- a/flang/lib/Parser/tools.cpp
+++ b/flang/lib/Parser/tools.cpp
@@ -123,6 +123,10 @@ const Name &GetFirstName(const Variable &x) {
       x.u);
 }
 
+const Name &GetFirstName(const EntityDecl &x) {
+  return std::get<ObjectName>(x.t);
+}
+
 const CoindexedNamedObject *GetCoindexedNamedObject(const DataRef &base) {
   return common::visit(
       common::visitors{
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 73450c49d5fe..2e88a2daff2c 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -955,7 +955,7 @@ public:
   void Post(const parser::TargetStmt &) { objectDeclAttr_ = std::nullopt; }
   void Post(const parser::DimensionStmt::Declaration &);
   void Post(const parser::CodimensionDecl &);
-  bool Pre(const parser::TypeDeclarationStmt &) { return BeginDecl(); }
+  bool Pre(const parser::TypeDeclarationStmt &);
   void Post(const parser::TypeDeclarationStmt &);
   void Post(const parser::IntegerTypeSpec &);
   void Post(const parser::IntrinsicTypeSpec::Real &);
@@ -1202,6 +1202,7 @@ private:
   bool MustBeScalar(const Symbol &symbol) const {
     return mustBeScalar_.find(symbol) != mustBeScalar_.end();
   }
+  void DeclareIntrinsic(const parser::Name &);
 };
 
 // Resolve construct entities and statement entities.
@@ -4550,6 +4551,20 @@ void DeclarationVisitor::CheckAccessibility(
   }
 }
 
+bool DeclarationVisitor::Pre(const parser::TypeDeclarationStmt &x) {
+  BeginDecl();
+  // If INTRINSIC appears as an attr-spec, handle it now as if the
+  // names had appeared on an INTRINSIC attribute statement beforehand.
+  for (const auto &attr : std::get<std::list<parser::AttrSpec>>(x.t)) {
+    if (std::holds_alternative<parser::Intrinsic>(attr.u)) {
+      for (const auto &decl : std::get<std::list<parser::EntityDecl>>(x.t)) {
+        DeclareIntrinsic(parser::GetFirstName(decl));
+      }
+      break;
+    }
+  }
+  return true;
+}
 void DeclarationVisitor::Post(const parser::TypeDeclarationStmt &) {
   EndDecl();
 }
@@ -4571,6 +4586,7 @@ bool DeclarationVisitor::Pre(const parser::Initialization &) {
 void DeclarationVisitor::Post(const parser::EntityDecl &x) {
   const auto &name{std::get<parser::ObjectName>(x.t)};
   Attrs attrs{attrs_ ? HandleSaveName(name.source, *attrs_) : Attrs{}};
+  attrs.set(Attr::INTRINSIC, false); // dealt with in Pre(TypeDeclarationStmt)
   Symbol &symbol{DeclareUnknownEntity(name, attrs)};
   symbol.ReplaceName(name.source);
   SetCUDADataAttr(name.source, symbol, cudaDataAttr());
@@ -4811,45 +4827,47 @@ bool DeclarationVisitor::Pre(const parser::IntentStmt &x) {
       HandleAttributeStmt(IntentSpecToAttr(intentSpec), names);
 }
 bool DeclarationVisitor::Pre(const parser::IntrinsicStmt &x) {
-  HandleAttributeStmt(Attr::INTRINSIC, x.v);
   for (const auto &name : x.v) {
-    if (!IsIntrinsic(name.source, std::nullopt)) {
-      Say(name.source, "'%s' is not a known intrinsic procedure"_err_en_US);
-    }
-    auto &symbol{DEREF(FindSymbol(name))};
-    if (symbol.has<GenericDetails>()) {
-      // Generic interface is extending intrinsic; ok
-    } else if (!ConvertToProcEntity(symbol)) {
-      SayWithDecl(
-          name, symbol, "INTRINSIC attribute not allowed on '%s'"_err_en_US);
-    } else if (symbol.attrs().test(Attr::EXTERNAL)) { // C840
+    DeclareIntrinsic(name);
+  }
+  return false;
+}
+void DeclarationVisitor::DeclareIntrinsic(const parser::Name &name) {
+  HandleAttributeStmt(Attr::INTRINSIC, name);
+  if (!IsIntrinsic(name.source, std::nullopt)) {
+    Say(name.source, "'%s' is not a known intrinsic procedure"_err_en_US);
+  }
+  auto &symbol{DEREF(FindSymbol(name))};
+  if (symbol.has<GenericDetails>()) {
+    // Generic interface is extending intrinsic; ok
+  } else if (!ConvertToProcEntity(symbol)) {
+    SayWithDecl(
+        name, symbol, "INTRINSIC attribute not allowed on '%s'"_err_en_US);
+  } else if (symbol.attrs().test(Attr::EXTERNAL)) { // C840
+    Say(symbol.name(),
+        "Symbol '%s' cannot have both EXTERNAL and INTRINSIC attributes"_err_en_US,
+        symbol.name());
+  } else {
+    if (symbol.GetType()) {
+      // These warnings are worded so that they should make sense in either
+      // order.
       Say(symbol.name(),
-          "Symbol '%s' cannot have both EXTERNAL and INTRINSIC attributes"_err_en_US,
-          symbol.name());
-    } else {
-      if (symbol.GetType()) {
-        // These warnings are worded so that they should make sense in either
-        // order.
-        Say(symbol.name(),
-            "Explicit type declaration ignored for intrinsic function '%s'"_warn_en_US,
-            symbol.name())
-            .Attach(name.source,
-                "INTRINSIC statement for explicitly-typed '%s'"_en_US,
-                name.source);
-      }
-      if (!symbol.test(Symbol::Flag::Function) &&
-          !symbol.test(Symbol::Flag::Subroutine)) {
-        if (context().intrinsics().IsIntrinsicFunction(
-                name.source.ToString())) {
-          symbol.set(Symbol::Flag::Function);
-        } else if (context().intrinsics().IsIntrinsicSubroutine(
-                       name.source.ToString())) {
-          symbol.set(Symbol::Flag::Subroutine);
-        }
+          "Explicit type declaration ignored for intrinsic function '%s'"_warn_en_US,
+          symbol.name())
+          .Attach(name.source,
+              "INTRINSIC statement for explicitly-typed '%s'"_en_US,
+              name.source);
+    }
+    if (!symbol.test(Symbol::Flag::Function) &&
+        !symbol.test(Symbol::Flag::Subroutine)) {
+      if (context().intrinsics().IsIntrinsicFunction(name.source.ToString())) {
+        symbol.set(Symbol::Flag::Function);
+      } else if (context().intrinsics().IsIntrinsicSubroutine(
+                     name.source.ToString())) {
+        symbol.set(Symbol::Flag::Subroutine);
       }
     }
   }
-  return false;
 }
 bool DeclarationVisitor::Pre(const parser::OptionalStmt &x) {
   return CheckNotInBlock("OPTIONAL") && // C1107
diff --git a/flang/test/Semantics/init01.f90 b/flang/test/Semantics/init01.f90
index f85feef097cd..65d524b16a23 100644
--- a/flang/test/Semantics/init01.f90
+++ b/flang/test/Semantics/init01.f90
@@ -158,8 +158,10 @@ subroutine notObjects
   real, external :: x1 = 1.
 !ERROR: 'x2' is not a pointer but is initialized like one
   real, external :: x2 => sin
+!ERROR: 'x3' is not a known intrinsic procedure
 !ERROR: 'x3' is not an object that can be initialized
   real, intrinsic :: x3 = 1.
+!ERROR: 'x4' is not a known intrinsic procedure
 !ERROR: 'x4' is not a pointer but is initialized like one
   real, intrinsic :: x4 => cos
 end subroutine
diff --git a/flang/test/Semantics/resolve81.f90 b/flang/test/Semantics/resolve81.f90
index 2a0b961d48e5..87901fd7d2ef 100644
--- a/flang/test/Semantics/resolve81.f90
+++ b/flang/test/Semantics/resolve81.f90
@@ -28,6 +28,7 @@ module m
   !WARNING: Attribute 'EXTERNAL' cannot be used more than once
   real, external, external :: externFunc
   !WARNING: Attribute 'INTRINSIC' cannot be used more than once
+  !ERROR: An interface name with BIND attribute must be specified if the BIND attribute is specified in a procedure declaration statement
   real, intrinsic, bind(c), intrinsic :: cos
   !WARNING: Attribute 'BIND(C)' cannot be used more than once
   integer, bind(c), volatile, bind(c) :: bindVar
-- 
GitLab


From 39c03a2fbcdd6f20e0457faa5e169f4948f9422b Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 25 Mar 2024 23:28:32 -0700
Subject: [PATCH 308/404] [RISCV] Remove unnecessary overrides of a defaulted
 template argument. NFC

---
 llvm/lib/Target/RISCV/RISCVInstrInfoV.td | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
index 469ceaf6418e..875f2e382d54 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
@@ -575,7 +575,7 @@ multiclass VALU_IV_X<string opcodestr, bits<6> funct6> {
 }
 
 multiclass VALU_IV_I<string opcodestr, bits<6> funct6> {
-  def I  : VALUVI<funct6, opcodestr # ".vi", simm5>,
+  def I  : VALUVI<funct6, opcodestr # ".vi">,
            SchedUnaryMC<"WriteVIALUI", "ReadVIALUV">;
 }
 
@@ -661,7 +661,7 @@ multiclass VALUNoVm_IV_V_X<string opcodestr, bits<6> funct6> {
 
 multiclass VALUNoVm_IV_V_X_I<string opcodestr, bits<6> funct6>
    : VALUNoVm_IV_V_X<opcodestr, funct6> {
-  def I : VALUVINoVm<funct6, opcodestr # ".vi", simm5>,
+  def I : VALUVINoVm<funct6, opcodestr # ".vi">,
           SchedUnaryMC<"WriteVICALUI", "ReadVICALUV", forceMasked=0>;
 }
 
@@ -900,7 +900,7 @@ multiclass VCMP_IV_X<string opcodestr, bits<6> funct6> {
 }
 
 multiclass VCMP_IV_I<string opcodestr, bits<6> funct6> {
-  def I  : VALUVI<funct6, opcodestr # ".vi", simm5>,
+  def I  : VALUVI<funct6, opcodestr # ".vi">,
            SchedUnaryMC<"WriteVICmpI", "ReadVICmpV">;
 }
 
@@ -947,7 +947,7 @@ multiclass VSALU_IV_V_X<string opcodestr, bits<6> funct6> {
 
 multiclass VSALU_IV_V_X_I<string opcodestr, bits<6> funct6>
     : VSALU_IV_V_X<opcodestr, funct6> {
-  def I  : VALUVI<funct6, opcodestr # ".vi", simm5>,
+  def I  : VALUVI<funct6, opcodestr # ".vi">,
            SchedUnaryMC<"WriteVSALUI", "ReadVSALUV">;
 }
 
-- 
GitLab


From bf4fc00bb73c73ccc7a78cd6d28287dd58c71e2c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Tue, 26 Mar 2024 16:58:41 +0000
Subject: [PATCH 309/404] [mlir][vector] Refactor linearize.mlir (#86648)

This patch refactors the `linearize.mlir` test - currently it contains
some duplication and can be tricky to follow.

Summary of changes:
  * reduce duplication by introducing a shared check prefix (`ALL`) and
    by introducing `-check-prefixes`,
  * make sure that every "check" line is directly above the
    corresponding line of input MLIR,
  * group check lines corresponding to a particular prefix together (so
    that it's easier to see the expected output for a particular
    prefix),
  * remove `CHECK` from prefix names (with multiple prefixes that's just
    noise that can be avoided) and use a bit more descriptive prefixes
    instead (`CHECK0` -> `BW-0`, where `BW` stands for bitwidth),
  * unify indentation,
  * `nonvec_result` -> `test_tensor_no_linearize` (for consistency with
    `test_index_no_linearize`).

NOTE: This change only updates the format of the "CHECK" lines and
doesn't affect what's being tested.

This change is intended as preparation for adding support for scalable
vectors to `LinearizeConstant` and `LinearizeVectorizable` - i.e.
patterns that `linearlize.mlir` is meant to test.
---
 mlir/test/Dialect/Vector/linearize.mlir | 128 +++++++++++-------------
 1 file changed, 61 insertions(+), 67 deletions(-)

diff --git a/mlir/test/Dialect/Vector/linearize.mlir b/mlir/test/Dialect/Vector/linearize.mlir
index e865fcb7a419..1b225c7a97d2 100644
--- a/mlir/test/Dialect/Vector/linearize.mlir
+++ b/mlir/test/Dialect/Vector/linearize.mlir
@@ -1,92 +1,87 @@
-// RUN: mlir-opt %s -split-input-file -test-vector-linearize | FileCheck %s
-// RUN: mlir-opt %s -split-input-file -test-vector-linearize=target-vector-bitwidth=128 | FileCheck %s --check-prefix=CHECK128
-// RUN: mlir-opt %s -split-input-file -test-vector-linearize=target-vector-bitwidth=0 | FileCheck %s --check-prefix=CHECK0
-
-// CHECK-LABEL: test_linearize
-// CHECK128-LABEL: test_linearize
-// CHECK0-LABEL: test_linearize
-//  CHECK-SAME: (%[[ORIG_ARG:.*]]: vector<2x2xf32>)
-//  CHECK128-SAME: (%[[ORIG_ARG:.*]]: vector<2x2xf32>)
-//       CHECK: %[[ARG:.*]] = vector.shape_cast %[[ORIG_ARG]] : vector<2x2xf32> to vector<4xf32>
-//       CHECK128: %[[ARG:.*]] = vector.shape_cast %[[ORIG_ARG]] : vector<2x2xf32> to vector<4xf32>
+// RUN: mlir-opt %s -split-input-file -test-vector-linearize | FileCheck %s --check-prefixes=ALL,DEFAULT
+// RUN: mlir-opt %s -split-input-file -test-vector-linearize=target-vector-bitwidth=128 | FileCheck %s --check-prefixes=ALL,BW-128
+// RUN: mlir-opt %s -split-input-file -test-vector-linearize=target-vector-bitwidth=0 | FileCheck %s --check-prefixes=ALL,BW-0
+
+// ALL-LABEL: test_linearize
+// ALL-SAME: (%[[ORIG_ARG:.*]]: vector<2x2xf32>)
 func.func @test_linearize(%arg0: vector<2x2xf32>) -> vector<2x2xf32> {
-//       CHECK: %[[C1:.*]] = arith.constant dense<[1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00]> : vector<4xf32>
-//       CHECK128: %[[C1:.*]] = arith.constant dense<[1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00]> : vector<4xf32>
-//       CHECK0: %[[C1:.*]] = arith.constant dense<{{.*}}> : vector<2x2xf32>
+  // DEFAULT: %[[ARG:.*]] = vector.shape_cast %[[ORIG_ARG]] : vector<2x2xf32> to vector<4xf32>
+  // DEFAULT: %[[CST:.*]] = arith.constant dense<[1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00]> : vector<4xf32>
+  // DEFAULT: %[[RES:.*]] = vector.shape_cast %[[CST]] : vector<4xf32> to vector<2x2xf32>
+
+  // BW-128: %[[ARG:.*]] = vector.shape_cast %[[ORIG_ARG]] : vector<2x2xf32> to vector<4xf32>
+  // BW-128: %[[CST:.*]] = arith.constant dense<[1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00]> : vector<4xf32>
+  // BW-128: %[[RES:.*]] = vector.shape_cast %[[CST]] : vector<4xf32> to vector<2x2xf32>
 
+  // BW-0: %[[RES:.*]] = arith.constant dense<{{.*}}> : vector<2x2xf32>
   %0 = arith.constant dense<[[1.0, 2.0], [3.0, 4.0]]> : vector<2x2xf32>
-//       CHECK: %[[RES:.*]] = vector.shape_cast %[[C1]] : vector<4xf32> to vector<2x2xf32>
-//       CHECK128: %[[RES:.*]] = vector.shape_cast %[[C1]] : vector<4xf32> to vector<2x2xf32>
-// Arith and math ops are handled in generic way, check some of them
-//       CHECK: %{{.*}} =  math.sin %[[ARG]] : vector<4xf32>
-//       CHECK128: %{{.*}} =  math.sin %[[ARG]] : vector<4xf32>
-//       CHECK0: %{{.*}} =  math.sin %{{.*}} : vector<2x2xf32>
+
+  // DEFAULT: %{{.*}} =  math.sin %[[ARG]] : vector<4xf32>
+  // BW-128: %{{.*}} =  math.sin %[[ARG]] : vector<4xf32>
+  // BW-0: %{{.*}} =  math.sin %{{.*}} : vector<2x2xf32>
   %1 = math.sin %arg0 : vector<2x2xf32>
-//       CHECK: %{{.*}} = arith.addf %[[ARG]], %[[C1]] : vector<4xf32>
-//       CHECK128: %{{.*}} = arith.addf %[[ARG]], %[[C1]] : vector<4xf32>
-//       CHECK0: %{{.*}} = arith.addf %{{.*}} : vector<2x2xf32>
 
+  // DEFAULT: %{{.*}} = arith.addf %[[ARG]], %[[CST]] : vector<4xf32>
+  // BW-128: %{{.*}} = arith.addf %[[ARG]], %[[CST]] : vector<4xf32>
+  // BW-0: %{{.*}} = arith.addf %{{.*}} : vector<2x2xf32>
   %2 = arith.addf %arg0, %0 :  vector<2x2xf32>
 
-//       CHECK: return %[[RES]] : vector<2x2xf32>
-//       CHECK128: return %[[RES]] : vector<2x2xf32>
+  // ALL: return %[[RES]] : vector<2x2xf32>
   return %0 : vector<2x2xf32>
 }
 
-// CHECK-LABEL: test_partial_linearize
-// CHECK128-LABEL: test_partial_linearize
-// CHECK0-LABEL: test_partial_linearize
-//  CHECK-SAME: (%[[ORIG_ARG:.*]]: vector<2x2xf32>, %[[ORIG_ARG2:.*]]: vector<4x4xf32>)
-//  CHECK128-SAME: (%[[ORIG_ARG:.*]]: vector<2x2xf32>, %[[ORIG_ARG2:.*]]: vector<4x4xf32>)
-//  CHECK0-SAME: (%[[ORIG_ARG:.*]]: vector<2x2xf32>, %[[ORIG_ARG2:.*]]: vector<4x4xf32>)
-//       CHECK: %[[ARG:.*]] = vector.shape_cast %[[ORIG_ARG]] : vector<2x2xf32> to vector<4xf32>
-//       CHECK128: %[[ARG:.*]] = vector.shape_cast %[[ORIG_ARG]] : vector<2x2xf32> to vector<4xf32>
-//       CHECK: %[[ARG2:.*]] = vector.shape_cast %[[ORIG_ARG2]] : vector<4x4xf32> to vector<16xf32>
+// -----
+
+// ALL-LABEL: test_partial_linearize
+// ALL-SAME: (%[[ORIG_ARG:.*]]: vector<2x2xf32>, %[[ORIG_ARG2:.*]]: vector<4x4xf32>)
 func.func @test_partial_linearize(%arg0: vector<2x2xf32>, %arg1: vector<4x4xf32>) -> vector<2x2xf32> {
-//       CHECK: %[[C1:.*]] = arith.constant dense<[1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00]> : vector<4xf32>
-//       CHECK128: %[[C1:.*]] = arith.constant dense<[1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00]> : vector<4xf32>
-//       CHECK0: %[[C1:.*]] = arith.constant dense<{{.*}}> : vector<2x2xf32>
+  // DEFAULT: %[[ARG:.*]] = vector.shape_cast %[[ORIG_ARG]] : vector<2x2xf32> to vector<4xf32>
+  // DEFAULT: %[[ARG2:.*]] = vector.shape_cast %[[ORIG_ARG2]] : vector<4x4xf32> to vector<16xf32>
+  // DEFAULT: %[[CST:.*]] = arith.constant dense<{{.*}}> : vector<4xf32>
+  // DEFAULT: %[[RES:.*]] = vector.shape_cast %[[CST]] : vector<4xf32> to vector<2x2xf32>
 
+  // BW-128: %[[ARG:.*]] = vector.shape_cast %[[ORIG_ARG]] : vector<2x2xf32> to vector<4xf32>
+  // BW-128: %[[CST:.*]] = arith.constant dense<{{.*}}> : vector<4xf32>
+  // BW-128: %[[RES:.*]] = vector.shape_cast %[[CST]] : vector<4xf32> to vector<2x2xf32>
+
+  // BW-0: %[[RES:.*]] = arith.constant dense<{{.*}}> : vector<2x2xf32>
   %0 = arith.constant dense<[[1.0, 2.0], [3.0, 4.0]]> : vector<2x2xf32>
-//       CHECK: %[[RES:.*]] = vector.shape_cast %[[C1]] : vector<4xf32> to vector<2x2xf32>
-//       CHECK128: %[[RES:.*]] = vector.shape_cast %[[C1]] : vector<4xf32> to vector<2x2xf32>
 
-  // CHECK: %[[C2:.*]] = arith.constant dense<[1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00, 1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00, 1.000000e+00, 2.000000e+00, 3.000000e+00, 4.000000e+00, 1.000000e+00, 2.000000e+00, 5.000000e+00, 6.000000e+00]> : vector<16xf32>
-  // CHECK128: %[[C2:.*]] = arith.constant dense<{{.*}}> : vector<4x4xf32>
-  // CHECK0: %[[C2:.*]] = arith.constant dense<{{.*}}> : vector<4x4xf32>
+  // DEFAULT: %[[C2:.*]] = arith.constant dense<{{.*}}> : vector<16xf32>
+  // BW-128: %[[C2:.*]] = arith.constant dense<{{.*}}> : vector<4x4xf32>
+  // BW-0: %[[C2:.*]] = arith.constant dense<{{.*}}> : vector<4x4xf32>
   %5 = arith.constant dense<[[1.0, 2.0, 3.0, 4.0], [1.0, 2.0,3.0, 4.0], [1.0, 2.0, 3.0, 4.0], [1.0, 2.0, 5.0, 6.0]]> : vector<4x4xf32>
-// Arith and math ops are handled in generic way, check some of them
-//       CHECK: %[[SIN:.*]] =  math.sin %[[ARG]] : vector<4xf32>
-//       CHECK128: %[[SIN:.*]] =  math.sin %[[ARG]] : vector<4xf32>
-//       CHECK0: %[[SIN:.*]] =  math.sin %[[ORIG_ARG]] : vector<2x2xf32>
+
+  // Arith and math ops are handled in generic way, check some of them
+  // DEFAULT: %[[SIN:.*]] =  math.sin %[[ARG]] : vector<4xf32>
+  // BW-128: %[[SIN:.*]] =  math.sin %[[ARG]] : vector<4xf32>
+  // BW-0: %[[SIN:.*]] =  math.sin %[[ORIG_ARG]] : vector<2x2xf32>
   %1 = math.sin %arg0 : vector<2x2xf32>
 
-  //     CHECK: %[[SIN1:.*]] =  math.sin %[[ARG2]] : vector<16xf32>
-//       CHECK128: %[[SIN1:.*]] =  math.sin %[[ORIG_ARG2]] : vector<4x4xf32>
-//       CHECK0: %[[SIN1:.*]] =  math.sin %[[ORIG_ARG2]] : vector<4x4xf32>
+  // DEFAULT: %[[SIN1:.*]] =  math.sin %[[ARG2]] : vector<16xf32>
+  // BW-128: %[[SIN1:.*]] =  math.sin %[[ORIG_ARG2]] : vector<4x4xf32>
+  // BW-0: %[[SIN1:.*]] =  math.sin %[[ORIG_ARG2]] : vector<4x4xf32>
   %6 = math.sin %arg1 : vector<4x4xf32>
-//       CHECK: %{{.*}} = arith.addf %[[ARG]], %[[C1]] : vector<4xf32>
-//       CHECK128: %{{.*}} = arith.addf %[[ARG]], %[[C1]] : vector<4xf32>
-//       CHECK0: %{{.*}} = arith.addf %{{.*}} : vector<2x2xf32>
 
+  // DEFAULT: %{{.*}} = arith.addf %[[ARG]], %[[CST]] : vector<4xf32>
+  // BW-128: %{{.*}} = arith.addf %[[ARG]], %[[CST]] : vector<4xf32>
+  // BW-0: %{{.*}} = arith.addf %{{.*}} : vector<2x2xf32>
   %2 = arith.addf %arg0, %0 :  vector<2x2xf32>
 
-  // CHECK: %[[ADD2:.*]] = arith.addf %[[ARG2]], %[[C2]] : vector<16xf32>
-  // CHECK128: %[[ADD2:.*]] = arith.addf %[[ORIG_ARG2]], %[[C2]] : vector<4x4xf32>
-  // CHECK0: %[[ADD2:.*]] = arith.addf %[[ORIG_ARG2]], %[[C2]] : vector<4x4xf32>
+  // DEFAULT: %[[ADD2:.*]] = arith.addf %[[ARG2]], %[[C2]] : vector<16xf32>
+  // BW-128: %[[ADD2:.*]] = arith.addf %[[ORIG_ARG2]], %[[C2]] : vector<4x4xf32>
+  // BW-0: %[[ADD2:.*]] = arith.addf %[[ORIG_ARG2]], %[[C2]] : vector<4x4xf32>
   %7 = arith.addf %arg1, %5 : vector<4x4xf32>
-//       CHECK: return %[[RES]] : vector<2x2xf32>
-//       CHECK128: return %[[RES]] : vector<2x2xf32>
+
+  // ALL: return %[[RES]] : vector<2x2xf32>
   return %0 : vector<2x2xf32>
 }
 
-// CHECK-LABEL: test_index_no_linearize
-// CHECK128-LABEL: test_index_no_linearize
-// CHECK0-LABEL: test_index_no_linearize
+// -----
+
+// ALL-LABEL: test_index_no_linearize
 func.func @test_index_no_linearize(%arg0: vector<2x2xindex>, %arg1: vector<2x2xindex>) -> vector<2x2xindex> {
-    // CHECK: %[[ADD:.*]] = arith.addi {{.*}} : vector<2x2xindex>
-    // CHECK128: %[[ADD:.*]] = arith.addi {{.*}} : vector<2x2xindex>
-    // CHECK0: %[[ADD:.*]] = arith.addi {{.*}} : vector<2x2xindex>
+    // ALL: %[[ADD:.*]] = arith.addi {{.*}} : vector<2x2xindex>
     %0 = arith.addi %arg0, %arg1 : vector<2x2xindex>
     return %0 : vector<2x2xindex>
 }
@@ -95,10 +90,9 @@ func.func @test_index_no_linearize(%arg0: vector<2x2xindex>, %arg1: vector<2x2xi
 
 // vectorizable operation (arith.mulf) with tensor result types.
 
-func.func @nonvec_result(%arg0: tensor<2x2xf32>, %arg1: tensor<2x2xf32>) -> (tensor<2x2xf32>, tensor<2x2xf32>) {
-    // CHECK: %[[MULF:.*]] = arith.mulf %arg0, %arg1 : tensor<2x2xf32>
-    // CHECK128: %[[MULF:.*]] = arith.mulf %arg0, %arg1 : tensor<2x2xf32>
-    // CHECK0: %[[MULF:.*]] = arith.mulf %arg0, %arg1 : tensor<2x2xf32>
+// ALL-LABEL: test_tensor_no_linearize
+func.func @test_tensor_no_linearize(%arg0: tensor<2x2xf32>, %arg1: tensor<2x2xf32>) -> (tensor<2x2xf32>, tensor<2x2xf32>) {
+    // ALL: %[[MULF:.*]] = arith.mulf %arg0, %arg1 : tensor<2x2xf32>
     %0 = arith.mulf %arg0, %arg1 : tensor<2x2xf32>
 
     return %0, %arg0 : tensor<2x2xf32>, tensor<2x2xf32>
-- 
GitLab


From f92f77521263c8f20171b62688ecc86de9c9dfa9 Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Tue, 26 Mar 2024 10:02:22 -0700
Subject: [PATCH 310/404] [flang][NFC] Address reported "possible missing
 return" (#86523)

A function uses "if constexpr" to consider all possible types in a
variant, but looks as if it can fall out without returning an
expression. Add a final "else" with a crash to make things more clear
and to protect against unlikely future extensions of the type.

Fixes https://github.com/llvm/llvm-project/issues/86391.
---
 flang/lib/Semantics/tools.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index 2230047abd72..654ebb1fa335 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -1287,6 +1287,8 @@ static bool StopAtComponentPre(const Symbol &component) {
     return !IsPointer(component);
   } else if constexpr (componentKind == ComponentKind::PotentialAndPointer) {
     return true;
+  } else {
+    DIE("unexpected ComponentKind");
   }
 }
 
-- 
GitLab


From 81fc43a39878dbd938b45d7735fe6e0fe37b03c3 Mon Sep 17 00:00:00 2001
From: Jonas Hahnfeld <jonas.hahnfeld@cern.ch>
Date: Tue, 26 Mar 2024 18:05:18 +0100
Subject: [PATCH 311/404] [unittests] Fix `TableGenTests` with
 `LLVM_LINK_LLVM_DYLIB` (#86664)

Since the restructuring in commit fa3d789df1, the option
DISABLE_LLVM_LINK_LLVM_DYLIB is counter-productive and leads to
CommandLine Error: Option 'debug-counter' registered more than once!
---
 llvm/unittests/TableGen/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/unittests/TableGen/CMakeLists.txt b/llvm/unittests/TableGen/CMakeLists.txt
index fae0eee06e4b..57b237306b19 100644
--- a/llvm/unittests/TableGen/CMakeLists.txt
+++ b/llvm/unittests/TableGen/CMakeLists.txt
@@ -9,7 +9,7 @@ tablegen(LLVM AutomataTables.inc -gen-searchable-tables)
 tablegen(LLVM AutomataAutomata.inc -gen-automata)
 add_public_tablegen_target(AutomataTestTableGen)
 
-add_llvm_unittest(TableGenTests DISABLE_LLVM_LINK_LLVM_DYLIB
+add_llvm_unittest(TableGenTests
   AutomataTest.cpp
   CodeExpanderTest.cpp
   ParserEntryPointTest.cpp
-- 
GitLab


From 4c4ea249cef8db71d30517aab60a081e764260e1 Mon Sep 17 00:00:00 2001
From: smanna12 <soumi.manna@intel.com>
Date: Tue, 26 Mar 2024 12:05:40 -0500
Subject: [PATCH 312/404] [NFC][CLANG] Fix static analyzer bugs about
 unnecessary object copies with auto keyword (#85962)

Reported by Static Analyzer Tool:
In clang::dataflow::Environment::initialize(): Using the auto keyword
without an & causes the copy of an object of type LambdaCapture
---
 clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
index cc1ebd511191..70e0623805a8 100644
--- a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
+++ b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
@@ -416,7 +416,7 @@ void Environment::initialize() {
     assert(Parent != nullptr);
 
     if (Parent->isLambda()) {
-      for (auto Capture : Parent->captures()) {
+      for (const auto &Capture : Parent->captures()) {
         if (Capture.capturesVariable()) {
           const auto *VarDecl = Capture.getCapturedVar();
           assert(VarDecl != nullptr);
-- 
GitLab


From f050a098b5e0745b6c230410f433e48dcc46d160 Mon Sep 17 00:00:00 2001
From: Ivan Butygin <ivan.butygin@gmail.com>
Date: Tue, 26 Mar 2024 20:06:06 +0300
Subject: [PATCH 313/404] [mlir][spirv] Remove `enableFastMathMode` flag from
 SPIR-V conversion (#86578)

Most of arith/math ops support fastmath attribute, use it instead of
global flag.
---
 mlir/include/mlir/Conversion/Passes.td             |  4 ----
 .../Dialect/SPIRV/Transforms/SPIRVConversion.h     |  5 -----
 mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp  |  7 +++----
 mlir/test/Conversion/ArithToSPIRV/fast-math.mlir   | 14 +++++++-------
 4 files changed, 10 insertions(+), 20 deletions(-)

diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index 7e7ee3a2f780..d094ee3b36ab 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -172,10 +172,6 @@ def ConvertArithToSPIRV : Pass<"convert-arith-to-spirv"> {
            "bool", /*default=*/"true",
            "Emulate narrower scalar types with 32-bit ones if not supported by "
            "the target">,
-    Option<"enableFastMath", "enable-fast-math",
-           "bool", /*default=*/"false",
-           "Enable fast math mode (assuming no NaN and infinity for floating "
-           "point values) when performing conversion">
   ];
 }
 
diff --git a/mlir/include/mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h b/mlir/include/mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h
index 933d62e35fce..09eecafc0c8a 100644
--- a/mlir/include/mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h
+++ b/mlir/include/mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h
@@ -55,11 +55,6 @@ struct SPIRVConversionOptions {
   /// values will be packed into one 32-bit value to be memory efficient.
   bool emulateLT32BitScalarTypes{true};
 
-  /// Whether to enable fast math mode during conversion. If true, various
-  /// patterns would assume no NaN/infinity numbers as inputs, and thus there
-  /// will be no special guards emitted to check and handle such cases.
-  bool enableFastMathMode{false};
-
   /// Use 64-bit integers when converting index types.
   bool use64bitIndex{false};
 };
diff --git a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
index edf81bd7a8f3..eb338c2da4e8 100644
--- a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
+++ b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
@@ -995,7 +995,7 @@ public:
     auto *converter = getTypeConverter<SPIRVTypeConverter>();
 
     Value replace;
-    if (converter->getOptions().enableFastMathMode) {
+    if (bitEnumContainsAll(op.getFastmath(), arith::FastMathFlags::nnan)) {
       if (op.getPredicate() == arith::CmpFPredicate::ORD) {
         // Ordered comparsion checks if neither operand is NaN.
         replace = spirv::ConstantOp::getOne(op.getType(), loc, rewriter);
@@ -1122,7 +1122,7 @@ public:
     Value spirvOp =
         rewriter.create<SPIRVOp>(loc, dstType, adaptor.getOperands());
 
-    if (converter->getOptions().enableFastMathMode) {
+    if (bitEnumContainsAll(op.getFastmath(), arith::FastMathFlags::nnan)) {
       rewriter.replaceOp(op, spirvOp);
       return success();
     }
@@ -1177,7 +1177,7 @@ public:
         rewriter.create<SPIRVOp>(loc, dstType, adaptor.getOperands());
 
     if (!shouldInsertNanGuards<SPIRVOp>() ||
-        converter->getOptions().enableFastMathMode) {
+        bitEnumContainsAll(op.getFastmath(), arith::FastMathFlags::nnan)) {
       rewriter.replaceOp(op, spirvOp);
       return success();
     }
@@ -1286,7 +1286,6 @@ struct ConvertArithToSPIRVPass
 
     SPIRVConversionOptions options;
     options.emulateLT32BitScalarTypes = this->emulateLT32BitScalarTypes;
-    options.enableFastMathMode = this->enableFastMath;
     SPIRVTypeConverter typeConverter(targetAttr, options);
 
     // Use UnrealizedConversionCast as the bridge so that we don't need to pull
diff --git a/mlir/test/Conversion/ArithToSPIRV/fast-math.mlir b/mlir/test/Conversion/ArithToSPIRV/fast-math.mlir
index dbf0361c2ab3..9bbe28fb127a 100644
--- a/mlir/test/Conversion/ArithToSPIRV/fast-math.mlir
+++ b/mlir/test/Conversion/ArithToSPIRV/fast-math.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -split-input-file -convert-arith-to-spirv=enable-fast-math -verify-diagnostics %s | FileCheck %s
+// RUN: mlir-opt -split-input-file -convert-arith-to-spirv -verify-diagnostics %s | FileCheck %s
 
 module attributes {
   spirv.target_env = #spirv.target_env<#spirv.vce<v1.0, [], []>, #spirv.resource_limits<>>
@@ -8,7 +8,7 @@ module attributes {
 // CHECK-SAME: %[[LHS:.+]]: f32, %[[RHS:.+]]: f32
 func.func @cmpf_ordered(%arg0 : f32, %arg1 : f32) -> i1 {
   // CHECK: %[[T:.+]] = spirv.Constant true
-  %0 = arith.cmpf ord, %arg0, %arg1 : f32
+  %0 = arith.cmpf ord, %arg0, %arg1 fastmath<fast> : f32
   // CHECK: return %[[T]]
   return %0: i1
 }
@@ -17,7 +17,7 @@ func.func @cmpf_ordered(%arg0 : f32, %arg1 : f32) -> i1 {
 // CHECK-SAME: %[[LHS:.+]]: vector<4xf32>, %[[RHS:.+]]: vector<4xf32>
 func.func @cmpf_unordered(%arg0 : vector<4xf32>, %arg1 : vector<4xf32>) -> vector<4xi1> {
   // CHECK: %[[F:.+]] = spirv.Constant dense<false>
-  %0 = arith.cmpf uno, %arg0, %arg1 : vector<4xf32>
+  %0 = arith.cmpf uno, %arg0, %arg1 fastmath<nnan> : vector<4xf32>
   // CHECK: return %[[F]]
   return %0: vector<4xi1>
 }
@@ -34,7 +34,7 @@ module attributes {
 // CHECK-SAME: %[[LHS:.+]]: f32, %[[RHS:.+]]: f32
 func.func @minimumf(%arg0 : f32, %arg1 : f32) -> f32 {
   // CHECK: %[[F:.+]] = spirv.GL.FMin %[[LHS]], %[[RHS]]
-  %0 = arith.minimumf %arg0, %arg1 : f32
+  %0 = arith.minimumf %arg0, %arg1 fastmath<fast> : f32
   // CHECK: return %[[F]]
   return %0: f32
 }
@@ -43,7 +43,7 @@ func.func @minimumf(%arg0 : f32, %arg1 : f32) -> f32 {
 // CHECK-SAME: %[[LHS:.+]]: vector<4xf32>, %[[RHS:.+]]: vector<4xf32>
 func.func @maximumf(%arg0 : vector<4xf32>, %arg1 : vector<4xf32>) -> vector<4xf32> {
   // CHECK: %[[F:.+]] = spirv.GL.FMax %[[LHS]], %[[RHS]]
-  %0 = arith.maximumf %arg0, %arg1 : vector<4xf32>
+  %0 = arith.maximumf %arg0, %arg1 fastmath<fast> : vector<4xf32>
   // CHECK: return %[[F]]
   return %0: vector<4xf32>
 }
@@ -52,7 +52,7 @@ func.func @maximumf(%arg0 : vector<4xf32>, %arg1 : vector<4xf32>) -> vector<4xf3
 // CHECK-SAME: %[[LHS:.+]]: f32, %[[RHS:.+]]: f32
 func.func @minnumf(%arg0 : f32, %arg1 : f32) -> f32 {
   // CHECK: %[[F:.+]] = spirv.GL.FMin %[[LHS]], %[[RHS]]
-  %0 = arith.minnumf %arg0, %arg1 : f32
+  %0 = arith.minnumf %arg0, %arg1 fastmath<fast> : f32
   // CHECK: return %[[F]]
   return %0: f32
 }
@@ -61,7 +61,7 @@ func.func @minnumf(%arg0 : f32, %arg1 : f32) -> f32 {
 // CHECK-SAME: %[[LHS:.+]]: vector<4xf32>, %[[RHS:.+]]: vector<4xf32>
 func.func @maxnumf(%arg0 : vector<4xf32>, %arg1 : vector<4xf32>) -> vector<4xf32> {
   // CHECK: %[[F:.+]] = spirv.GL.FMax %[[LHS]], %[[RHS]]
-  %0 = arith.maxnumf %arg0, %arg1 : vector<4xf32>
+  %0 = arith.maxnumf %arg0, %arg1 fastmath<fast> : vector<4xf32>
   // CHECK: return %[[F]]
   return %0: vector<4xf32>
 }
-- 
GitLab


From f4fc959c35eb862776ac2e83cb9f41aa88989e3f Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Tue, 26 Mar 2024 10:11:19 -0700
Subject: [PATCH 314/404] [flang] Catch impossible but necessary TBP override
 (#86558)

An apparent attempt to override a type-bound procedure is not allowed to
be interpreted as on override when the procedure is PRIVATE and the
override attempt appears in another module. However, if the TBP that
would have been overridden is a DEFERRED procedure in an abstract base
type, the override must take place. PRIVATE DEFERRED procedures must
therefore have all of their overrides appear in the same module as the
abstract base type.
---
 flang/include/flang/Semantics/tools.h      |  3 ++-
 flang/lib/Semantics/check-declarations.cpp |  9 ++++++-
 flang/lib/Semantics/tools.cpp              | 11 ++++++---
 flang/test/Semantics/deferred01.f90        | 28 ++++++++++++++++++++++
 4 files changed, 46 insertions(+), 5 deletions(-)
 create mode 100644 flang/test/Semantics/deferred01.f90

diff --git a/flang/include/flang/Semantics/tools.h b/flang/include/flang/Semantics/tools.h
index 66774b51316c..f0eb82eebefa 100644
--- a/flang/include/flang/Semantics/tools.h
+++ b/flang/include/flang/Semantics/tools.h
@@ -53,7 +53,8 @@ const Symbol *FindPointerComponent(const Symbol &);
 const Symbol *FindInterface(const Symbol &);
 const Symbol *FindSubprogram(const Symbol &);
 const Symbol *FindFunctionResult(const Symbol &);
-const Symbol *FindOverriddenBinding(const Symbol &);
+const Symbol *FindOverriddenBinding(
+    const Symbol &, bool &isInaccessibleDeferred);
 const Symbol *FindGlobal(const Symbol &);
 
 const DeclTypeSpec *FindParentTypeSpec(const DerivedTypeSpec &);
diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index 581371ff7a00..dec8fee774c5 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -2346,7 +2346,14 @@ void CheckHelper::CheckProcBinding(
         "Intrinsic procedure '%s' is not a specific intrinsic permitted for use in the definition of binding '%s'"_err_en_US,
         binding.symbol().name(), symbol.name());
   }
-  if (const Symbol *overridden{FindOverriddenBinding(symbol)}) {
+  bool isInaccessibleDeferred{false};
+  if (const Symbol *
+      overridden{FindOverriddenBinding(symbol, isInaccessibleDeferred)}) {
+    if (isInaccessibleDeferred) {
+      SayWithDeclaration(*overridden,
+          "Override of PRIVATE DEFERRED '%s' must appear in its module"_err_en_US,
+          symbol.name());
+    }
     if (overridden->attrs().test(Attr::NON_OVERRIDABLE)) {
       SayWithDeclaration(*overridden,
           "Override of NON_OVERRIDABLE '%s' is not permitted"_err_en_US,
diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index 654ebb1fa335..df435906af68 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -528,7 +528,9 @@ const Symbol *FindSubprogram(const Symbol &symbol) {
       symbol.details());
 }
 
-const Symbol *FindOverriddenBinding(const Symbol &symbol) {
+const Symbol *FindOverriddenBinding(
+    const Symbol &symbol, bool &isInaccessibleDeferred) {
+  isInaccessibleDeferred = false;
   if (symbol.has<ProcBindingDetails>()) {
     if (const DeclTypeSpec * parentType{FindParentTypeSpec(symbol.owner())}) {
       if (const DerivedTypeSpec * parentDerived{parentType->AsDerived()}) {
@@ -537,8 +539,11 @@ const Symbol *FindOverriddenBinding(const Symbol &symbol) {
               overridden{parentScope->FindComponent(symbol.name())}) {
             // 7.5.7.3 p1: only accessible bindings are overridden
             if (!overridden->attrs().test(Attr::PRIVATE) ||
-                (FindModuleContaining(overridden->owner()) ==
-                    FindModuleContaining(symbol.owner()))) {
+                FindModuleContaining(overridden->owner()) ==
+                    FindModuleContaining(symbol.owner())) {
+              return overridden;
+            } else if (overridden->attrs().test(Attr::DEFERRED)) {
+              isInaccessibleDeferred = true;
               return overridden;
             }
           }
diff --git a/flang/test/Semantics/deferred01.f90 b/flang/test/Semantics/deferred01.f90
new file mode 100644
index 000000000000..87818c10bd39
--- /dev/null
+++ b/flang/test/Semantics/deferred01.f90
@@ -0,0 +1,28 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+! Deferred TBPs must be overridden, but when they are private, those
+! overrides must appear in the same module.
+module m1
+  type, abstract :: absBase
+   contains
+    procedure(deferredInterface), deferred, private :: deferredTbp
+  end type
+  abstract interface
+    subroutine deferredInterface(x)
+      import absBase
+      class(absBase), intent(in) :: x
+    end
+  end interface
+end
+
+module m2
+  use m1
+  type, extends(absBase) :: ext
+   contains
+    !ERROR: Override of PRIVATE DEFERRED 'deferredtbp' must appear in its module
+    procedure :: deferredTbp => implTbp
+  end type
+ contains
+  subroutine implTbp(x)
+    class(ext), intent(in) :: x
+  end
+end
-- 
GitLab


From 313734795c67ca0bfb82f96b39e123774ddf24b4 Mon Sep 17 00:00:00 2001
From: Piotr Zegar <me@piotrzegar.pl>
Date: Tue, 26 Mar 2024 18:20:25 +0100
Subject: [PATCH 315/404] [clang-tidy] Fix fix-it overlaps in
 readability-static-definition-in-anonymous-namespace (#86599)

Because check emitted multiple warnings for every template instance
fix-it couldn't be applied due to overlaps.

Using TK_IgnoreUnlessSpelledInSource and restricting check to C++ only.
---
 .../StaticDefinitionInAnonymousNamespaceCheck.h       |  6 ++++++
 clang-tools-extra/docs/ReleaseNotes.rst               |  5 +++++
 .../static-definition-in-anonymous-namespace.cpp      | 11 +++++++++++
 3 files changed, 22 insertions(+)

diff --git a/clang-tools-extra/clang-tidy/readability/StaticDefinitionInAnonymousNamespaceCheck.h b/clang-tools-extra/clang-tidy/readability/StaticDefinitionInAnonymousNamespaceCheck.h
index c28087e07e9b..620cd6e3f2f8 100644
--- a/clang-tools-extra/clang-tidy/readability/StaticDefinitionInAnonymousNamespaceCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/StaticDefinitionInAnonymousNamespaceCheck.h
@@ -24,6 +24,12 @@ public:
       : ClangTidyCheck(Name, Context) {}
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
+  std::optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_IgnoreUnlessSpelledInSource;
+  }
 };
 
 } // namespace clang::tidy::readability
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 2392ccaf6575..f6ae712510ca 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -253,6 +253,11 @@ Changes in existing checks
   mode by resolving symbolic links to header files. Fixed handling of Hungarian
   Prefix when configured to `LowerCase`.
 
+- Improved :doc:`readability-static-definition-in-anonymous-namespace
+  <clang-tidy/checks/readability/static-definition-in-anonymous-namespace>`
+  check by resolving fix-it overlaps in template code by disregarding implicit
+  instances.
+
 Removed checks
 ^^^^^^^^^^^^^^
 
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/static-definition-in-anonymous-namespace.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/static-definition-in-anonymous-namespace.cpp
index e9938db4f5b8..e204199393db 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/static-definition-in-anonymous-namespace.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/static-definition-in-anonymous-namespace.cpp
@@ -51,6 +51,17 @@ static int c = 1;
 } // namespace deep_inner
 } // namespace inner
 
+template<typename T>
+static void printTemplate(T&&) {}
+// CHECK-MESSAGES: :[[@LINE-1]]:13: warning: 'printTemplate' is a static definition in anonymous namespace; static is redundant here [readability-static-definition-in-anonymous-namespace]
+// CHECK-FIXES: {{^}}void printTemplate(T&&) {}
+
+void testTemplate() {
+  printTemplate(5);
+  printTemplate(5U);
+  printTemplate("some string");
+}
+
 } // namespace
 
 namespace N {
-- 
GitLab


From 26dd12871ca8a3dcc5e6920add231af354310088 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Tue, 26 Mar 2024 10:05:13 -0700
Subject: [PATCH 316/404] [SLP]Do not propagate nuw/nsw flags for alt nodes,
 affected by minbitwidth analysis.

Need to drop nuw/nsw flags, if the alternate node is resized after the
minbitwidth analysis, to avoid producing poison values in corner cases.
---
 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp           | 4 ++--
 .../Transforms/SLPVectorizer/X86/reorder_diamond_match.ll | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index f7cb06ad3e96..551d97570756 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -12621,8 +12621,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) {
           },
           Mask, &OpScalars, &AltScalars);
 
-      propagateIRFlags(V0, OpScalars);
-      propagateIRFlags(V1, AltScalars);
+      propagateIRFlags(V0, OpScalars, E->getMainOp(), !MinBWs.contains(E));
+      propagateIRFlags(V1, AltScalars, E->getAltOp(), !MinBWs.contains(E));
 
       Value *V = Builder.CreateShuffleVector(V0, V1, Mask);
       if (auto *I = dyn_cast<Instruction>(V)) {
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reorder_diamond_match.ll b/llvm/test/Transforms/SLPVectorizer/X86/reorder_diamond_match.ll
index dce85b4b2a19..9682567b173c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reorder_diamond_match.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reorder_diamond_match.ll
@@ -11,11 +11,11 @@ define void @test() {
 ; CHECK-NEXT:    [[TMP6:%.*]] = shl <4 x i16> [[TMP5]], zeroinitializer
 ; CHECK-NEXT:    [[TMP7:%.*]] = add <4 x i16> [[TMP6]], zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i16> [[TMP7]], <4 x i16> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; CHECK-NEXT:    [[TMP9:%.*]] = add nsw <4 x i16> [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = sub nsw <4 x i16> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add <4 x i16> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = sub <4 x i16> [[TMP7]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i16> [[TMP9]], <4 x i16> [[TMP10]], <4 x i32> <i32 1, i32 4, i32 3, i32 6>
-; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <4 x i16> zeroinitializer, [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = sub nsw <4 x i16> zeroinitializer, [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = add <4 x i16> zeroinitializer, [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = sub <4 x i16> zeroinitializer, [[TMP11]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i16> [[TMP12]], <4 x i16> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP15:%.*]] = sext <4 x i16> [[TMP14]] to <4 x i32>
 ; CHECK-NEXT:    store <4 x i32> [[TMP15]], ptr [[TMP2]], align 16
-- 
GitLab


From 1eaef44532266694c28f49f5c0373e7fc93d4f82 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Tue, 26 Mar 2024 10:33:17 -0700
Subject: [PATCH 317/404] [TosaToTensor] Fix a warning (#86703)

This patch fixes:

  mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp:76:46: error:
  'multiplies' may not intend to support class template argument
  deduction [-Werror,-Wctad-maybe-unsupported]
---
 mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
index 11ba98ddf352..cd6da3558246 100644
--- a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
+++ b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
@@ -73,7 +73,7 @@ TensorType inferReshapeExpandedType(TensorType inputType,
     // Calculate the product of all elements in 'newShape' except for the -1
     // placeholder, which we discard by negating the result.
     int64_t totalSizeNoPlaceholder = -std::accumulate(
-        newShape.begin(), newShape.end(), 1, std::multiplies());
+        newShape.begin(), newShape.end(), 1, std::multiplies<int64_t>());
 
     // If there is a 0 component in 'newShape', resolve the placeholder as 0.
     if (totalSizeNoPlaceholder == 0)
-- 
GitLab


From 8cb4eb991401093fbfbc255ea752210d0bd5acfe Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 26 Mar 2024 10:07:11 -0700
Subject: [PATCH 318/404] [RISCV] Remove unneeded VAESKF_MV_I tablegen class.
 NFC

It is identical to PALUVINoVm so we can use that instead.
---
 llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
index ff69b0665617..035ce63e91e9 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
@@ -68,7 +68,7 @@ class PALUVVNoVm<bits<6> funct6, RISCVVFormat opv, string opcodestr>
 }
 
 // op vd, vs2, imm
-class PALUVINoVm<bits<6> funct6, string opcodestr, Operand optype = simm5>
+class PALUVINoVm<bits<6> funct6, string opcodestr, Operand optype>
     : VALUVINoVm<funct6, opcodestr, optype> {
   let Inst{6-0} = OPC_OP_P.Value;
   let Inst{14-12} = OPMVV.Value;
@@ -85,14 +85,6 @@ multiclass VAES_MV_V_S<bits<6> funct6_vv, bits<6> funct6_vs, bits<5> vs1,
   def NAME # _VV : PALUVs2NoVm<funct6_vv, vs1, opv, opcodestr # ".vv">;
   def NAME # _VS : PALUVs2NoVm<funct6_vs, vs1, opv, opcodestr # ".vs">;
 }
-
-// vaeskf1.vi and vaeskf2.vi uses different opcode and format, we need
-// to customize one for them.
-class VAESKF_MV_I<bits<6> funct6, string opcodestr, Operand optype>
-    : VALUVINoVm<funct6, opcodestr, optype> {
-  let Inst{6-0} = OPC_OP_P.Value;
-  let Inst{14-12} = OPMVV.Value;
-}
 } // hasSideEffects = 0, mayLoad = 0, mayStore = 0
 
 //===----------------------------------------------------------------------===//
@@ -137,8 +129,8 @@ let Predicates = [HasStdExtZvkned], RVVConstraint = NoConstraint in {
   defm VAESDM     : VAES_MV_V_S<0b101000, 0b101001, 0b00000, OPMVV, "vaesdm">;
   defm VAESEF     : VAES_MV_V_S<0b101000, 0b101001, 0b00011, OPMVV, "vaesef">;
   defm VAESEM     : VAES_MV_V_S<0b101000, 0b101001, 0b00010, OPMVV, "vaesem">;
-  def  VAESKF1_VI : VAESKF_MV_I<0b100010, "vaeskf1.vi", uimm5>;
-  def  VAESKF2_VI : VAESKF_MV_I<0b101010, "vaeskf2.vi", uimm5>;
+  def  VAESKF1_VI : PALUVINoVm<0b100010, "vaeskf1.vi", uimm5>;
+  def  VAESKF2_VI : PALUVINoVm<0b101010, "vaeskf2.vi", uimm5>;
   def  VAESZ_VS   : PALUVs2NoVm<0b101001, 0b00111, OPMVV, "vaesz.vs">;
 } // Predicates = [HasStdExtZvkned]
 
-- 
GitLab


From d0e97fe38b91dac909e6a53ab16d306e5ef0b512 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Tue, 26 Mar 2024 10:44:20 -0700
Subject: [PATCH 319/404] [ArithToSPIRV] Fix a warning (#86702)

mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp:995:11: error:
  unused variable 'converter' [-Werror,-Wunused-variable]
---
 mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
index eb338c2da4e8..7456bf7a87a3 100644
--- a/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
+++ b/mlir/lib/Conversion/ArithToSPIRV/ArithToSPIRV.cpp
@@ -992,7 +992,6 @@ public:
       return failure();
 
     Location loc = op.getLoc();
-    auto *converter = getTypeConverter<SPIRVTypeConverter>();
 
     Value replace;
     if (bitEnumContainsAll(op.getFastmath(), arith::FastMathFlags::nnan)) {
-- 
GitLab


From da6cc4a24ff8953d51f7dc2c4974e8fc9089d693 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= <schuett@gmail.com>
Date: Tue, 26 Mar 2024 18:44:34 +0100
Subject: [PATCH 320/404] [CodeGen] Add nneg and disjoint flags (#86650)

MachineInstr learned the new flags.
---
 llvm/include/llvm/CodeGen/MachineInstr.h      |   2 +
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  |   7 +-
 llvm/lib/CodeGen/MIRParser/MILexer.cpp        |   2 +
 llvm/lib/CodeGen/MIRParser/MILexer.h          |   2 +
 llvm/lib/CodeGen/MIRParser/MIParser.cpp       |   8 +-
 llvm/lib/CodeGen/MIRPrinter.cpp               |   4 +
 llvm/lib/CodeGen/MachineInstr.cpp             |  15 ++
 .../GlobalISel/irtranslator-nneg-disjoint.ll  | 135 ++++++++++++++++++
 8 files changed, 173 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-nneg-disjoint.ll

diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h
index fcdd73d8b65f..7249f812d2cc 100644
--- a/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -115,6 +115,8 @@ public:
                              // this instruction.
     Unpredictable = 1 << 16, // Instruction with unpredictable condition.
     NoConvergent = 1 << 17,  // Call does not require convergence guarantees.
+    NonNeg = 1 << 18,        // The operand is non-negative.
+    Disjoint = 1 << 19,      // Each bit is zero in at least one of the inputs.
   };
 
 private:
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index ef95ceec18a1..80b3f5825922 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1562,9 +1562,14 @@ bool IRTranslator::translateCast(unsigned Opcode, const User &U,
   if (U.getType()->getScalarType()->isBFloatTy() ||
       U.getOperand(0)->getType()->getScalarType()->isBFloatTy())
     return false;
+
+  uint32_t Flags = 0;
+  if (const Instruction *I = dyn_cast<Instruction>(&U))
+    Flags = MachineInstr::copyFlagsFromInstruction(*I);
+
   Register Op = getOrCreateVReg(*U.getOperand(0));
   Register Res = getOrCreateVReg(U);
-  MIRBuilder.buildInstr(Opcode, {Res}, {Op});
+  MIRBuilder.buildInstr(Opcode, {Res}, {Op}, Flags);
   return true;
 }
 
diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.cpp b/llvm/lib/CodeGen/MIRParser/MILexer.cpp
index 870611248466..7bb216553204 100644
--- a/llvm/lib/CodeGen/MIRParser/MILexer.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MILexer.cpp
@@ -213,6 +213,8 @@ static MIToken::TokenKind getIdentifierKind(StringRef Identifier) {
       .Case("nuw", MIToken::kw_nuw)
       .Case("nsw", MIToken::kw_nsw)
       .Case("exact", MIToken::kw_exact)
+      .Case("nneg", MIToken::kw_nneg)
+      .Case("disjoint", MIToken::kw_disjoint)
       .Case("nofpexcept", MIToken::kw_nofpexcept)
       .Case("unpredictable", MIToken::kw_unpredictable)
       .Case("debug-location", MIToken::kw_debug_location)
diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.h b/llvm/lib/CodeGen/MIRParser/MILexer.h
index 0f344da52182..6617ec68e941 100644
--- a/llvm/lib/CodeGen/MIRParser/MILexer.h
+++ b/llvm/lib/CodeGen/MIRParser/MILexer.h
@@ -74,6 +74,8 @@ struct MIToken {
     kw_exact,
     kw_nofpexcept,
     kw_unpredictable,
+    kw_nneg,
+    kw_disjoint,
     kw_debug_location,
     kw_debug_instr_number,
     kw_dbg_instr_ref,
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index 691c60d22724..95924f056628 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -1471,7 +1471,9 @@ bool MIParser::parseInstruction(unsigned &OpCode, unsigned &Flags) {
          Token.is(MIToken::kw_exact) ||
          Token.is(MIToken::kw_nofpexcept) ||
          Token.is(MIToken::kw_noconvergent) ||
-         Token.is(MIToken::kw_unpredictable)) {
+         Token.is(MIToken::kw_unpredictable) ||
+         Token.is(MIToken::kw_nneg) ||
+         Token.is(MIToken::kw_disjoint)) {
     // clang-format on
     // Mine frame and fast math flags
     if (Token.is(MIToken::kw_frame_setup))
@@ -1504,6 +1506,10 @@ bool MIParser::parseInstruction(unsigned &OpCode, unsigned &Flags) {
       Flags |= MachineInstr::Unpredictable;
     if (Token.is(MIToken::kw_noconvergent))
       Flags |= MachineInstr::NoConvergent;
+    if (Token.is(MIToken::kw_nneg))
+      Flags |= MachineInstr::NonNeg;
+    if (Token.is(MIToken::kw_disjoint))
+      Flags |= MachineInstr::Disjoint;
 
     lex();
   }
diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
index 8efe67a9a72b..4cf3074ea3ff 100644
--- a/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -806,6 +806,10 @@ void MIPrinter::print(const MachineInstr &MI) {
     OS << "unpredictable ";
   if (MI.getFlag(MachineInstr::NoConvergent))
     OS << "noconvergent ";
+  if (MI.getFlag(MachineInstr::NonNeg))
+    OS << "nneg ";
+  if (MI.getFlag(MachineInstr::Disjoint))
+    OS << "disjoint ";
 
   OS << TII->getName(MI.getOpcode());
   if (I < E)
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 8102bb971ba6..717c81f01bc3 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -555,6 +555,17 @@ uint32_t MachineInstr::copyFlagsFromInstruction(const Instruction &I) {
       MIFlags |= MachineInstr::MIFlag::NoUWrap;
   }
 
+  // Copy the nonneg flag.
+  if (const PossiblyNonNegInst *PNI = dyn_cast<PossiblyNonNegInst>(&I)) {
+    if (PNI->hasNonNeg())
+      MIFlags |= MachineInstr::MIFlag::NonNeg;
+    // Copy the disjoint flag.
+  } else if (const PossiblyDisjointInst *PD =
+                 dyn_cast<PossiblyDisjointInst>(&I)) {
+    if (PD->isDisjoint())
+      MIFlags |= MachineInstr::MIFlag::Disjoint;
+  }
+
   // Copy the exact flag.
   if (const PossiblyExactOperator *PE = dyn_cast<PossiblyExactOperator>(&I))
     if (PE->isExact())
@@ -1706,6 +1717,10 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST,
     OS << "nofpexcept ";
   if (getFlag(MachineInstr::NoMerge))
     OS << "nomerge ";
+  if (getFlag(MachineInstr::NonNeg))
+    OS << "nneg ";
+  if (getFlag(MachineInstr::Disjoint))
+    OS << "disjoint ";
 
   // Print the opcode name.
   if (TII)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-nneg-disjoint.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-nneg-disjoint.ll
new file mode 100644
index 000000000000..32c742316b87
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-nneg-disjoint.ll
@@ -0,0 +1,135 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=aarch64-linux-gnu -O0 -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
+
+define i32 @call_nneg(i16 %a) {
+  ; CHECK-LABEL: name: call_nneg
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   liveins: $w0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK-NEXT:   %2:_(s32) = nneg G_ZEXT [[TRUNC]](s16)
+  ; CHECK-NEXT:   $w0 = COPY %2(s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
+entry:
+  %result = zext nneg i16 %a to i32
+  ret i32 %result
+}
+
+define i32 @call_not_nneg(i16 %a) {
+  ; CHECK-LABEL: name: call_not_nneg
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   liveins: $w0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK-NEXT:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+  ; CHECK-NEXT:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s16)
+  ; CHECK-NEXT:   $w0 = COPY [[ZEXT]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
+entry:
+  %result = zext i16 %a to i32
+  ret i32 %result
+}
+
+define i32 @call_disjoint(i32 %a, i32 %b) {
+  ; CHECK-LABEL: name: call_disjoint
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   liveins: $w0, $w1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+  ; CHECK-NEXT:   %2:_(s32) = disjoint G_OR [[COPY]], [[COPY1]]
+  ; CHECK-NEXT:   $w0 = COPY %2(s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
+entry:
+  %result = or disjoint i32 %a, %b
+  ret i32 %result
+}
+
+define i32 @call_add(i32 %a, i32 %b) {
+  ; CHECK-LABEL: name: call_add
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   liveins: $w0, $w1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+  ; CHECK-NEXT:   [[ADD:%[0-9]+]]:_(s32) = nsw G_ADD [[COPY]], [[COPY1]]
+  ; CHECK-NEXT:   $w0 = COPY [[ADD]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
+entry:
+  %result = add nsw i32 %a, %b
+  ret i32 %result
+}
+
+define i32 @call_not_disjoint(i32 %a, i32 %b) {
+  ; CHECK-LABEL: name: call_not_disjoint
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   liveins: $w0, $w1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+  ; CHECK-NEXT:   [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY]], [[COPY1]]
+  ; CHECK-NEXT:   $w0 = COPY [[OR]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
+entry:
+  %result = or i32 %a, %b
+  ret i32 %result
+}
+
+define <2 x i64> @call_not_disjoint_vector(<2 x i64> %a, <2 x i64> %b) {
+  ; CHECK-LABEL: name: call_not_disjoint_vector
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   liveins: $q0, $q1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
+  ; CHECK-NEXT:   [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[COPY]], [[COPY1]]
+  ; CHECK-NEXT:   $q0 = COPY [[OR]](<2 x s64>)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $q0
+entry:
+  %result = or <2 x i64> %a, %b
+  ret <2 x i64> %result
+}
+
+define <2 x i64> @call_disjoint_vector(<2 x i64> %a, <2 x i64> %b) {
+  ; CHECK-LABEL: name: call_disjoint_vector
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   liveins: $q0, $q1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
+  ; CHECK-NEXT:   %2:_(<2 x s64>) = disjoint G_OR [[COPY]], [[COPY1]]
+  ; CHECK-NEXT:   $q0 = COPY %2(<2 x s64>)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $q0
+entry:
+  %result = or disjoint <2 x i64> %a, %b
+  ret <2 x i64> %result
+}
+
+define <2 x i64> @call_nneg_vector(<2 x i32> %a) {
+  ; CHECK-LABEL: name: call_nneg_vector
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   liveins: $d0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+  ; CHECK-NEXT:   %1:_(<2 x s64>) = nneg G_ZEXT [[COPY]](<2 x s32>)
+  ; CHECK-NEXT:   $q0 = COPY %1(<2 x s64>)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $q0
+entry:
+  %result = zext nneg <2 x i32> %a to <2 x i64>
+  ret <2 x i64> %result
+}
+
+define <2 x i64> @call_not_nneg_vector(<2 x i32> %a) {
+  ; CHECK-LABEL: name: call_not_nneg_vector
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   liveins: $d0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+  ; CHECK-NEXT:   [[ZEXT:%[0-9]+]]:_(<2 x s64>) = G_ZEXT [[COPY]](<2 x s32>)
+  ; CHECK-NEXT:   $q0 = COPY [[ZEXT]](<2 x s64>)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $q0
+entry:
+  %result = zext <2 x i32> %a to <2 x i64>
+  ret <2 x i64> %result
+}
-- 
GitLab


From 29318abe1d2c55e8543255d70f26ac93261b74a4 Mon Sep 17 00:00:00 2001
From: Jason Molenda <jason@molenda.com>
Date: Tue, 26 Mar 2024 10:54:26 -0700
Subject: [PATCH 321/404] [lldb] Remove test for add-symbol-file adds unwind
 source

In

commit 2f63718f8567413a1c596bda803663eb58d6da5a
Author: Jason Molenda <jmolenda@apple.com>
Date:   Tue Mar 26 09:07:15 2024 -0700

    [lldb] Don't clear a Module's UnwindTable when adding a SymbolFile (#86603)

I changed lldb to not clear a Module's UnwindTable when we add a
SymbolFile to a binary, because the added benefit is marginal, and
handling this reconstruction correctly is difficult.  This test was
written to explicitly create a test without unwind info in the
binary, then add a symbol file with the unwind info, and check that
it is present.  I've intentionally broken this, so I'm removing the
test.
---
 .../SymbolFile/target-symbols-add-unwind.test | 27 -------------------
 1 file changed, 27 deletions(-)
 delete mode 100644 lldb/test/Shell/SymbolFile/target-symbols-add-unwind.test

diff --git a/lldb/test/Shell/SymbolFile/target-symbols-add-unwind.test b/lldb/test/Shell/SymbolFile/target-symbols-add-unwind.test
deleted file mode 100644
index 5420213d405e..000000000000
--- a/lldb/test/Shell/SymbolFile/target-symbols-add-unwind.test
+++ /dev/null
@@ -1,27 +0,0 @@
-# TODO: When it's possible to run "image show-unwind" without a running
-# process, we can remove the unsupported line below, and hard-code an ELF
-# triple in the test.
-# UNSUPPORTED: system-windows, system-darwin
-
-# RUN: cd %T
-# RUN: %clang_host %S/Inputs/target-symbols-add-unwind.c -g \
-# RUN:   -fno-unwind-tables -fno-asynchronous-unwind-tables \
-# RUN:   -o target-symbols-add-unwind.debug
-# RUN: llvm-objcopy --strip-debug target-symbols-add-unwind.debug \
-# RUN:   target-symbols-add-unwind.stripped
-# RUN: %lldb target-symbols-add-unwind.stripped -s %s -o quit | FileCheck %s
-
-process launch --stop-at-entry
-image show-unwind -n main
-# CHECK-LABEL: image show-unwind -n main
-# CHECK-NOT: debug_frame UnwindPlan:
-
-target symbols add -s target-symbols-add-unwind.stripped target-symbols-add-unwind.debug
-# CHECK-LABEL: target symbols add
-# CHECK: symbol file {{.*}} has been added to {{.*}}
-
-image show-unwind -n main
-# CHECK-LABEL: image show-unwind -n main
-# CHECK: debug_frame UnwindPlan:
-# CHECK-NEXT: This UnwindPlan originally sourced from DWARF CFI
-# CHECK-NEXT: This UnwindPlan is sourced from the compiler: yes.
-- 
GitLab


From 0c3e24f7c90243040fedcdfbbb417505f7cc0102 Mon Sep 17 00:00:00 2001
From: Ella Ma <alansnape3058@gmail.com>
Date: Wed, 27 Mar 2024 01:56:56 +0800
Subject: [PATCH 322/404] [analyzer] Allow egraph rewriter not to open the
 generated HTML directly (#85515)

When developing on a headless device through SSH, we do not have a
browser or even an X environment. Hence, it would be more convenient if
the rewriter could stop before attempting to open the generated HTML
file. Then, it can be opened remotely through an HTML server.

This patch adds a new option `--dump-html-only` to make the rewriter
stop before opening the generated HTML in a browser. The new option is
marked in conflict with the existing `--dump-dot-only` option to prevent
unexpected behaviors.
---
 .../utils/analyzer/exploded-graph-rewriter.py | 31 +++++++++++++++++--
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/clang/utils/analyzer/exploded-graph-rewriter.py b/clang/utils/analyzer/exploded-graph-rewriter.py
index c7c6315a0a27..5eaa7738103f 100755
--- a/clang/utils/analyzer/exploded-graph-rewriter.py
+++ b/clang/utils/analyzer/exploded-graph-rewriter.py
@@ -479,11 +479,19 @@ class ExplodedGraph:
 # A visitor that dumps the ExplodedGraph into a DOT file with fancy HTML-based
 # syntax highlighing.
 class DotDumpVisitor:
-    def __init__(self, do_diffs, dark_mode, gray_mode, topo_mode, dump_dot_only):
+    def __init__(
+        self, do_diffs, dark_mode, gray_mode, topo_mode, dump_html_only, dump_dot_only
+    ):
+        assert not (dump_html_only and dump_dot_only), (
+            "Option dump_html_only and dump_dot_only are conflict, "
+            "they cannot be true at the same time."
+        )
+
         self._do_diffs = do_diffs
         self._dark_mode = dark_mode
         self._gray_mode = gray_mode
         self._topo_mode = topo_mode
+        self._dump_html_only = dump_html_only
         self._dump_dot_only = dump_dot_only
         self._output = []
 
@@ -998,6 +1006,8 @@ class DotDumpVisitor:
                 '<html><body bgcolor="%s">%s</body></html>'
                 % ("#1a1a1a" if self._dark_mode else "white", svg),
             )
+            if self._dump_html_only:
+                return
             if sys.platform == "win32":
                 os.startfile(filename)
             elif sys.platform == "darwin":
@@ -1176,7 +1186,17 @@ def main():
         default=False,
         help="black-and-white mode",
     )
-    parser.add_argument(
+    dump_conflict = parser.add_mutually_exclusive_group()
+    dump_conflict.add_argument(
+        "--dump-html-only",
+        action="store_const",
+        dest="dump_html_only",
+        const=True,
+        default=False,
+        help="dump the rewritten egraph to a temporary HTML file, "
+        "but do not open it immediately as by default",
+    )
+    dump_conflict.add_argument(
         "--dump-dot-only",
         action="store_const",
         dest="dump_dot_only",
@@ -1206,7 +1226,12 @@ def main():
     explorer = BasicExplorer()
 
     visitor = DotDumpVisitor(
-        args.diff, args.dark, args.gray, args.topology, args.dump_dot_only
+        args.diff,
+        args.dark,
+        args.gray,
+        args.topology,
+        args.dump_html_only,
+        args.dump_dot_only,
     )
 
     for trimmer in trimmers:
-- 
GitLab


From 1ad29a5c7131a31883d24cc41388c5a3565c7a4f Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Tue, 26 Mar 2024 11:11:34 -0700
Subject: [PATCH 323/404] [SLP][NFC]Add a test with truncated loads, but
 incorrect trunc after minbitwidth analysis.

---
 .../RISCV/trunc-to-large-than-bw.ll           | 41 +++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/trunc-to-large-than-bw.ll

diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/trunc-to-large-than-bw.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/trunc-to-large-than-bw.ll
new file mode 100644
index 000000000000..04d275742832
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/trunc-to-large-than-bw.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -S --passes=slp-vectorizer -mtriple=riscv64-unknown-linux-gnu -mattr="+v" < %s | FileCheck %s
+
+@c = global [12 x i64] zeroinitializer
+
+define i32 @test() {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i64> @llvm.experimental.vp.strided.load.v4i64.p0.i64(ptr align 8 @c, i64 24, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, i32 4)
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <4 x i64> [[TMP0]] to <4 x i16>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <4 x i16> [[TMP1]], <i16 -1, i16 -1, i16 -1, i16 -1>
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <4 x i16> [[TMP2]], <i16 -1, i16 -1, i16 -1, i16 -1>
+; CHECK-NEXT:    [[TMP4:%.*]] = call i16 @llvm.vector.reduce.umax.v4i16(<4 x i16> [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = zext i16 [[TMP4]] to i32
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP5]], i32 1)
+; CHECK-NEXT:    ret i32 [[TMP6]]
+;
+entry:
+  %0 = load i64, ptr @c, align 8
+  %conv = trunc i64 %0 to i32
+  %conv3 = and i32 %conv, 65535
+  %conv4 = xor i32 %conv3, 65535
+  %.conv4 = tail call i32 @llvm.umax.i32(i32 1, i32 %conv4)
+  %1 = load i64, ptr getelementptr inbounds ([12 x i64], ptr @c, i64 0, i64 3), align 8
+  %conv.1 = trunc i64 %1 to i32
+  %conv3.1 = and i32 %conv.1, 65535
+  %conv4.1 = xor i32 %conv3.1, 65535
+  %.conv4.1 = tail call i32 @llvm.umax.i32(i32 %.conv4, i32 %conv4.1)
+  %2 = load i64, ptr getelementptr inbounds ([12 x i64], ptr @c, i64 0, i64 6), align 8
+  %conv.2 = trunc i64 %2 to i32
+  %conv3.2 = and i32 %conv.2, 65535
+  %conv4.2 = xor i32 %conv3.2, 65535
+  %.conv4.2 = tail call i32 @llvm.umax.i32(i32 %.conv4.1, i32 %conv4.2)
+  %3 = load i64, ptr getelementptr inbounds ([12 x i64], ptr @c, i64 0, i64 9), align 8
+  %conv.3 = trunc i64 %3 to i32
+  %conv3.3 = and i32 %conv.3, 65535
+  %conv4.3 = xor i32 %conv3.3, 65535
+  %.conv4.3 = tail call i32 @llvm.umax.i32(i32 %.conv4.2, i32 %conv4.3)
+  ret i32 %.conv4.3
+}
-- 
GitLab


From 342f7d0d35774ffdcb56e8b92252763a59bd2c29 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Tue, 26 Mar 2024 11:27:17 -0700
Subject: [PATCH 324/404] [SLP]Fix PR86620: check final minbitwidth for
 truncs/exts before accepting it.

If the minbitwidth is deduced from the demanded elements, need to check
the final bitwidthfor trunc/ext instruction, bot blindly accepting
the used one.
---
 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp          | 2 ++
 .../SLPVectorizer/RISCV/trunc-to-large-than-bw.ll        | 9 ++++-----
 .../SLPVectorizer/X86/int-bitcast-minbitwidth.ll         | 9 ++++-----
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 551d97570756..fbf1cb6a976f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14092,12 +14092,14 @@ bool BoUpSLP::collectValuesToDemote(
       MaxDepthLevel = 1;
     if (IsProfitableToDemoteRoot)
       IsProfitableToDemote = true;
+    (void)IsPotentiallyTruncated(V, BitWidth);
     break;
   case Instruction::ZExt:
   case Instruction::SExt:
     if (!IsTruncRoot)
       MaxDepthLevel = 1;
     IsProfitableToDemote = true;
+    (void)IsPotentiallyTruncated(V, BitWidth);
     break;
 
   // We can demote certain binary operations if we can demote both of their
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/trunc-to-large-than-bw.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/trunc-to-large-than-bw.ll
index 04d275742832..2d69c7c984dc 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/trunc-to-large-than-bw.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/trunc-to-large-than-bw.ll
@@ -8,11 +8,10 @@ define i32 @test() {
 ; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i64> @llvm.experimental.vp.strided.load.v4i64.p0.i64(ptr align 8 @c, i64 24, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, i32 4)
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc <4 x i64> [[TMP0]] to <4 x i16>
-; CHECK-NEXT:    [[TMP2:%.*]] = and <4 x i16> [[TMP1]], <i16 -1, i16 -1, i16 -1, i16 -1>
-; CHECK-NEXT:    [[TMP3:%.*]] = xor <4 x i16> [[TMP2]], <i16 -1, i16 -1, i16 -1, i16 -1>
-; CHECK-NEXT:    [[TMP4:%.*]] = call i16 @llvm.vector.reduce.umax.v4i16(<4 x i16> [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = zext i16 [[TMP4]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <4 x i64> [[TMP0]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <4 x i32> [[TMP1]], <i32 65535, i32 65535, i32 65535, i32 65535>
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <4 x i32> [[TMP2]], <i32 65535, i32 65535, i32 65535, i32 65535>
+; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP3]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP5]], i32 1)
 ; CHECK-NEXT:    ret i32 [[TMP6]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll
index 5ee801607653..f4a471493f1b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-6 < %s | FileCheck %s
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-9 < %s | FileCheck %s
 
 define void @t(i64 %v) {
 ; CHECK-LABEL: define void @t(
@@ -7,10 +7,9 @@ define void @t(i64 %v) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i64> poison, i64 [[V]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc <4 x i64> [[TMP1]] to <4 x i16>
-; CHECK-NEXT:    [[TMP3:%.*]] = mul <4 x i16> [[TMP2]], <i16 5, i16 6, i16 3, i16 2>
-; CHECK-NEXT:    [[TMP4:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = sext i16 [[TMP4]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc <4 x i64> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = mul <4 x i32> [[TMP2]], <i32 5, i32 6, i32 3, i32 2>
+; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP3]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[TMP5]], 65535
 ; CHECK-NEXT:    store i32 [[TMP6]], ptr null, align 4
 ; CHECK-NEXT:    ret void
-- 
GitLab


From 1c9d5c25ae43778a1f5bd14898223c33a203d4d5 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 26 Mar 2024 18:12:01 +0000
Subject: [PATCH 325/404] [DAG] foldAddSubBoolOfMaskedVal - reuse existing
 SDLoc instead of regenerating it. NFC.

---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 6f7a2a8a4c14..39ee95d7007c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -2493,7 +2493,8 @@ SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
   return SelectOp;
 }
 
-static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG) {
+static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, const SDLoc &DL,
+                                         SelectionDAG &DAG) {
   assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&
          "Expecting add or sub");
 
@@ -2525,7 +2526,6 @@ static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG) {
   // add (zext i1 (seteq (X & 1), 0)), C --> sub C+1, (zext (X & 1))
   // sub C, (zext i1 (seteq (X & 1), 0)) --> add C-1, (zext (X & 1))
   EVT VT = C.getValueType();
-  SDLoc DL(N);
   SDValue LowBit = DAG.getZExtOrTrunc(SetCC.getOperand(0), DL, VT);
   SDValue C1 = IsAdd ? DAG.getConstant(CN->getAPIntValue() + 1, DL, VT) :
                        DAG.getConstant(CN->getAPIntValue() - 1, DL, VT);
@@ -2875,7 +2875,7 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
   if (SDValue Combined = visitADDLike(N))
     return Combined;
 
-  if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
+  if (SDValue V = foldAddSubBoolOfMaskedVal(N, DL, DAG))
     return V;
 
   if (SDValue V = foldAddSubOfSignBit(N, DAG))
@@ -3874,7 +3874,7 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
   if (N1.isUndef())
     return N1;
 
-  if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
+  if (SDValue V = foldAddSubBoolOfMaskedVal(N, DL, DAG))
     return V;
 
   if (SDValue V = foldAddSubOfSignBit(N, DAG))
-- 
GitLab


From 8edb12fe6cf81d194a4553da5d10e976bc4cfd4f Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Tue, 26 Mar 2024 01:18:05 -0500
Subject: [PATCH 326/404] [IR] Add `m_c_BitwiseLogic` in pattern match; NFC

Just a missing matcher that came up in #73362

Closes #86632
---
 llvm/include/llvm/IR/PatternMatch.h | 16 +++++++++---
 llvm/unittests/IR/PatternMatch.cpp  | 39 +++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 49f9ed5f6e7b..46372c78263a 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -1357,7 +1357,8 @@ m_NUWAddLike(const LHS &L, const RHS &R) {
 //===----------------------------------------------------------------------===//
 // Class that matches a group of binary opcodes.
 //
-template <typename LHS_t, typename RHS_t, typename Predicate>
+template <typename LHS_t, typename RHS_t, typename Predicate,
+          bool Commutable = false>
 struct BinOpPred_match : Predicate {
   LHS_t L;
   RHS_t R;
@@ -1366,8 +1367,10 @@ struct BinOpPred_match : Predicate {
 
   template <typename OpTy> bool match(OpTy *V) {
     if (auto *I = dyn_cast<Instruction>(V))
-      return this->isOpType(I->getOpcode()) && L.match(I->getOperand(0)) &&
-             R.match(I->getOperand(1));
+      return this->isOpType(I->getOpcode()) &&
+             ((L.match(I->getOperand(0)) && R.match(I->getOperand(1))) ||
+              (Commutable && L.match(I->getOperand(1)) &&
+               R.match(I->getOperand(0))));
     return false;
   }
 };
@@ -1434,6 +1437,13 @@ m_BitwiseLogic(const LHS &L, const RHS &R) {
   return BinOpPred_match<LHS, RHS, is_bitwiselogic_op>(L, R);
 }
 
+/// Matches bitwise logic operations in either order.
+template <typename LHS, typename RHS>
+inline BinOpPred_match<LHS, RHS, is_bitwiselogic_op, true>
+m_c_BitwiseLogic(const LHS &L, const RHS &R) {
+  return BinOpPred_match<LHS, RHS, is_bitwiselogic_op, true>(L, R);
+}
+
 /// Matches integer division operations.
 template <typename LHS, typename RHS>
 inline BinOpPred_match<LHS, RHS, is_idiv_op> m_IDiv(const LHS &L,
diff --git a/llvm/unittests/IR/PatternMatch.cpp b/llvm/unittests/IR/PatternMatch.cpp
index 533a30bfba45..a0b873de2d58 100644
--- a/llvm/unittests/IR/PatternMatch.cpp
+++ b/llvm/unittests/IR/PatternMatch.cpp
@@ -494,6 +494,45 @@ TEST_F(PatternMatchTest, Unless) {
   EXPECT_FALSE(m_Unless(m_c_Add(m_Zero(), m_One())).match(X));
 }
 
+TEST_F(PatternMatchTest, BitWise) {
+  Value *Or = IRB.CreateOr(IRB.getInt32(1), IRB.getInt32(0));
+  Value *Xor = IRB.CreateXor(IRB.getInt32(1), IRB.getInt32(0));
+  Value *And = IRB.CreateXor(IRB.getInt32(1), IRB.getInt32(0));
+  Constant *T = IRB.getInt1(true);
+  Constant *F = IRB.getInt1(false);
+  Value *Alloca = IRB.CreateAlloca(IRB.getInt1Ty());
+  Value *X = IRB.CreateLoad(IRB.getInt1Ty(), Alloca);
+  Value *Y = IRB.CreateLoad(IRB.getInt1Ty(), Alloca);
+  Value *LAnd = IRB.CreateSelect(X, Y, F);
+  Value *LOr = IRB.CreateSelect(X, T, Y);
+  Value *Add = IRB.CreateAdd(IRB.getInt32(1), IRB.getInt32(0));
+
+  EXPECT_TRUE(m_BitwiseLogic(m_One(), m_Zero()).match(Or));
+  EXPECT_TRUE(m_BitwiseLogic(m_One(), m_Zero()).match(Xor));
+  EXPECT_TRUE(m_BitwiseLogic(m_One(), m_Zero()).match(And));
+  EXPECT_FALSE(m_BitwiseLogic(m_Value(), m_Value()).match(LAnd));
+  EXPECT_FALSE(m_BitwiseLogic(m_Value(), m_Value()).match(LOr));
+  EXPECT_FALSE(m_BitwiseLogic(m_Value(), m_Value()).match(Add));
+
+  EXPECT_FALSE(m_BitwiseLogic(m_Zero(), m_One()).match(Or));
+  EXPECT_FALSE(m_BitwiseLogic(m_Zero(), m_One()).match(Xor));
+  EXPECT_FALSE(m_BitwiseLogic(m_Zero(), m_One()).match(And));
+
+  EXPECT_TRUE(m_c_BitwiseLogic(m_One(), m_Zero()).match(Or));
+  EXPECT_TRUE(m_c_BitwiseLogic(m_One(), m_Zero()).match(Xor));
+  EXPECT_TRUE(m_c_BitwiseLogic(m_One(), m_Zero()).match(And));
+  EXPECT_FALSE(m_c_BitwiseLogic(m_Value(), m_Value()).match(LAnd));
+  EXPECT_FALSE(m_c_BitwiseLogic(m_Value(), m_Value()).match(LOr));
+  EXPECT_FALSE(m_c_BitwiseLogic(m_Value(), m_Value()).match(Add));
+
+  EXPECT_TRUE(m_c_BitwiseLogic(m_Zero(), m_One()).match(Or));
+  EXPECT_TRUE(m_c_BitwiseLogic(m_Zero(), m_One()).match(Xor));
+  EXPECT_TRUE(m_c_BitwiseLogic(m_Zero(), m_One()).match(And));
+
+  EXPECT_FALSE(m_c_BitwiseLogic(m_One(), m_One()).match(Or));
+  EXPECT_FALSE(m_c_BitwiseLogic(m_Zero(), m_Zero()).match(Xor));
+}
+
 TEST_F(PatternMatchTest, ZExtSExtSelf) {
   LLVMContext &Ctx = IRB.getContext();
 
-- 
GitLab


From cf6e62d8c308c7bf2885c422983c80842d735456 Mon Sep 17 00:00:00 2001
From: Alex Zinenko <zinenko@google.com>
Date: Tue, 26 Mar 2024 18:47:40 +0000
Subject: [PATCH 327/404] [mlir][doc] NFC fix incorrect filename in td match
 interface

---
 mlir/include/mlir/Dialect/Transform/IR/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/include/mlir/Dialect/Transform/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/Transform/IR/CMakeLists.txt
index 3ccac2f5c165..e90d04a20244 100644
--- a/mlir/include/mlir/Dialect/Transform/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Transform/IR/CMakeLists.txt
@@ -26,5 +26,5 @@ add_mlir_doc(TransformOps TransformOps Dialects/ -gen-op-doc -dialect=transform)
 
 add_mlir_interface(MatchInterfaces)
 add_dependencies(MLIRMatchInterfacesIncGen MLIRTransformInterfacesIncGen)
-add_mlir_doc(TransformInterfaces MatchOpInterfaces Dialects/ -gen-op-interface-docs)
+add_mlir_doc(MatchInterfaces MatchOpInterfaces Dialects/ -gen-op-interface-docs)
 
-- 
GitLab


From 2fbc40d36dd8e21fab4cc6cb43984baaa39bcd59 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 26 Mar 2024 11:50:41 -0700
Subject: [PATCH 328/404] [RISCV] Split compound if statement to fix a crash.

We're not allowed to call getELEN when the vector extension
is not enabled. If we're looking at a vector type, isTypeLegal would
only return true if the vector extensions are enabled. So early out
for non-vector types before we call isTypeLegal and getELEN.
---
 llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 4bcb9acdae19..df289be18932 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -898,12 +898,15 @@ InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
                                                TTI::TargetCostKind CostKind,
                                                const Instruction *I) {
   bool IsVectorType = isa<VectorType>(Dst) && isa<VectorType>(Src);
+  if (!IsVectorType)
+    return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
+
   bool IsTypeLegal = isTypeLegal(Src) && isTypeLegal(Dst) &&
                      (Src->getScalarSizeInBits() <= ST->getELen()) &&
                      (Dst->getScalarSizeInBits() <= ST->getELen());
 
   // FIXME: Need to compute legalizing cost for illegal types.
-  if (!IsVectorType || !IsTypeLegal)
+  if (!IsTypeLegal)
     return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
 
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
-- 
GitLab


From 148a55795de7ac465a8e494d5d382e100da643f6 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Tue, 26 Mar 2024 13:54:26 -0500
Subject: [PATCH 329/404] [flang][OpenMP] Make OpenMP clause representation
 language-agnostic (#86289)

The clause templates defined in ClauseT.h were originally based on
flang's parse tree nodes. Since those representations are going to be
reused for clang (together with the clause splitting code), it makes
sense to separate them from flang, and instead have them based on the
actual OpenMP spec (v5.2).

The member names in the templates follow the naming presented in the
spec, and the representation (e.g. members) is derived from the clause
definitions as described in the spec.

Since the representations of some clauses has changed (while preserving
the information), the current code using the clauses (especially the
code converting parser::OmpClause to omp::Clause) needs to be adjusted.

This patch does not make any functional changes.
---
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp    |  184 ++-
 flang/lib/Lower/OpenMP/ClauseProcessor.h      |    3 +-
 flang/lib/Lower/OpenMP/ClauseT.h              |  714 -----------
 flang/lib/Lower/OpenMP/Clauses.cpp            |  784 +++++++++---
 flang/lib/Lower/OpenMP/Clauses.h              |  245 ++--
 .../lib/Lower/OpenMP/DataSharingProcessor.cpp |    8 +-
 flang/lib/Lower/OpenMP/OpenMP.cpp             |   26 +-
 flang/lib/Lower/OpenMP/ReductionProcessor.cpp |    6 +-
 llvm/include/llvm/Frontend/OpenMP/ClauseT.h   | 1123 +++++++++++++++++
 9 files changed, 2011 insertions(+), 1082 deletions(-)
 delete mode 100644 flang/lib/Lower/OpenMP/ClauseT.h
 create mode 100644 llvm/include/llvm/Frontend/OpenMP/ClauseT.h

diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index 52c3479b1ea9..0a57a1496289 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -31,57 +31,33 @@ static void checkMapType(mlir::Location location, mlir::Type type) {
 }
 
 static mlir::omp::ScheduleModifier
-translateScheduleModifier(const omp::clause::Schedule::ModType &m) {
+translateScheduleModifier(const omp::clause::Schedule::OrderingModifier &m) {
   switch (m) {
-  case omp::clause::Schedule::ModType::Monotonic:
+  case omp::clause::Schedule::OrderingModifier::Monotonic:
     return mlir::omp::ScheduleModifier::monotonic;
-  case omp::clause::Schedule::ModType::Nonmonotonic:
+  case omp::clause::Schedule::OrderingModifier::Nonmonotonic:
     return mlir::omp::ScheduleModifier::nonmonotonic;
-  case omp::clause::Schedule::ModType::Simd:
-    return mlir::omp::ScheduleModifier::simd;
   }
   return mlir::omp::ScheduleModifier::none;
 }
 
 static mlir::omp::ScheduleModifier
 getScheduleModifier(const omp::clause::Schedule &clause) {
-  using ScheduleModifier = omp::clause::Schedule::ScheduleModifier;
-  const auto &modifier = std::get<std::optional<ScheduleModifier>>(clause.t);
-  // The input may have the modifier any order, so we look for one that isn't
-  // SIMD. If modifier is not set at all, fall down to the bottom and return
-  // "none".
-  if (modifier) {
-    using ModType = omp::clause::Schedule::ModType;
-    const auto &modType1 = std::get<ModType>(modifier->t);
-    if (modType1 == ModType::Simd) {
-      const auto &modType2 = std::get<std::optional<ModType>>(modifier->t);
-      if (modType2 && *modType2 != ModType::Simd)
-        return translateScheduleModifier(*modType2);
-      return mlir::omp::ScheduleModifier::none;
-    }
-
-    return translateScheduleModifier(modType1);
-  }
+  using Schedule = omp::clause::Schedule;
+  const auto &modifier =
+      std::get<std::optional<Schedule::OrderingModifier>>(clause.t);
+  if (modifier)
+    return translateScheduleModifier(*modifier);
   return mlir::omp::ScheduleModifier::none;
 }
 
 static mlir::omp::ScheduleModifier
 getSimdModifier(const omp::clause::Schedule &clause) {
-  using ScheduleModifier = omp::clause::Schedule::ScheduleModifier;
-  const auto &modifier = std::get<std::optional<ScheduleModifier>>(clause.t);
-  // Either of the two possible modifiers in the input can be the SIMD modifier,
-  // so look in either one, and return simd if we find one. Not found = return
-  // "none".
-  if (modifier) {
-    using ModType = omp::clause::Schedule::ModType;
-    const auto &modType1 = std::get<ModType>(modifier->t);
-    if (modType1 == ModType::Simd)
-      return mlir::omp::ScheduleModifier::simd;
-
-    const auto &modType2 = std::get<std::optional<ModType>>(modifier->t);
-    if (modType2 && *modType2 == ModType::Simd)
-      return mlir::omp::ScheduleModifier::simd;
-  }
+  using Schedule = omp::clause::Schedule;
+  const auto &modifier =
+      std::get<std::optional<Schedule::ChunkModifier>>(clause.t);
+  if (modifier && *modifier == Schedule::ChunkModifier::Simd)
+    return mlir::omp::ScheduleModifier::simd;
   return mlir::omp::ScheduleModifier::none;
 }
 
@@ -94,36 +70,31 @@ genAllocateClause(Fortran::lower::AbstractConverter &converter,
   mlir::Location currentLocation = converter.getCurrentLocation();
   Fortran::lower::StatementContext stmtCtx;
 
-  const omp::ObjectList &objectList = std::get<omp::ObjectList>(clause.t);
-  const auto &modifier =
-      std::get<std::optional<omp::clause::Allocate::Modifier>>(clause.t);
-
-  // If the allocate modifier is present, check if we only use the allocator
-  // submodifier.  ALIGN in this context is unimplemented
-  const bool onlyAllocator =
-      modifier &&
-      std::holds_alternative<omp::clause::Allocate::Modifier::Allocator>(
-          modifier->u);
+  auto &objects = std::get<omp::ObjectList>(clause.t);
 
-  if (modifier && !onlyAllocator) {
+  using Allocate = omp::clause::Allocate;
+  // ALIGN in this context is unimplemented
+  if (std::get<std::optional<Allocate::AlignModifier>>(clause.t))
     TODO(currentLocation, "OmpAllocateClause ALIGN modifier");
-  }
 
   // Check if allocate clause has allocator specified. If so, add it
   // to list of allocators, otherwise, add default allocator to
   // list of allocators.
-  if (onlyAllocator) {
-    const auto &value =
-        std::get<omp::clause::Allocate::Modifier::Allocator>(modifier->u);
-    mlir::Value operand =
-        fir::getBase(converter.genExprValue(value.v, stmtCtx));
-    allocatorOperands.append(objectList.size(), operand);
+  using SimpleModifier = Allocate::AllocatorSimpleModifier;
+  using ComplexModifier = Allocate::AllocatorComplexModifier;
+  if (auto &mod = std::get<std::optional<SimpleModifier>>(clause.t)) {
+    mlir::Value operand = fir::getBase(converter.genExprValue(*mod, stmtCtx));
+    allocatorOperands.append(objects.size(), operand);
+  } else if (auto &mod = std::get<std::optional<ComplexModifier>>(clause.t)) {
+    mlir::Value operand = fir::getBase(converter.genExprValue(mod->v, stmtCtx));
+    allocatorOperands.append(objects.size(), operand);
   } else {
     mlir::Value operand = firOpBuilder.createIntegerConstant(
         currentLocation, firOpBuilder.getI32Type(), 1);
-    allocatorOperands.append(objectList.size(), operand);
+    allocatorOperands.append(objects.size(), operand);
   }
-  genObjectList(objectList, converter, allocateOperands);
+
+  genObjectList(objects, converter, allocateOperands);
 }
 
 static mlir::omp::ClauseProcBindKindAttr
@@ -131,16 +102,16 @@ genProcBindKindAttr(fir::FirOpBuilder &firOpBuilder,
                     const omp::clause::ProcBind &clause) {
   mlir::omp::ClauseProcBindKind procBindKind;
   switch (clause.v) {
-  case omp::clause::ProcBind::Type::Master:
+  case omp::clause::ProcBind::AffinityPolicy::Master:
     procBindKind = mlir::omp::ClauseProcBindKind::Master;
     break;
-  case omp::clause::ProcBind::Type::Close:
+  case omp::clause::ProcBind::AffinityPolicy::Close:
     procBindKind = mlir::omp::ClauseProcBindKind::Close;
     break;
-  case omp::clause::ProcBind::Type::Spread:
+  case omp::clause::ProcBind::AffinityPolicy::Spread:
     procBindKind = mlir::omp::ClauseProcBindKind::Spread;
     break;
-  case omp::clause::ProcBind::Type::Primary:
+  case omp::clause::ProcBind::AffinityPolicy::Primary:
     procBindKind = mlir::omp::ClauseProcBindKind::Primary;
     break;
   }
@@ -150,21 +121,22 @@ genProcBindKindAttr(fir::FirOpBuilder &firOpBuilder,
 
 static mlir::omp::ClauseTaskDependAttr
 genDependKindAttr(fir::FirOpBuilder &firOpBuilder,
-                  const omp::clause::Depend &clause) {
+                  const omp::clause::Depend::TaskDependenceType kind) {
   mlir::omp::ClauseTaskDepend pbKind;
-  const auto &inOut = std::get<omp::clause::Depend::InOut>(clause.u);
-  switch (std::get<omp::clause::Depend::Type>(inOut.t)) {
-  case omp::clause::Depend::Type::In:
+  switch (kind) {
+  case omp::clause::Depend::TaskDependenceType::In:
     pbKind = mlir::omp::ClauseTaskDepend::taskdependin;
     break;
-  case omp::clause::Depend::Type::Out:
+  case omp::clause::Depend::TaskDependenceType::Out:
     pbKind = mlir::omp::ClauseTaskDepend::taskdependout;
     break;
-  case omp::clause::Depend::Type::Inout:
+  case omp::clause::Depend::TaskDependenceType::Inout:
     pbKind = mlir::omp::ClauseTaskDepend::taskdependinout;
     break;
-  default:
-    llvm_unreachable("unknown parser task dependence type");
+  case omp::clause::Depend::TaskDependenceType::Mutexinoutset:
+  case omp::clause::Depend::TaskDependenceType::Inoutset:
+  case omp::clause::Depend::TaskDependenceType::Depobj:
+    llvm_unreachable("unhandled parser task dependence type");
     break;
   }
   return mlir::omp::ClauseTaskDependAttr::get(firOpBuilder.getContext(),
@@ -295,16 +267,16 @@ bool ClauseProcessor::processDefault() const {
   if (auto *clause = findUniqueClause<omp::clause::Default>()) {
     // Private, Firstprivate, Shared, None
     switch (clause->v) {
-    case omp::clause::Default::Type::Shared:
-    case omp::clause::Default::Type::None:
+    case omp::clause::Default::DataSharingAttribute::Shared:
+    case omp::clause::Default::DataSharingAttribute::None:
       // Default clause with shared or none do not require any handling since
       // Shared is the default behavior in the IR and None is only required
       // for semantic checks.
       break;
-    case omp::clause::Default::Type::Private:
+    case omp::clause::Default::DataSharingAttribute::Private:
       // TODO Support default(private)
       break;
-    case omp::clause::Default::Type::Firstprivate:
+    case omp::clause::Default::DataSharingAttribute::Firstprivate:
       // TODO Support default(firstprivate)
       break;
     }
@@ -337,13 +309,13 @@ bool ClauseProcessor::processDeviceType(
   if (auto *clause = findUniqueClause<omp::clause::DeviceType>()) {
     // Case: declare target ... device_type(any | host | nohost)
     switch (clause->v) {
-    case omp::clause::DeviceType::Type::Nohost:
+    case omp::clause::DeviceType::DeviceTypeDescription::Nohost:
       result = mlir::omp::DeclareTargetDeviceType::nohost;
       break;
-    case omp::clause::DeviceType::Type::Host:
+    case omp::clause::DeviceType::DeviceTypeDescription::Host:
       result = mlir::omp::DeclareTargetDeviceType::host;
       break;
-    case omp::clause::DeviceType::Type::Any:
+    case omp::clause::DeviceType::DeviceTypeDescription::Any:
       result = mlir::omp::DeclareTargetDeviceType::any;
       break;
     }
@@ -391,7 +363,9 @@ bool ClauseProcessor::processNumTeams(Fortran::lower::StatementContext &stmtCtx,
   // TODO Get lower and upper bounds for num_teams when parser is updated to
   // accept both.
   if (auto *clause = findUniqueClause<omp::clause::NumTeams>()) {
-    result = fir::getBase(converter.genExprValue(clause->v, stmtCtx));
+    // auto lowerBound = std::get<std::optional<ExprTy>>(clause->t);
+    auto &upperBound = std::get<ExprTy>(clause->t);
+    result = fir::getBase(converter.genExprValue(upperBound, stmtCtx));
     return true;
   }
   return false;
@@ -456,24 +430,23 @@ bool ClauseProcessor::processSchedule(
   if (auto *clause = findUniqueClause<omp::clause::Schedule>()) {
     fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
     mlir::MLIRContext *context = firOpBuilder.getContext();
-    const auto &scheduleType =
-        std::get<omp::clause::Schedule::ScheduleType>(clause->t);
+    const auto &scheduleType = std::get<omp::clause::Schedule::Kind>(clause->t);
 
     mlir::omp::ClauseScheduleKind scheduleKind;
     switch (scheduleType) {
-    case omp::clause::Schedule::ScheduleType::Static:
+    case omp::clause::Schedule::Kind::Static:
       scheduleKind = mlir::omp::ClauseScheduleKind::Static;
       break;
-    case omp::clause::Schedule::ScheduleType::Dynamic:
+    case omp::clause::Schedule::Kind::Dynamic:
       scheduleKind = mlir::omp::ClauseScheduleKind::Dynamic;
       break;
-    case omp::clause::Schedule::ScheduleType::Guided:
+    case omp::clause::Schedule::Kind::Guided:
       scheduleKind = mlir::omp::ClauseScheduleKind::Guided;
       break;
-    case omp::clause::Schedule::ScheduleType::Auto:
+    case omp::clause::Schedule::Kind::Auto:
       scheduleKind = mlir::omp::ClauseScheduleKind::Auto;
       break;
-    case omp::clause::Schedule::ScheduleType::Runtime:
+    case omp::clause::Schedule::Kind::Runtime:
       scheduleKind = mlir::omp::ClauseScheduleKind::Runtime;
       break;
     }
@@ -749,13 +722,15 @@ bool ClauseProcessor::processDepend(
   return findRepeatableClause<omp::clause::Depend>(
       [&](const omp::clause::Depend &clause,
           const Fortran::parser::CharBlock &) {
-        assert(std::holds_alternative<omp::clause::Depend::InOut>(clause.u) &&
-               "Only InOut is handled at the moment");
-        const auto &inOut = std::get<omp::clause::Depend::InOut>(clause.u);
-        const auto &objects = std::get<omp::ObjectList>(inOut.t);
+        using Depend = omp::clause::Depend;
+        assert(std::holds_alternative<Depend::WithLocators>(clause.u) &&
+               "Only the modern form is handled at the moment");
+        auto &modern = std::get<Depend::WithLocators>(clause.u);
+        auto kind = std::get<Depend::TaskDependenceType>(modern.t);
+        auto &objects = std::get<omp::ObjectList>(modern.t);
 
         mlir::omp::ClauseTaskDependAttr dependTypeOperand =
-            genDependKindAttr(firOpBuilder, clause);
+            genDependKindAttr(firOpBuilder, kind);
         dependTypeOperands.append(objects.size(), dependTypeOperand);
 
         for (const omp::Object &object : objects) {
@@ -844,39 +819,42 @@ bool ClauseProcessor::processMap(
           const Fortran::parser::CharBlock &source) {
         using Map = omp::clause::Map;
         mlir::Location clauseLocation = converter.genLocation(source);
-        const auto &oMapType = std::get<std::optional<Map::MapType>>(clause.t);
+        const auto &mapType = std::get<std::optional<Map::MapType>>(clause.t);
         llvm::omp::OpenMPOffloadMappingFlags mapTypeBits =
             llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE;
         // If the map type is specified, then process it else Tofrom is the
         // default.
-        if (oMapType) {
-          const Map::MapType::Type &mapType =
-              std::get<Map::MapType::Type>(oMapType->t);
-          switch (mapType) {
-          case Map::MapType::Type::To:
+        if (mapType) {
+          switch (*mapType) {
+          case Map::MapType::To:
             mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO;
             break;
-          case Map::MapType::Type::From:
+          case Map::MapType::From:
             mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM;
             break;
-          case Map::MapType::Type::Tofrom:
+          case Map::MapType::Tofrom:
             mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |
                            llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM;
             break;
-          case Map::MapType::Type::Alloc:
-          case Map::MapType::Type::Release:
+          case Map::MapType::Alloc:
+          case Map::MapType::Release:
             // alloc and release is the default map_type for the Target Data
             // Ops, i.e. if no bits for map_type is supplied then alloc/release
             // is implicitly assumed based on the target directive. Default
             // value for Target Data and Enter Data is alloc and for Exit Data
             // it is release.
             break;
-          case Map::MapType::Type::Delete:
+          case Map::MapType::Delete:
             mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
           }
 
-          if (std::get<std::optional<Map::MapType::Always>>(oMapType->t))
-            mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
+          auto &modTypeMods =
+              std::get<std::optional<Map::MapTypeModifiers>>(clause.t);
+          if (modTypeMods) {
+            if (llvm::is_contained(*modTypeMods, Map::MapTypeModifier::Always))
+              mapTypeBits |=
+                  llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
+          }
         } else {
           mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |
                          llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM;
@@ -972,7 +950,7 @@ bool ClauseProcessor::processTo(
   return findRepeatableClause<omp::clause::To>(
       [&](const omp::clause::To &clause, const Fortran::parser::CharBlock &) {
         // Case: declare target to(func, var1, var2)...
-        gatherFuncAndVarSyms(clause.v,
+        gatherFuncAndVarSyms(std::get<ObjectList>(clause.t),
                              mlir::omp::DeclareTargetCaptureClause::to, result);
       });
 }
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index 8582716e6b9a..c0c603feb296 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -200,7 +200,8 @@ bool ClauseProcessor::processMotionClauses(
                 ? llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO
                 : llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM;
 
-        for (const omp::Object &object : clause.v) {
+        auto &objects = std::get<ObjectList>(clause.t);
+        for (const omp::Object &object : objects) {
           llvm::SmallVector<mlir::Value> bounds;
           std::stringstream asFortran;
           Fortran::lower::AddrAndBoundsInfo info =
diff --git a/flang/lib/Lower/OpenMP/ClauseT.h b/flang/lib/Lower/OpenMP/ClauseT.h
deleted file mode 100644
index 2aae29af2921..000000000000
--- a/flang/lib/Lower/OpenMP/ClauseT.h
+++ /dev/null
@@ -1,714 +0,0 @@
-//===- ClauseT -- clause template definitions -----------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-#ifndef FORTRAN_LOWER_OPENMP_CLAUSET_H
-#define FORTRAN_LOWER_OPENMP_CLAUSET_H
-
-#include "flang/Parser/parse-tree.h" // For enum reuse
-
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/Support/raw_ostream.h"
-
-#include <algorithm>
-#include <iterator>
-#include <optional>
-#include <tuple>
-#include <type_traits>
-#include <utility>
-#include <variant>
-
-#include "llvm/Frontend/OpenMP/OMP.h.inc"
-
-namespace tomp {
-
-template <typename T>
-using ListT = llvm::SmallVector<T, 0>;
-
-// A specialization of ObjectT<Id, Expr> must provide the following definitions:
-// {
-//    using IdType = Id;
-//    using ExprType = Expr;
-//
-//    auto id() const -> Id {
-//      return the identifier of the object (for use in tests for
-//         presence/absence of the object)
-//    }
-//
-//    auto ref() const -> const Expr& {
-//      return the expression accessing (referencing) the object
-//    }
-// }
-//
-// For example, the ObjectT instance created for "var[x+1]" would have
-// the `id()` return the identifier for `var`, and the `ref()` return the
-// representation of the array-access `var[x+1]`.
-template <typename Id, typename Expr>
-struct ObjectT;
-
-template <typename I, typename E>
-using ObjectListT = ListT<ObjectT<I, E>>;
-
-namespace clause {
-// Helper objects
-
-template <typename I, typename E>
-struct DefinedOperatorT {
-  struct DefinedOpName {
-    using WrapperTrait = std::true_type;
-    ObjectT<I, E> v;
-  };
-  using IntrinsicOperator = Fortran::parser::DefinedOperator::IntrinsicOperator;
-  using UnionTrait = std::true_type;
-  std::variant<DefinedOpName, IntrinsicOperator> u;
-};
-
-template <typename I, typename E>
-struct ProcedureDesignatorT {
-  using WrapperTrait = std::true_type;
-  ObjectT<I, E> v;
-};
-
-template <typename I, typename E>
-struct ReductionOperatorT {
-  using UnionTrait = std::true_type;
-  std::variant<DefinedOperatorT<I, E>, ProcedureDesignatorT<I, E>> u;
-};
-
-template <typename I, typename E>
-struct AcqRelT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct AcquireT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct AdjustArgsT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct AffinityT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct AlignT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct AppendArgsT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct AtT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct BindT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct CancellationConstructTypeT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct CaptureT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct CompareT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct DepobjT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct DestroyT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct DetachT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct DoacrossT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct DynamicAllocatorsT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct ExclusiveT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct FailT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct FlushT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct FullT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct InbranchT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct InclusiveT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct IndirectT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct InitT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct MatchT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct MemoryOrderT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct MergeableT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct MessageT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct NogroupT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct NotinbranchT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct NowaitT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct OmpxAttributeT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct OmpxBareT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct ReadT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct RelaxedT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct ReleaseT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct ReverseOffloadT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct SeqCstT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct SeverityT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct SimdT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct ThreadprivateT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct ThreadsT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct UnifiedAddressT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct UnifiedSharedMemoryT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct UnknownT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct UntiedT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct UpdateT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct UseT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct UsesAllocatorsT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct WeakT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct WhenT {
-  using EmptyTrait = std::true_type;
-};
-template <typename I, typename E>
-struct WriteT {
-  using EmptyTrait = std::true_type;
-};
-
-template <typename I, typename E>
-struct AlignedT {
-  using TupleTrait = std::true_type;
-  std::tuple<ObjectListT<I, E>, std::optional<E>> t;
-};
-
-template <typename I, typename E>
-struct AllocateT {
-  struct Modifier {
-    struct Allocator {
-      using WrapperTrait = std::true_type;
-      E v;
-    };
-    struct Align {
-      using WrapperTrait = std::true_type;
-      E v;
-    };
-    struct ComplexModifier {
-      using TupleTrait = std::true_type;
-      std::tuple<Allocator, Align> t;
-    };
-    using UnionTrait = std::true_type;
-    std::variant<Allocator, ComplexModifier, Align> u;
-  };
-  using TupleTrait = std::true_type;
-  std::tuple<std::optional<Modifier>, ObjectListT<I, E>> t;
-};
-
-template <typename I, typename E>
-struct AllocatorT {
-  using WrapperTrait = std::true_type;
-  E v;
-};
-
-template <typename I, typename E>
-struct AtomicDefaultMemOrderT {
-  using WrapperTrait = std::true_type;
-  using OmpAtomicDefaultMemOrderType =
-      Fortran::common::OmpAtomicDefaultMemOrderType;
-  OmpAtomicDefaultMemOrderType v;
-};
-
-template <typename I, typename E>
-struct CollapseT {
-  using WrapperTrait = std::true_type;
-  E v;
-};
-
-template <typename I, typename E>
-struct CopyinT {
-  using WrapperTrait = std::true_type;
-  ObjectListT<I, E> v;
-};
-
-template <typename I, typename E>
-struct CopyprivateT {
-  using WrapperTrait = std::true_type;
-  ObjectListT<I, E> v;
-};
-
-template <typename I, typename E>
-struct DefaultmapT {
-  using ImplicitBehavior =
-      Fortran::parser::OmpDefaultmapClause::ImplicitBehavior;
-  using VariableCategory =
-      Fortran::parser::OmpDefaultmapClause::VariableCategory;
-  using TupleTrait = std::true_type;
-  std::tuple<ImplicitBehavior, std::optional<VariableCategory>> t;
-};
-
-template <typename I, typename E>
-struct DefaultT {
-  using Type = Fortran::parser::OmpDefaultClause::Type;
-  using WrapperTrait = std::true_type;
-  Type v;
-};
-
-template <typename I, typename E>
-struct DependT {
-  struct Source {
-    using EmptyTrait = std::true_type;
-  };
-  struct Sink {
-    using Length = std::tuple<DefinedOperatorT<I, E>, E>;
-    using Vec = std::tuple<ObjectT<I, E>, std::optional<Length>>;
-    using WrapperTrait = std::true_type;
-    ListT<Vec> v;
-  };
-  using Type = Fortran::parser::OmpDependenceType::Type;
-  struct InOut {
-    using TupleTrait = std::true_type;
-    std::tuple<Type, ObjectListT<I, E>> t;
-  };
-  using UnionTrait = std::true_type;
-  std::variant<Source, Sink, InOut> u;
-};
-
-template <typename I, typename E>
-struct DeviceT {
-  using DeviceModifier = Fortran::parser::OmpDeviceClause::DeviceModifier;
-  using TupleTrait = std::true_type;
-  std::tuple<std::optional<DeviceModifier>, E> t;
-};
-
-template <typename I, typename E>
-struct DeviceTypeT {
-  using Type = Fortran::parser::OmpDeviceTypeClause::Type;
-  using WrapperTrait = std::true_type;
-  Type v;
-};
-
-template <typename I, typename E>
-struct DistScheduleT {
-  using WrapperTrait = std::true_type;
-  std::optional<E> v;
-};
-
-template <typename I, typename E>
-struct EnterT {
-  using WrapperTrait = std::true_type;
-  ObjectListT<I, E> v;
-};
-
-template <typename I, typename E>
-struct FilterT {
-  using WrapperTrait = std::true_type;
-  E v;
-};
-
-template <typename I, typename E>
-struct FinalT {
-  using WrapperTrait = std::true_type;
-  E v;
-};
-
-template <typename I, typename E>
-struct FirstprivateT {
-  using WrapperTrait = std::true_type;
-  ObjectListT<I, E> v;
-};
-
-template <typename I, typename E>
-struct FromT {
-  using WrapperTrait = std::true_type;
-  ObjectListT<I, E> v;
-};
-
-template <typename I, typename E>
-struct GrainsizeT {
-  using WrapperTrait = std::true_type;
-  E v;
-};
-
-template <typename I, typename E>
-struct HasDeviceAddrT {
-  using WrapperTrait = std::true_type;
-  ObjectListT<I, E> v;
-};
-
-template <typename I, typename E>
-struct HintT {
-  using WrapperTrait = std::true_type;
-  E v;
-};
-
-template <typename I, typename E>
-struct IfT {
-  using DirectiveNameModifier =
-      Fortran::parser::OmpIfClause::DirectiveNameModifier;
-  using TupleTrait = std::true_type;
-  std::tuple<std::optional<DirectiveNameModifier>, E> t;
-};
-
-template <typename I, typename E>
-struct InReductionT {
-  using TupleTrait = std::true_type;
-  std::tuple<ReductionOperatorT<I, E>, ObjectListT<I, E>> t;
-};
-
-template <typename I, typename E>
-struct IsDevicePtrT {
-  using WrapperTrait = std::true_type;
-  ObjectListT<I, E> v;
-};
-
-template <typename I, typename E>
-struct LastprivateT {
-  using WrapperTrait = std::true_type;
-  ObjectListT<I, E> v;
-};
-
-template <typename I, typename E>
-struct LinearT {
-  struct Modifier {
-    using Type = Fortran::parser::OmpLinearModifier::Type;
-    using WrapperTrait = std::true_type;
-    Type v;
-  };
-  using TupleTrait = std::true_type;
-  std::tuple<std::optional<Modifier>, ObjectListT<I, E>, std::optional<E>> t;
-};
-
-template <typename I, typename E>
-struct LinkT {
-  using WrapperTrait = std::true_type;
-  ObjectListT<I, E> v;
-};
-
-template <typename I, typename E>
-struct MapT {
-  struct MapType {
-    struct Always {
-      using EmptyTrait = std::true_type;
-    };
-    using Type = Fortran::parser::OmpMapType::Type;
-    using TupleTrait = std::true_type;
-    std::tuple<std::optional<Always>, Type> t;
-  };
-  using TupleTrait = std::true_type;
-  std::tuple<std::optional<MapType>, ObjectListT<I, E>> t;
-};
-
-template <typename I, typename E>
-struct NocontextT {
-  using WrapperTrait = std::true_type;
-  E v;
-};
-
-template <typename I, typename E>
-struct NontemporalT {
-  using WrapperTrait = std::true_type;
-  ObjectListT<I, E> v;
-};
-
-template <typename I, typename E>
-struct NovariantsT {
-  using WrapperTrait = std::true_type;
-  E v;
-};
-
-template <typename I, typename E>
-struct NumTasksT {
-  using WrapperTrait = std::true_type;
-  E v;
-};
-
-template <typename I, typename E>
-struct NumTeamsT {
-  using WrapperTrait = std::true_type;
-  E v;
-};
-
-template <typename I, typename E>
-struct NumThreadsT {
-  using WrapperTrait = std::true_type;
-  E v;
-};
-
-template <typename I, typename E>
-struct OmpxDynCgroupMemT {
-  using WrapperTrait = std::true_type;
-  E v;
-};
-
-template <typename I, typename E>
-struct OrderedT {
-  using WrapperTrait = std::true_type;
-  std::optional<E> v;
-};
-
-template <typename I, typename E>
-struct OrderT {
-  using Kind = Fortran::parser::OmpOrderModifier::Kind;
-  using Type = Fortran::parser::OmpOrderClause::Type;
-  using TupleTrait = std::true_type;
-  std::tuple<std::optional<Kind>, Type> t;
-};
-
-template <typename I, typename E>
-struct PartialT {
-  using WrapperTrait = std::true_type;
-  std::optional<E> v;
-};
-
-template <typename I, typename E>
-struct PriorityT {
-  using WrapperTrait = std::true_type;
-  E v;
-};
-
-template <typename I, typename E>
-struct PrivateT {
-  using WrapperTrait = std::true_type;
-  ObjectListT<I, E> v;
-};
-
-template <typename I, typename E>
-struct ProcBindT {
-  using Type = Fortran::parser::OmpProcBindClause::Type;
-  using WrapperTrait = std::true_type;
-  Type v;
-};
-
-template <typename I, typename E>
-struct ReductionT {
-  using TupleTrait = std::true_type;
-  std::tuple<ReductionOperatorT<I, E>, ObjectListT<I, E>> t;
-};
-
-template <typename I, typename E>
-struct SafelenT {
-  using WrapperTrait = std::true_type;
-  E v;
-};
-
-template <typename I, typename E>
-struct ScheduleT {
-  using ModType = Fortran::parser::OmpScheduleModifierType::ModType;
-  struct ScheduleModifier {
-    using TupleTrait = std::true_type;
-    std::tuple<ModType, std::optional<ModType>> t;
-  };
-  using ScheduleType = Fortran::parser::OmpScheduleClause::ScheduleType;
-  using TupleTrait = std::true_type;
-  std::tuple<std::optional<ScheduleModifier>, ScheduleType, std::optional<E>> t;
-};
-
-template <typename I, typename E>
-struct SharedT {
-  using WrapperTrait = std::true_type;
-  ObjectListT<I, E> v;
-};
-
-template <typename I, typename E>
-struct SimdlenT {
-  using WrapperTrait = std::true_type;
-  E v;
-};
-
-template <typename I, typename E>
-struct SizesT {
-  using WrapperTrait = std::true_type;
-  ListT<E> v;
-};
-
-template <typename I, typename E>
-struct TaskReductionT {
-  using TupleTrait = std::true_type;
-  std::tuple<ReductionOperatorT<I, E>, ObjectListT<I, E>> t;
-};
-
-template <typename I, typename E>
-struct ThreadLimitT {
-  using WrapperTrait = std::true_type;
-  E v;
-};
-
-template <typename I, typename E>
-struct ToT {
-  using WrapperTrait = std::true_type;
-  ObjectListT<I, E> v;
-};
-
-template <typename I, typename E>
-struct UniformT {
-  using WrapperTrait = std::true_type;
-  ObjectListT<I, E> v;
-};
-
-template <typename I, typename E>
-struct UseDeviceAddrT {
-  using WrapperTrait = std::true_type;
-  ObjectListT<I, E> v;
-};
-
-template <typename I, typename E>
-struct UseDevicePtrT {
-  using WrapperTrait = std::true_type;
-  ObjectListT<I, E> v;
-};
-
-template <typename I, typename E>
-using UnionOfAllClausesT = std::variant<
-    AcqRelT<I, E>, AcquireT<I, E>, AdjustArgsT<I, E>, AffinityT<I, E>,
-    AlignT<I, E>, AlignedT<I, E>, AllocateT<I, E>, AllocatorT<I, E>,
-    AppendArgsT<I, E>, AtT<I, E>, AtomicDefaultMemOrderT<I, E>, BindT<I, E>,
-    CancellationConstructTypeT<I, E>, CaptureT<I, E>, CollapseT<I, E>,
-    CompareT<I, E>, CopyprivateT<I, E>, CopyinT<I, E>, DefaultT<I, E>,
-    DefaultmapT<I, E>, DependT<I, E>, DepobjT<I, E>, DestroyT<I, E>,
-    DetachT<I, E>, DeviceT<I, E>, DeviceTypeT<I, E>, DistScheduleT<I, E>,
-    DoacrossT<I, E>, DynamicAllocatorsT<I, E>, EnterT<I, E>, ExclusiveT<I, E>,
-    FailT<I, E>, FilterT<I, E>, FinalT<I, E>, FirstprivateT<I, E>, FlushT<I, E>,
-    FromT<I, E>, FullT<I, E>, GrainsizeT<I, E>, HasDeviceAddrT<I, E>,
-    HintT<I, E>, IfT<I, E>, InReductionT<I, E>, InbranchT<I, E>,
-    InclusiveT<I, E>, IndirectT<I, E>, InitT<I, E>, IsDevicePtrT<I, E>,
-    LastprivateT<I, E>, LinearT<I, E>, LinkT<I, E>, MapT<I, E>, MatchT<I, E>,
-    MemoryOrderT<I, E>, MergeableT<I, E>, MessageT<I, E>, NogroupT<I, E>,
-    NowaitT<I, E>, NocontextT<I, E>, NontemporalT<I, E>, NotinbranchT<I, E>,
-    NovariantsT<I, E>, NumTasksT<I, E>, NumTeamsT<I, E>, NumThreadsT<I, E>,
-    OmpxAttributeT<I, E>, OmpxDynCgroupMemT<I, E>, OmpxBareT<I, E>,
-    OrderT<I, E>, OrderedT<I, E>, PartialT<I, E>, PriorityT<I, E>,
-    PrivateT<I, E>, ProcBindT<I, E>, ReadT<I, E>, ReductionT<I, E>,
-    RelaxedT<I, E>, ReleaseT<I, E>, ReverseOffloadT<I, E>, SafelenT<I, E>,
-    ScheduleT<I, E>, SeqCstT<I, E>, SeverityT<I, E>, SharedT<I, E>, SimdT<I, E>,
-    SimdlenT<I, E>, SizesT<I, E>, TaskReductionT<I, E>, ThreadLimitT<I, E>,
-    ThreadprivateT<I, E>, ThreadsT<I, E>, ToT<I, E>, UnifiedAddressT<I, E>,
-    UnifiedSharedMemoryT<I, E>, UniformT<I, E>, UnknownT<I, E>, UntiedT<I, E>,
-    UpdateT<I, E>, UseT<I, E>, UseDeviceAddrT<I, E>, UseDevicePtrT<I, E>,
-    UsesAllocatorsT<I, E>, WeakT<I, E>, WhenT<I, E>, WriteT<I, E>>;
-} // namespace clause
-
-template <typename Id, typename Expr>
-struct ClauseT {
-  llvm::omp::Clause id; // The numeric id of the clause
-  using UnionTrait = std::true_type;
-  clause::UnionOfAllClausesT<Id, Expr> u;
-};
-
-} // namespace tomp
-
-#endif // FORTRAN_LOWER_OPENMP_CLAUSET_H
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index 70f232a4858e..f48e84f511a4 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -189,29 +189,96 @@ getBaseObject(const Object &object,
   return std::nullopt;
 }
 
-namespace clause {
-// Helper objects
-#ifdef EMPTY_CLASS
-#undef EMPTY_CLASS
-#endif
-#define EMPTY_CLASS(cls)                                                       \
-  cls make(const parser::OmpClause::cls &, semantics::SemanticsContext &) {    \
+// Helper macros
+#define MAKE_EMPTY_CLASS(cls, from_cls)                                        \
+  cls make(const parser::OmpClause::from_cls &,                                \
+           semantics::SemanticsContext &) {                                    \
+    static_assert(cls::EmptyTrait::value);                                     \
     return cls{};                                                              \
   }                                                                            \
   [[maybe_unused]] extern int xyzzy_semicolon_absorber
 
-#ifdef WRAPPER_CLASS
-#undef WRAPPER_CLASS
-#endif
-#define WRAPPER_CLASS(cls, content)                                            \
+#define MAKE_INCOMPLETE_CLASS(cls, from_cls)                                   \
+  cls make(const parser::OmpClause::from_cls &,                                \
+           semantics::SemanticsContext &) {                                    \
+    static_assert(cls::IncompleteTrait::value);                                \
+    return cls{};                                                              \
+  }                                                                            \
   [[maybe_unused]] extern int xyzzy_semicolon_absorber
-#define GEN_FLANG_CLAUSE_PARSER_CLASSES
-#include "llvm/Frontend/OpenMP/OMP.inc"
-#undef EMPTY_CLASS
-#undef WRAPPER_CLASS
+
+#define MS(x, y) CLAUSET_SCOPED_ENUM_MEMBER_CONVERT(x, y)
+#define MU(x, y) CLAUSET_UNSCOPED_ENUM_MEMBER_CONVERT(x, y)
+
+namespace clause {
+MAKE_EMPTY_CLASS(AcqRel, AcqRel);
+MAKE_EMPTY_CLASS(Acquire, Acquire);
+MAKE_EMPTY_CLASS(Capture, Capture);
+MAKE_EMPTY_CLASS(Compare, Compare);
+MAKE_EMPTY_CLASS(DynamicAllocators, DynamicAllocators);
+MAKE_EMPTY_CLASS(Full, Full);
+MAKE_EMPTY_CLASS(Inbranch, Inbranch);
+MAKE_EMPTY_CLASS(Mergeable, Mergeable);
+MAKE_EMPTY_CLASS(Nogroup, Nogroup);
+// MAKE_EMPTY_CLASS(NoOpenmp, );         // missing-in-parser
+// MAKE_EMPTY_CLASS(NoOpenmpRoutines, ); // missing-in-parser
+// MAKE_EMPTY_CLASS(NoParallelism, );    // missing-in-parser
+MAKE_EMPTY_CLASS(Notinbranch, Notinbranch);
+MAKE_EMPTY_CLASS(Nowait, Nowait);
+MAKE_EMPTY_CLASS(OmpxAttribute, OmpxAttribute);
+MAKE_EMPTY_CLASS(OmpxBare, OmpxBare);
+MAKE_EMPTY_CLASS(Read, Read);
+MAKE_EMPTY_CLASS(Relaxed, Relaxed);
+MAKE_EMPTY_CLASS(Release, Release);
+MAKE_EMPTY_CLASS(ReverseOffload, ReverseOffload);
+MAKE_EMPTY_CLASS(SeqCst, SeqCst);
+MAKE_EMPTY_CLASS(Simd, Simd);
+MAKE_EMPTY_CLASS(Threads, Threads);
+MAKE_EMPTY_CLASS(UnifiedAddress, UnifiedAddress);
+MAKE_EMPTY_CLASS(UnifiedSharedMemory, UnifiedSharedMemory);
+MAKE_EMPTY_CLASS(Unknown, Unknown);
+MAKE_EMPTY_CLASS(Untied, Untied);
+MAKE_EMPTY_CLASS(Weak, Weak);
+MAKE_EMPTY_CLASS(Write, Write);
+
+// Artificial clauses
+MAKE_EMPTY_CLASS(CancellationConstructType, CancellationConstructType);
+MAKE_EMPTY_CLASS(Depobj, Depobj);
+MAKE_EMPTY_CLASS(Flush, Flush);
+MAKE_EMPTY_CLASS(MemoryOrder, MemoryOrder);
+MAKE_EMPTY_CLASS(Threadprivate, Threadprivate);
+
+MAKE_INCOMPLETE_CLASS(AdjustArgs, AdjustArgs);
+MAKE_INCOMPLETE_CLASS(AppendArgs, AppendArgs);
+MAKE_INCOMPLETE_CLASS(Match, Match);
+// MAKE_INCOMPLETE_CLASS(Otherwise, );   // missing-in-parser
+MAKE_INCOMPLETE_CLASS(When, When);
 
 DefinedOperator makeDefinedOperator(const parser::DefinedOperator &inp,
                                     semantics::SemanticsContext &semaCtx) {
+  CLAUSET_ENUM_CONVERT( //
+      convert, parser::DefinedOperator::IntrinsicOperator,
+      DefinedOperator::IntrinsicOperator,
+      // clang-format off
+      MS(Add,      Add)
+      MS(AND,      AND)
+      MS(Concat,   Concat)
+      MS(Divide,   Divide)
+      MS(EQ,       EQ)
+      MS(EQV,      EQV)
+      MS(GE,       GE)
+      MS(GT,       GT)
+      MS(NOT,      NOT)
+      MS(LE,       LE)
+      MS(LT,       LT)
+      MS(Multiply, Multiply)
+      MS(NE,       NE)
+      MS(NEQV,     NEQV)
+      MS(OR,       OR)
+      MS(Power,    Power)
+      MS(Subtract, Subtract)
+      // clang-format on
+  );
+
   return std::visit(
       common::visitors{
           [&](const parser::DefinedOpName &s) {
@@ -219,7 +286,7 @@ DefinedOperator makeDefinedOperator(const parser::DefinedOperator &inp,
                 DefinedOperator::DefinedOpName{makeObject(s.v, semaCtx)}};
           },
           [&](const parser::DefinedOperator::IntrinsicOperator &s) {
-            return DefinedOperator{s};
+            return DefinedOperator{convert(s)};
           },
       },
       inp.u);
@@ -252,7 +319,26 @@ ReductionOperator makeReductionOperator(const parser::OmpReductionOperator &inp,
       inp.u);
 }
 
-// Actual clauses. Each T (where OmpClause::T exists) has its "make".
+// --------------------------------------------------------------------
+// Actual clauses. Each T (where tomp::T exists in ClauseT) has its "make".
+
+// Absent: missing-in-parser
+// AcqRel: empty
+// Acquire: empty
+// AdjustArgs: incomplate
+
+Affinity make(const parser::OmpClause::Affinity &inp,
+              semantics::SemanticsContext &semaCtx) {
+  // inp -> empty
+  llvm_unreachable("Empty: affinity");
+}
+
+Align make(const parser::OmpClause::Align &inp,
+           semantics::SemanticsContext &semaCtx) {
+  // inp -> empty
+  llvm_unreachable("Empty: align");
+}
+
 Aligned make(const parser::OmpClause::Aligned &inp,
              semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpAlignedClause
@@ -260,8 +346,8 @@ Aligned make(const parser::OmpClause::Aligned &inp,
   auto &t1 = std::get<std::optional<parser::ScalarIntConstantExpr>>(inp.v.t);
 
   return Aligned{{
-      makeList(t0, semaCtx),
-      maybeApply(makeExprFn(semaCtx), t1),
+      /*Alignment=*/maybeApply(makeExprFn(semaCtx), t1),
+      /*List=*/makeList(t0, semaCtx),
   }};
 }
 
@@ -272,39 +358,57 @@ Allocate make(const parser::OmpClause::Allocate &inp,
   auto &t0 = std::get<std::optional<wrapped::AllocateModifier>>(inp.v.t);
   auto &t1 = std::get<parser::OmpObjectList>(inp.v.t);
 
-  auto convert = [&](auto &&s) -> Allocate::Modifier {
-    using Modifier = Allocate::Modifier;
-    using Allocator = Modifier::Allocator;
-    using Align = Modifier::Align;
-    using ComplexModifier = Modifier::ComplexModifier;
+  if (!t0) {
+    return Allocate{{/*AllocatorSimpleModifier=*/std::nullopt,
+                     /*AllocatorComplexModifier=*/std::nullopt,
+                     /*AlignModifier=*/std::nullopt,
+                     /*List=*/makeList(t1, semaCtx)}};
+  }
 
-    return std::visit(
-        common::visitors{
-            [&](const wrapped::AllocateModifier::Allocator &v) {
-              return Modifier{Allocator{makeExpr(v.v, semaCtx)}};
-            },
-            [&](const wrapped::AllocateModifier::ComplexModifier &v) {
-              auto &s0 = std::get<wrapped::AllocateModifier::Allocator>(v.t);
-              auto &s1 = std::get<wrapped::AllocateModifier::Align>(v.t);
-              return Modifier{ComplexModifier{{
-                  Allocator{makeExpr(s0.v, semaCtx)},
-                  Align{makeExpr(s1.v, semaCtx)},
-              }}};
-            },
-            [&](const wrapped::AllocateModifier::Align &v) {
-              return Modifier{Align{makeExpr(v.v, semaCtx)}};
-            },
-        },
-        s.u);
-  };
+  using Tuple = decltype(Allocate::t);
 
-  return Allocate{{maybeApply(convert, t0), makeList(t1, semaCtx)}};
+  return Allocate{std::visit(
+      common::visitors{
+          // simple-modifier
+          [&](const wrapped::AllocateModifier::Allocator &v) -> Tuple {
+            return {/*AllocatorSimpleModifier=*/makeExpr(v.v, semaCtx),
+                    /*AllocatorComplexModifier=*/std::nullopt,
+                    /*AlignModifier=*/std::nullopt,
+                    /*List=*/makeList(t1, semaCtx)};
+          },
+          // complex-modifier + align-modifier
+          [&](const wrapped::AllocateModifier::ComplexModifier &v) -> Tuple {
+            auto &s0 = std::get<wrapped::AllocateModifier::Allocator>(v.t);
+            auto &s1 = std::get<wrapped::AllocateModifier::Align>(v.t);
+            return {
+                /*AllocatorSimpleModifier=*/std::nullopt,
+                /*AllocatorComplexModifier=*/Allocator{makeExpr(s0.v, semaCtx)},
+                /*AlignModifier=*/Align{makeExpr(s1.v, semaCtx)},
+                /*List=*/makeList(t1, semaCtx)};
+          },
+          // align-modifier
+          [&](const wrapped::AllocateModifier::Align &v) -> Tuple {
+            return {/*AllocatorSimpleModifier=*/std::nullopt,
+                    /*AllocatorComplexModifier=*/std::nullopt,
+                    /*AlignModifier=*/Align{makeExpr(v.v, semaCtx)},
+                    /*List=*/makeList(t1, semaCtx)};
+          },
+      },
+      t0->u)};
 }
 
 Allocator make(const parser::OmpClause::Allocator &inp,
                semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::ScalarIntExpr
-  return Allocator{makeExpr(inp.v, semaCtx)};
+  return Allocator{/*Allocator=*/makeExpr(inp.v, semaCtx)};
+}
+
+// AppendArgs: incomplete
+
+At make(const parser::OmpClause::At &inp,
+        semantics::SemanticsContext &semaCtx) {
+  // inp -> empty
+  llvm_unreachable("Empty: at");
 }
 
 // Never called, but needed for using "make" as a Clause visitor.
@@ -312,25 +416,65 @@ Allocator make(const parser::OmpClause::Allocator &inp,
 AtomicDefaultMemOrder make(const parser::OmpClause::AtomicDefaultMemOrder &inp,
                            semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpAtomicDefaultMemOrderClause
-  return AtomicDefaultMemOrder{inp.v.v};
+  CLAUSET_ENUM_CONVERT( //
+      convert, common::OmpAtomicDefaultMemOrderType,
+      AtomicDefaultMemOrder::MemoryOrder,
+      // clang-format off
+      MS(AcqRel,   AcqRel)
+      MS(Relaxed,  Relaxed)
+      MS(SeqCst,   SeqCst)
+      // clang-format on
+  );
+
+  return AtomicDefaultMemOrder{/*MemoryOrder=*/convert(inp.v.v)};
 }
 
+Bind make(const parser::OmpClause::Bind &inp,
+          semantics::SemanticsContext &semaCtx) {
+  // inp -> empty
+  llvm_unreachable("Empty: bind");
+}
+
+// CancellationConstructType: empty
+// Capture: empty
+
 Collapse make(const parser::OmpClause::Collapse &inp,
               semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::ScalarIntConstantExpr
-  return Collapse{makeExpr(inp.v, semaCtx)};
+  return Collapse{/*N=*/makeExpr(inp.v, semaCtx)};
 }
 
+// Compare: empty
+// Contains: missing-in-parser
+
 Copyin make(const parser::OmpClause::Copyin &inp,
             semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpObjectList
-  return Copyin{makeList(inp.v, semaCtx)};
+  return Copyin{/*List=*/makeList(inp.v, semaCtx)};
 }
 
 Copyprivate make(const parser::OmpClause::Copyprivate &inp,
                  semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpObjectList
-  return Copyprivate{makeList(inp.v, semaCtx)};
+  return Copyprivate{/*List=*/makeList(inp.v, semaCtx)};
+}
+
+Default make(const parser::OmpClause::Default &inp,
+             semantics::SemanticsContext &semaCtx) {
+  // inp.v -> parser::OmpDefaultClause
+  using wrapped = parser::OmpDefaultClause;
+
+  CLAUSET_ENUM_CONVERT( //
+      convert, wrapped::Type, Default::DataSharingAttribute,
+      // clang-format off
+      MS(Firstprivate, Firstprivate)
+      MS(None,         None)
+      MS(Private,      Private)
+      MS(Shared,       Shared)
+      // clang-format on
+  );
+
+  return Default{/*DataSharingAttribute=*/convert(inp.v.v)};
 }
 
 Defaultmap make(const parser::OmpClause::Defaultmap &inp,
@@ -338,51 +482,111 @@ Defaultmap make(const parser::OmpClause::Defaultmap &inp,
   // inp.v -> parser::OmpDefaultmapClause
   using wrapped = parser::OmpDefaultmapClause;
 
+  CLAUSET_ENUM_CONVERT( //
+      convert1, wrapped::ImplicitBehavior, Defaultmap::ImplicitBehavior,
+      // clang-format off
+      MS(Alloc,        Alloc)
+      MS(To,           To)
+      MS(From,         From)
+      MS(Tofrom,       Tofrom)
+      MS(Firstprivate, Firstprivate)
+      MS(None,         None)
+      MS(Default,      Default)
+      // MS(, Present)  missing-in-parser
+      // clang-format on
+  );
+
+  CLAUSET_ENUM_CONVERT( //
+      convert2, wrapped::VariableCategory, Defaultmap::VariableCategory,
+      // clang-format off
+      MS(Scalar,       Scalar)
+      MS(Aggregate,    Aggregate)
+      MS(Pointer,      Pointer)
+      MS(Allocatable,  Allocatable)
+      // clang-format on
+  );
+
   auto &t0 = std::get<wrapped::ImplicitBehavior>(inp.v.t);
   auto &t1 = std::get<std::optional<wrapped::VariableCategory>>(inp.v.t);
-  return Defaultmap{{t0, t1}};
-}
-
-Default make(const parser::OmpClause::Default &inp,
-             semantics::SemanticsContext &semaCtx) {
-  // inp.v -> parser::OmpDefaultClause
-  return Default{inp.v.v};
+  return Defaultmap{{/*ImplicitBehavior=*/convert1(t0),
+                     /*VariableCategory=*/maybeApply(convert2, t1)}};
 }
 
 Depend make(const parser::OmpClause::Depend &inp,
             semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpDependClause
   using wrapped = parser::OmpDependClause;
-
-  return std::visit(
+  using Variant = decltype(Depend::u);
+  // Iteration is the equivalent of parser::OmpDependSinkVec
+  using Iteration = Doacross::Vector::value_type; // LoopIterationT
+
+  CLAUSET_ENUM_CONVERT( //
+      convert1, parser::OmpDependenceType::Type, Depend::TaskDependenceType,
+      // clang-format off
+      MS(In,     In)
+      MS(Out,    Out)
+      MS(Inout,  Inout)
+      // MS(, Mutexinoutset)   // missing-in-parser
+      // MS(, Inputset)        // missing-in-parser
+      // MS(, Depobj)          // missing-in-parser
+      // clang-format on
+  );
+
+  return Depend{std::visit( //
       common::visitors{
-          [&](const wrapped::Source &s) { return Depend{Depend::Source{}}; },
-          [&](const wrapped::Sink &s) {
-            auto convert = [&](const parser::OmpDependSinkVec &v) {
+          // Doacross
+          [&](const wrapped::Source &s) -> Variant {
+            return Doacross{
+                {/*DependenceType=*/Doacross::DependenceType::Source,
+                 /*Vector=*/{}}};
+          },
+          // Doacross
+          [&](const wrapped::Sink &s) -> Variant {
+            using DependLength = parser::OmpDependSinkVecLength;
+            auto convert2 = [&](const parser::OmpDependSinkVec &v) {
               auto &t0 = std::get<parser::Name>(v.t);
-              auto &t1 =
-                  std::get<std::optional<parser::OmpDependSinkVecLength>>(v.t);
-              auto convert1 = [&](const parser::OmpDependSinkVecLength &u) {
+              auto &t1 = std::get<std::optional<DependLength>>(v.t);
+
+              auto convert3 = [&](const DependLength &u) {
                 auto &s0 = std::get<parser::DefinedOperator>(u.t);
                 auto &s1 = std::get<parser::ScalarIntConstantExpr>(u.t);
-                return Depend::Sink::Length{makeDefinedOperator(s0, semaCtx),
-                                            makeExpr(s1, semaCtx)};
+                return Iteration::Distance{
+                    {makeDefinedOperator(s0, semaCtx), makeExpr(s1, semaCtx)}};
               };
-              return Depend::Sink::Vec{makeObject(t0, semaCtx),
-                                       maybeApply(convert1, t1)};
+              return Iteration{
+                  {makeObject(t0, semaCtx), maybeApply(convert3, t1)}};
             };
-            return Depend{Depend::Sink{makeList(s.v, convert)}};
+            return Doacross{{/*DependenceType=*/Doacross::DependenceType::Sink,
+                             /*Vector=*/makeList(s.v, convert2)}};
           },
-          [&](const wrapped::InOut &s) {
+          // Depend::WithLocators
+          [&](const wrapped::InOut &s) -> Variant {
             auto &t0 = std::get<parser::OmpDependenceType>(s.t);
             auto &t1 = std::get<std::list<parser::Designator>>(s.t);
-            auto convert = [&](const parser::Designator &t) {
+            auto convert4 = [&](const parser::Designator &t) {
               return makeObject(t, semaCtx);
             };
-            return Depend{Depend::InOut{{t0.v, makeList(t1, convert)}}};
+            return Depend::WithLocators{
+                {/*TaskDependenceType=*/convert1(t0.v),
+                 /*Iterator=*/std::nullopt,
+                 /*LocatorList=*/makeList(t1, convert4)}};
           },
       },
-      inp.v.u);
+      inp.v.u)};
+}
+
+// Depobj: empty
+
+Destroy make(const parser::OmpClause::Destroy &inp,
+             semantics::SemanticsContext &semaCtx) {
+  // inp -> empty
+  llvm_unreachable("Empty: destroy");
+}
+
+Detach make(const parser::OmpClause::Detach &inp,
+            semantics::SemanticsContext &semaCtx) {
+  // inp -> empty
+  llvm_unreachable("Empty: detach");
 }
 
 Device make(const parser::OmpClause::Device &inp,
@@ -390,100 +594,188 @@ Device make(const parser::OmpClause::Device &inp,
   // inp.v -> parser::OmpDeviceClause
   using wrapped = parser::OmpDeviceClause;
 
+  CLAUSET_ENUM_CONVERT( //
+      convert, parser::OmpDeviceClause::DeviceModifier, Device::DeviceModifier,
+      // clang-format off
+      MS(Ancestor,   Ancestor)
+      MS(Device_Num, DeviceNum)
+      // clang-format on
+  );
   auto &t0 = std::get<std::optional<wrapped::DeviceModifier>>(inp.v.t);
   auto &t1 = std::get<parser::ScalarIntExpr>(inp.v.t);
-  return Device{{t0, makeExpr(t1, semaCtx)}};
+  return Device{{/*DeviceModifier=*/maybeApply(convert, t0),
+                 /*DeviceDescription=*/makeExpr(t1, semaCtx)}};
 }
 
 DeviceType make(const parser::OmpClause::DeviceType &inp,
                 semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpDeviceTypeClause
-  return DeviceType{inp.v.v};
+  using wrapped = parser::OmpDeviceTypeClause;
+
+  CLAUSET_ENUM_CONVERT( //
+      convert, wrapped::Type, DeviceType::DeviceTypeDescription,
+      // clang-format off
+      MS(Any,    Any)
+      MS(Host,   Host)
+      MS(Nohost, Nohost)
+      // clang-format om
+  );
+  return DeviceType{/*DeviceTypeDescription=*/convert(inp.v.v)};
 }
 
 DistSchedule make(const parser::OmpClause::DistSchedule &inp,
                   semantics::SemanticsContext &semaCtx) {
   // inp.v -> std::optional<parser::ScalarIntExpr>
-  return DistSchedule{maybeApply(makeExprFn(semaCtx), inp.v)};
+  return DistSchedule{{/*Kind=*/DistSchedule::Kind::Static,
+                       /*ChunkSize=*/maybeApply(makeExprFn(semaCtx), inp.v)}};
 }
 
+Doacross make(const parser::OmpClause::Doacross &inp,
+              semantics::SemanticsContext &semaCtx) {
+  // inp -> empty
+  llvm_unreachable("Empty: doacross");
+}
+
+// DynamicAllocators: empty
+
 Enter make(const parser::OmpClause::Enter &inp,
            semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpObjectList
-  return Enter{makeList(inp.v, semaCtx)};
+  return Enter{makeList(/*List=*/inp.v, semaCtx)};
+}
+
+Exclusive make(const parser::OmpClause::Exclusive &inp,
+               semantics::SemanticsContext &semaCtx) {
+  // inp -> empty
+  llvm_unreachable("Empty: exclusive");
+}
+
+Fail make(const parser::OmpClause::Fail &inp,
+          semantics::SemanticsContext &semaCtx) {
+  // inp -> empty
+  llvm_unreachable("Empty: fail");
 }
 
 Filter make(const parser::OmpClause::Filter &inp,
             semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::ScalarIntExpr
-  return Filter{makeExpr(inp.v, semaCtx)};
+  return Filter{/*ThreadNum=*/makeExpr(inp.v, semaCtx)};
 }
 
 Final make(const parser::OmpClause::Final &inp,
            semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::ScalarLogicalExpr
-  return Final{makeExpr(inp.v, semaCtx)};
+  return Final{/*Finalize=*/makeExpr(inp.v, semaCtx)};
 }
 
 Firstprivate make(const parser::OmpClause::Firstprivate &inp,
                   semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpObjectList
-  return Firstprivate{makeList(inp.v, semaCtx)};
+  return Firstprivate{/*List=*/makeList(inp.v, semaCtx)};
 }
 
+// Flush: empty
+
 From make(const parser::OmpClause::From &inp,
           semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpObjectList
-  return From{makeList(inp.v, semaCtx)};
+  return From{{/*Expectation=*/std::nullopt, /*Mapper=*/std::nullopt,
+               /*Iterator=*/std::nullopt,
+               /*LocatorList=*/makeList(inp.v, semaCtx)}};
 }
 
+// Full: empty
+
 Grainsize make(const parser::OmpClause::Grainsize &inp,
                semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::ScalarIntExpr
-  return Grainsize{makeExpr(inp.v, semaCtx)};
+  return Grainsize{{/*Prescriptiveness=*/std::nullopt,
+                    /*GrainSize=*/makeExpr(inp.v, semaCtx)}};
 }
 
 HasDeviceAddr make(const parser::OmpClause::HasDeviceAddr &inp,
                    semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpObjectList
-  return HasDeviceAddr{makeList(inp.v, semaCtx)};
+  return HasDeviceAddr{/*List=*/makeList(inp.v, semaCtx)};
 }
 
 Hint make(const parser::OmpClause::Hint &inp,
           semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::ConstantExpr
-  return Hint{makeExpr(inp.v, semaCtx)};
+  return Hint{/*HintExpr=*/makeExpr(inp.v, semaCtx)};
 }
 
+// Holds: missing-in-parser
+
 If make(const parser::OmpClause::If &inp,
         semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpIfClause
   using wrapped = parser::OmpIfClause;
 
+  CLAUSET_ENUM_CONVERT( //
+      convert, wrapped::DirectiveNameModifier, llvm::omp::Directive,
+      // clang-format off
+      MS(Parallel,         OMPD_parallel)
+      MS(Simd,             OMPD_simd)
+      MS(Target,           OMPD_target)
+      MS(TargetData,       OMPD_target_data)
+      MS(TargetEnterData,  OMPD_target_enter_data)
+      MS(TargetExitData,   OMPD_target_exit_data)
+      MS(TargetUpdate,     OMPD_target_update)
+      MS(Task,             OMPD_task)
+      MS(Taskloop,         OMPD_taskloop)
+      MS(Teams,            OMPD_teams)
+      // clang-format on
+  );
   auto &t0 = std::get<std::optional<wrapped::DirectiveNameModifier>>(inp.v.t);
   auto &t1 = std::get<parser::ScalarLogicalExpr>(inp.v.t);
-  return If{{t0, makeExpr(t1, semaCtx)}};
+  return If{{/*DirectiveNameModifier=*/maybeApply(convert, t0),
+             /*IfExpression=*/makeExpr(t1, semaCtx)}};
 }
 
+// Inbranch: empty
+
+Inclusive make(const parser::OmpClause::Inclusive &inp,
+               semantics::SemanticsContext &semaCtx) {
+  // inp -> empty
+  llvm_unreachable("Empty: inclusive");
+}
+
+Indirect make(const parser::OmpClause::Indirect &inp,
+              semantics::SemanticsContext &semaCtx) {
+  // inp -> empty
+  llvm_unreachable("Empty: indirect");
+}
+
+Init make(const parser::OmpClause::Init &inp,
+          semantics::SemanticsContext &semaCtx) {
+  // inp -> empty
+  llvm_unreachable("Empty: init");
+}
+
+// Initializer: missing-in-parser
+
 InReduction make(const parser::OmpClause::InReduction &inp,
                  semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpInReductionClause
   auto &t0 = std::get<parser::OmpReductionOperator>(inp.v.t);
   auto &t1 = std::get<parser::OmpObjectList>(inp.v.t);
   return InReduction{
-      {makeReductionOperator(t0, semaCtx), makeList(t1, semaCtx)}};
+      {/*ReductionIdentifiers=*/{makeReductionOperator(t0, semaCtx)},
+       /*List=*/makeList(t1, semaCtx)}};
 }
 
 IsDevicePtr make(const parser::OmpClause::IsDevicePtr &inp,
                  semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpObjectList
-  return IsDevicePtr{makeList(inp.v, semaCtx)};
+  return IsDevicePtr{/*List=*/makeList(inp.v, semaCtx)};
 }
 
 Lastprivate make(const parser::OmpClause::Lastprivate &inp,
                  semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpObjectList
-  return Lastprivate{makeList(inp.v, semaCtx)};
+  return Lastprivate{{/*LastprivateModifier=*/std::nullopt,
+                      /*List=*/makeList(inp.v, semaCtx)}};
 }
 
 Linear make(const parser::OmpClause::Linear &inp,
@@ -491,140 +783,244 @@ Linear make(const parser::OmpClause::Linear &inp,
   // inp.v -> parser::OmpLinearClause
   using wrapped = parser::OmpLinearClause;
 
-  return std::visit(
+  CLAUSET_ENUM_CONVERT( //
+      convert, parser::OmpLinearModifier::Type, Linear::LinearModifier,
+      // clang-format off
+      MS(Ref,  Ref)
+      MS(Val,  Val)
+      MS(Uval, Uval)
+      // clang-format on
+  );
+
+  using Tuple = decltype(Linear::t);
+
+  return Linear{std::visit(
       common::visitors{
-          [&](const wrapped::WithModifier &s) {
-            return Linear{{Linear::Modifier{s.modifier.v},
-                           makeList(s.names, makeObjectFn(semaCtx)),
-                           maybeApply(makeExprFn(semaCtx), s.step)}};
+          [&](const wrapped::WithModifier &s) -> Tuple {
+            return {
+                /*StepSimpleModifier=*/std::nullopt,
+                /*StepComplexModifier=*/maybeApply(makeExprFn(semaCtx), s.step),
+                /*LinearModifier=*/convert(s.modifier.v),
+                /*List=*/makeList(s.names, makeObjectFn(semaCtx))};
           },
-          [&](const wrapped::WithoutModifier &s) {
-            return Linear{{std::nullopt,
-                           makeList(s.names, makeObjectFn(semaCtx)),
-                           maybeApply(makeExprFn(semaCtx), s.step)}};
+          [&](const wrapped::WithoutModifier &s) -> Tuple {
+            return {
+                /*StepSimpleModifier=*/maybeApply(makeExprFn(semaCtx), s.step),
+                /*StepComplexModifier=*/std::nullopt,
+                /*LinearModifier=*/std::nullopt,
+                /*List=*/makeList(s.names, makeObjectFn(semaCtx))};
           },
       },
-      inp.v.u);
+      inp.v.u)};
 }
 
 Link make(const parser::OmpClause::Link &inp,
           semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpObjectList
-  return Link{makeList(inp.v, semaCtx)};
+  return Link{/*List=*/makeList(inp.v, semaCtx)};
 }
 
 Map make(const parser::OmpClause::Map &inp,
          semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpMapClause
+
+  CLAUSET_ENUM_CONVERT( //
+      convert1, parser::OmpMapType::Type, Map::MapType,
+      // clang-format off
+      MS(To,       To)
+      MS(From,     From)
+      MS(Tofrom,   Tofrom)
+      MS(Alloc,    Alloc)
+      MS(Release,  Release)
+      MS(Delete,   Delete)
+      // clang-format on
+  );
+
+  // No convert2: MapTypeModifier is not an enum in parser.
+
   auto &t0 = std::get<std::optional<parser::OmpMapType>>(inp.v.t);
   auto &t1 = std::get<parser::OmpObjectList>(inp.v.t);
-  auto convert = [](const parser::OmpMapType &s) {
-    auto &s0 = std::get<std::optional<parser::OmpMapType::Always>>(s.t);
-    auto &s1 = std::get<parser::OmpMapType::Type>(s.t);
-    auto convertT = [](parser::OmpMapType::Always) {
-      return Map::MapType::Always{};
-    };
-    return Map::MapType{{maybeApply(convertT, s0), s1}};
-  };
-  return Map{{maybeApply(convert, t0), makeList(t1, semaCtx)}};
+
+  if (!t0) {
+    return Map{{/*MapType=*/std::nullopt, /*MapTypeModifiers=*/std::nullopt,
+                /*Mapper=*/std::nullopt, /*Iterator=*/std::nullopt,
+                /*LocatorList=*/makeList(t1, semaCtx)}};
+  }
+
+  auto &s0 = std::get<std::optional<parser::OmpMapType::Always>>(t0->t);
+  auto &s1 = std::get<parser::OmpMapType::Type>(t0->t);
+
+  std::optional<Map::MapTypeModifiers> maybeList;
+  if (s0)
+    maybeList = Map::MapTypeModifiers{Map::MapTypeModifier::Always};
+
+  return Map{{/*MapType=*/convert1(s1),
+              /*MapTypeModifiers=*/maybeList,
+              /*Mapper=*/std::nullopt, /*Iterator=*/std::nullopt,
+              /*LocatorList=*/makeList(t1, semaCtx)}};
+}
+
+// Match: incomplete
+// MemoryOrder: empty
+// Mergeable: empty
+
+Message make(const parser::OmpClause::Message &inp,
+             semantics::SemanticsContext &semaCtx) {
+  // inp -> empty
+  llvm_unreachable("Empty: message");
 }
 
 Nocontext make(const parser::OmpClause::Nocontext &inp,
                semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::ScalarLogicalExpr
-  return Nocontext{makeExpr(inp.v, semaCtx)};
+  return Nocontext{/*DoNotUpdateContext=*/makeExpr(inp.v, semaCtx)};
 }
 
+// Nogroup: empty
+
 Nontemporal make(const parser::OmpClause::Nontemporal &inp,
                  semantics::SemanticsContext &semaCtx) {
   // inp.v -> std::list<parser::Name>
-  return Nontemporal{makeList(inp.v, makeObjectFn(semaCtx))};
+  return Nontemporal{/*List=*/makeList(inp.v, makeObjectFn(semaCtx))};
 }
 
+// NoOpenmp: missing-in-parser
+// NoOpenmpRoutines: missing-in-parser
+// NoParallelism: missing-in-parser
+// Notinbranch: empty
+
 Novariants make(const parser::OmpClause::Novariants &inp,
                 semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::ScalarLogicalExpr
-  return Novariants{makeExpr(inp.v, semaCtx)};
+  return Novariants{/*DoNotUseVariant=*/makeExpr(inp.v, semaCtx)};
 }
 
+// Nowait: empty
+
 NumTasks make(const parser::OmpClause::NumTasks &inp,
               semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::ScalarIntExpr
-  return NumTasks{makeExpr(inp.v, semaCtx)};
+  return NumTasks{{/*Prescriptiveness=*/std::nullopt,
+                   /*NumTasks=*/makeExpr(inp.v, semaCtx)}};
 }
 
 NumTeams make(const parser::OmpClause::NumTeams &inp,
               semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::ScalarIntExpr
-  return NumTeams{makeExpr(inp.v, semaCtx)};
+  return NumTeams{{/*LowerBound=*/std::nullopt,
+                   /*UpperBound=*/makeExpr(inp.v, semaCtx)}};
 }
 
 NumThreads make(const parser::OmpClause::NumThreads &inp,
                 semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::ScalarIntExpr
-  return NumThreads{makeExpr(inp.v, semaCtx)};
+  return NumThreads{/*Nthreads=*/makeExpr(inp.v, semaCtx)};
 }
 
+// OmpxAttribute: empty
+// OmpxBare: empty
+
 OmpxDynCgroupMem make(const parser::OmpClause::OmpxDynCgroupMem &inp,
                       semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::ScalarIntExpr
   return OmpxDynCgroupMem{makeExpr(inp.v, semaCtx)};
 }
 
-Ordered make(const parser::OmpClause::Ordered &inp,
-             semantics::SemanticsContext &semaCtx) {
-  // inp.v -> std::optional<parser::ScalarIntConstantExpr>
-  return Ordered{maybeApply(makeExprFn(semaCtx), inp.v)};
-}
-
 Order make(const parser::OmpClause::Order &inp,
            semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpOrderClause
   using wrapped = parser::OmpOrderClause;
+
+  CLAUSET_ENUM_CONVERT( //
+      convert1, parser::OmpOrderModifier::Kind, Order::OrderModifier,
+      // clang-format off
+      MS(Reproducible,   Reproducible)
+      MS(Unconstrained,  Unconstrained)
+      // clang-format on
+  );
+
+  CLAUSET_ENUM_CONVERT( //
+      convert2, wrapped::Type, Order::Ordering,
+      // clang-format off
+      MS(Concurrent, Concurrent)
+      // clang-format on
+  );
+
   auto &t0 = std::get<std::optional<parser::OmpOrderModifier>>(inp.v.t);
   auto &t1 = std::get<wrapped::Type>(inp.v.t);
-  auto convert = [](const parser::OmpOrderModifier &s) -> Order::Kind {
-    return std::get<parser::OmpOrderModifier::Kind>(s.u);
+
+  auto convert3 = [&](const parser::OmpOrderModifier &s) {
+    return std::visit(
+        [&](parser::OmpOrderModifier::Kind k) { return convert1(k); }, s.u);
   };
-  return Order{{maybeApply(convert, t0), t1}};
+  return Order{
+      {/*OrderModifier=*/maybeApply(convert3, t0), /*Ordering=*/convert2(t1)}};
 }
 
+Ordered make(const parser::OmpClause::Ordered &inp,
+             semantics::SemanticsContext &semaCtx) {
+  // inp.v -> std::optional<parser::ScalarIntConstantExpr>
+  return Ordered{/*N=*/maybeApply(makeExprFn(semaCtx), inp.v)};
+}
+
+// Otherwise: incomplete, missing-in-parser
+
 Partial make(const parser::OmpClause::Partial &inp,
              semantics::SemanticsContext &semaCtx) {
   // inp.v -> std::optional<parser::ScalarIntConstantExpr>
-  return Partial{maybeApply(makeExprFn(semaCtx), inp.v)};
+  return Partial{/*UnrollFactor=*/maybeApply(makeExprFn(semaCtx), inp.v)};
 }
 
 Priority make(const parser::OmpClause::Priority &inp,
               semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::ScalarIntExpr
-  return Priority{makeExpr(inp.v, semaCtx)};
+  return Priority{/*PriorityValue=*/makeExpr(inp.v, semaCtx)};
 }
 
 Private make(const parser::OmpClause::Private &inp,
              semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpObjectList
-  return Private{makeList(inp.v, semaCtx)};
+  return Private{/*List=*/makeList(inp.v, semaCtx)};
 }
 
 ProcBind make(const parser::OmpClause::ProcBind &inp,
               semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpProcBindClause
-  return ProcBind{inp.v.v};
+  using wrapped = parser::OmpProcBindClause;
+
+  CLAUSET_ENUM_CONVERT( //
+      convert, wrapped::Type, ProcBind::AffinityPolicy,
+      // clang-format off
+      MS(Close,    Close)
+      MS(Master,   Master)
+      MS(Spread,   Spread)
+      MS(Primary,  Primary)
+      // clang-format on
+  );
+  return ProcBind{/*AffinityPolicy=*/convert(inp.v.v)};
 }
 
+// Read: empty
+
 Reduction make(const parser::OmpClause::Reduction &inp,
                semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpReductionClause
   auto &t0 = std::get<parser::OmpReductionOperator>(inp.v.t);
   auto &t1 = std::get<parser::OmpObjectList>(inp.v.t);
-  return Reduction{{makeReductionOperator(t0, semaCtx), makeList(t1, semaCtx)}};
+  return Reduction{
+      {/*ReductionIdentifiers=*/{makeReductionOperator(t0, semaCtx)},
+       /*ReductionModifier=*/std::nullopt,
+       /*List=*/makeList(t1, semaCtx)}};
 }
 
+// Relaxed: empty
+// Release: empty
+// ReverseOffload: empty
+
 Safelen make(const parser::OmpClause::Safelen &inp,
              semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::ScalarIntConstantExpr
-  return Safelen{makeExpr(inp.v, semaCtx)};
+  return Safelen{/*Length=*/makeExpr(inp.v, semaCtx)};
 }
 
 Schedule make(const parser::OmpClause::Schedule &inp,
@@ -632,41 +1028,97 @@ Schedule make(const parser::OmpClause::Schedule &inp,
   // inp.v -> parser::OmpScheduleClause
   using wrapped = parser::OmpScheduleClause;
 
+  CLAUSET_ENUM_CONVERT( //
+      convert1, wrapped::ScheduleType, Schedule::Kind,
+      // clang-format off
+      MS(Static,   Static)
+      MS(Dynamic,  Dynamic)
+      MS(Guided,   Guided)
+      MS(Auto,     Auto)
+      MS(Runtime,  Runtime)
+      // clang-format on
+  );
+
+  CLAUSET_ENUM_CONVERT( //
+      convert2, parser::OmpScheduleModifierType::ModType,
+      Schedule::OrderingModifier,
+      // clang-format off
+      MS(Monotonic,    Monotonic)
+      MS(Nonmonotonic, Nonmonotonic)
+      // clang-format on
+  );
+
+  CLAUSET_ENUM_CONVERT( //
+      convert3, parser::OmpScheduleModifierType::ModType,
+      Schedule::ChunkModifier,
+      // clang-format off
+      MS(Simd, Simd)
+      // clang-format on
+  );
+
   auto &t0 = std::get<std::optional<parser::OmpScheduleModifier>>(inp.v.t);
   auto &t1 = std::get<wrapped::ScheduleType>(inp.v.t);
   auto &t2 = std::get<std::optional<parser::ScalarIntExpr>>(inp.v.t);
 
-  auto convert = [](auto &&s) -> Schedule::ScheduleModifier {
-    auto &s0 = std::get<parser::OmpScheduleModifier::Modifier1>(s.t);
-    auto &s1 =
-        std::get<std::optional<parser::OmpScheduleModifier::Modifier2>>(s.t);
+  if (!t0) {
+    return Schedule{{/*Kind=*/convert1(t1), /*OrderingModifier=*/std::nullopt,
+                     /*ChunkModifier=*/std::nullopt,
+                     /*ChunkSize=*/maybeApply(makeExprFn(semaCtx), t2)}};
+  }
 
-    auto convert1 = [](auto &&v) { // Modifier1 or Modifier2
-      return v.v.v;
-    };
-    return Schedule::ScheduleModifier{{s0.v.v, maybeApply(convert1, s1)}};
-  };
+  // The members of parser::OmpScheduleModifier correspond to OrderingModifier,
+  // and ChunkModifier, but they can appear in any order.
+  auto &m1 = std::get<parser::OmpScheduleModifier::Modifier1>(t0->t);
+  auto &m2 =
+      std::get<std::optional<parser::OmpScheduleModifier::Modifier2>>(t0->t);
+
+  std::optional<Schedule::OrderingModifier> omod;
+  std::optional<Schedule::ChunkModifier> cmod;
+
+  if (m1.v.v == parser::OmpScheduleModifierType::ModType::Simd) {
+    // m1 is chunk-modifier
+    cmod = convert3(m1.v.v);
+    if (m2)
+      omod = convert2(m2->v.v);
+  } else {
+    // m1 is ordering-modifier
+    omod = convert2(m1.v.v);
+    if (m2)
+      cmod = convert3(m2->v.v);
+  }
+
+  return Schedule{{/*Kind=*/convert1(t1),
+                   /*OrderingModifier=*/omod,
+                   /*ChunkModifier=*/cmod,
+                   /*ChunkSize=*/maybeApply(makeExprFn(semaCtx), t2)}};
+}
 
-  return Schedule{
-      {maybeApply(convert, t0), t1, maybeApply(makeExprFn(semaCtx), t2)}};
+// SeqCst: empty
+
+Severity make(const parser::OmpClause::Severity &inp,
+              semantics::SemanticsContext &semaCtx) {
+  // inp -> empty
+  llvm_unreachable("Empty: severity");
 }
 
 Shared make(const parser::OmpClause::Shared &inp,
             semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpObjectList
-  return Shared{makeList(inp.v, semaCtx)};
+  return Shared{/*List=*/makeList(inp.v, semaCtx)};
 }
 
+// Simd: empty
+
 Simdlen make(const parser::OmpClause::Simdlen &inp,
              semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::ScalarIntConstantExpr
-  return Simdlen{makeExpr(inp.v, semaCtx)};
+  return Simdlen{/*Length=*/makeExpr(inp.v, semaCtx)};
 }
 
 Sizes make(const parser::OmpClause::Sizes &inp,
            semantics::SemanticsContext &semaCtx) {
   // inp.v -> std::list<parser::ScalarIntExpr>
-  return Sizes{makeList(inp.v, makeExprFn(semaCtx))};
+  return Sizes{/*SizeList=*/makeList(inp.v, makeExprFn(semaCtx))};
 }
 
 TaskReduction make(const parser::OmpClause::TaskReduction &inp,
@@ -675,38 +1127,72 @@ TaskReduction make(const parser::OmpClause::TaskReduction &inp,
   auto &t0 = std::get<parser::OmpReductionOperator>(inp.v.t);
   auto &t1 = std::get<parser::OmpObjectList>(inp.v.t);
   return TaskReduction{
-      {makeReductionOperator(t0, semaCtx), makeList(t1, semaCtx)}};
+      {/*ReductionIdentifiers=*/{makeReductionOperator(t0, semaCtx)},
+       /*List=*/makeList(t1, semaCtx)}};
 }
 
 ThreadLimit make(const parser::OmpClause::ThreadLimit &inp,
                  semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::ScalarIntExpr
-  return ThreadLimit{makeExpr(inp.v, semaCtx)};
+  return ThreadLimit{/*Threadlim=*/makeExpr(inp.v, semaCtx)};
 }
 
+// Threadprivate: empty
+// Threads: empty
+
 To make(const parser::OmpClause::To &inp,
         semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpObjectList
-  return To{makeList(inp.v, semaCtx)};
+  return To{{/*Expectation=*/std::nullopt, /*Mapper=*/std::nullopt,
+             /*Iterator=*/std::nullopt,
+             /*LocatorList=*/makeList(inp.v, semaCtx)}};
 }
 
+// UnifiedAddress: empty
+// UnifiedSharedMemory: empty
+
 Uniform make(const parser::OmpClause::Uniform &inp,
              semantics::SemanticsContext &semaCtx) {
   // inp.v -> std::list<parser::Name>
-  return Uniform{makeList(inp.v, makeObjectFn(semaCtx))};
+  return Uniform{/*ParameterList=*/makeList(inp.v, makeObjectFn(semaCtx))};
+}
+
+// Unknown: empty
+// Untied: empty
+
+Update make(const parser::OmpClause::Update &inp,
+            semantics::SemanticsContext &semaCtx) {
+  // inp -> empty
+  return Update{/*TaskDependenceType=*/std::nullopt};
+}
+
+Use make(const parser::OmpClause::Use &inp,
+         semantics::SemanticsContext &semaCtx) {
+  // inp -> empty
+  llvm_unreachable("Empty: use");
 }
 
 UseDeviceAddr make(const parser::OmpClause::UseDeviceAddr &inp,
                    semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpObjectList
-  return UseDeviceAddr{makeList(inp.v, semaCtx)};
+  return UseDeviceAddr{/*List=*/makeList(inp.v, semaCtx)};
 }
 
 UseDevicePtr make(const parser::OmpClause::UseDevicePtr &inp,
                   semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpObjectList
-  return UseDevicePtr{makeList(inp.v, semaCtx)};
+  return UseDevicePtr{/*List=*/makeList(inp.v, semaCtx)};
+}
+
+UsesAllocators make(const parser::OmpClause::UsesAllocators &inp,
+                    semantics::SemanticsContext &semaCtx) {
+  // inp -> empty
+  llvm_unreachable("Empty: uses_allocators");
 }
+
+// Weak: empty
+// When: incomplete
+// Write: empty
 } // namespace clause
 
 Clause makeClause(const Fortran::parser::OmpClause &cls,
diff --git a/flang/lib/Lower/OpenMP/Clauses.h b/flang/lib/Lower/OpenMP/Clauses.h
index 3fba593b5349..af1318226e8c 100644
--- a/flang/lib/Lower/OpenMP/Clauses.h
+++ b/flang/lib/Lower/OpenMP/Clauses.h
@@ -8,8 +8,6 @@
 #ifndef FORTRAN_LOWER_OPENMP_CLAUSES_H
 #define FORTRAN_LOWER_OPENMP_CLAUSES_H
 
-#include "ClauseT.h"
-
 #include "flang/Evaluate/expression.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/expression.h"
@@ -17,6 +15,7 @@
 #include "flang/Semantics/symbol.h"
 
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/Frontend/OpenMP/ClauseT.h"
 
 #include <optional>
 #include <type_traits>
@@ -28,32 +27,32 @@ using SomeType = evaluate::SomeType;
 using SomeExpr = semantics::SomeExpr;
 using MaybeExpr = semantics::MaybeExpr;
 
-using SymIdent = semantics::Symbol *;
-using SymReference = SomeExpr;
+using TypeTy = SomeType;
+using IdTy = semantics::Symbol *;
+using ExprTy = SomeExpr;
 
 template <typename T>
 using List = tomp::ListT<T>;
 } // namespace Fortran::lower::omp
 
-namespace tomp {
+namespace tomp::type {
 template <>
-struct ObjectT<Fortran::lower::omp::SymIdent,
-               Fortran::lower::omp::SymReference> {
-  using IdType = Fortran::lower::omp::SymIdent;
-  using ExprType = Fortran::lower::omp::SymReference;
+struct ObjectT<Fortran::lower::omp::IdTy, Fortran::lower::omp::ExprTy> {
+  using IdTy = Fortran::lower::omp::IdTy;
+  using ExprTy = Fortran::lower::omp::ExprTy;
 
-  const IdType &id() const { return symbol; }
-  const std::optional<ExprType> &ref() const { return designator; }
+  const IdTy &id() const { return symbol; }
+  const std::optional<ExprTy> &ref() const { return designator; }
 
-  IdType symbol;
-  std::optional<ExprType> designator;
+  IdTy symbol;
+  std::optional<ExprTy> designator;
 };
-} // namespace tomp
+} // namespace tomp::type
 
 namespace Fortran::lower::omp {
 
-using Object = tomp::ObjectT<SymIdent, SymReference>;
-using ObjectList = tomp::ObjectListT<SymIdent, SymReference>;
+using Object = tomp::ObjectT<IdTy, ExprTy>;
+using ObjectList = tomp::ObjectListT<IdTy, ExprTy>;
 
 Object makeObject(const parser::OmpObject &object,
                   semantics::SemanticsContext &semaCtx);
@@ -94,13 +93,14 @@ inline ObjectList makeList(const parser::OmpObjectList &objects,
   return makeList(objects.v, makeObjectFn(semaCtx));
 }
 
-template <typename FuncTy, typename ElemTy,
-          typename ResultTy = std::invoke_result_t<FuncTy, ElemTy>>
+template <typename FuncTy, //
+          typename ArgTy,  //
+          typename ResultTy = std::invoke_result_t<FuncTy, ArgTy>>
 std::optional<ResultTy> maybeApply(FuncTy &&func,
-                                   const std::optional<ElemTy> &inp) {
-  if (!inp)
+                                   const std::optional<ArgTy> &arg) {
+  if (!arg)
     return std::nullopt;
-  return std::move(func(*inp));
+  return std::move(func(*arg));
 }
 
 std::optional<Object>
@@ -108,91 +108,142 @@ getBaseObject(const Object &object,
               Fortran::semantics::SemanticsContext &semaCtx);
 
 namespace clause {
-using DefinedOperator = tomp::clause::DefinedOperatorT<SymIdent, SymReference>;
-using ProcedureDesignator =
-    tomp::clause::ProcedureDesignatorT<SymIdent, SymReference>;
-using ReductionOperator =
-    tomp::clause::ReductionOperatorT<SymIdent, SymReference>;
-
-#ifdef EMPTY_CLASS
-#undef EMPTY_CLASS
-#endif
-#define EMPTY_CLASS(cls)                                                       \
-  using cls = tomp::clause::cls##T<SymIdent, SymReference>
-
-#ifdef WRAPPER_CLASS
-#undef WRAPPER_CLASS
-#endif
-#define WRAPPER_CLASS(cls, content)                                            \
-  [[maybe_unused]] extern int xyzzy_semicolon_absorber
-#define GEN_FLANG_CLAUSE_PARSER_CLASSES
-#include "llvm/Frontend/OpenMP/OMP.inc"
-#undef EMPTY_CLASS
-#undef WRAPPER_CLASS
+using DefinedOperator = tomp::type::DefinedOperatorT<IdTy, ExprTy>;
+using ProcedureDesignator = tomp::type::ProcedureDesignatorT<IdTy, ExprTy>;
+using ReductionOperator = tomp::type::ReductionIdentifierT<IdTy, ExprTy>;
 
 // "Requires" clauses are handled early on, and the aggregated information
 // is stored in the Symbol details of modules, programs, and subprograms.
 // These clauses are still handled here to cover all alternatives in the
 // main clause variant.
 
-using Aligned = tomp::clause::AlignedT<SymIdent, SymReference>;
-using Allocate = tomp::clause::AllocateT<SymIdent, SymReference>;
-using Allocator = tomp::clause::AllocatorT<SymIdent, SymReference>;
+using AcqRel = tomp::clause::AcqRelT<TypeTy, IdTy, ExprTy>;
+using Acquire = tomp::clause::AcquireT<TypeTy, IdTy, ExprTy>;
+using AdjustArgs = tomp::clause::AdjustArgsT<TypeTy, IdTy, ExprTy>;
+using Affinity = tomp::clause::AffinityT<TypeTy, IdTy, ExprTy>;
+using Aligned = tomp::clause::AlignedT<TypeTy, IdTy, ExprTy>;
+using Align = tomp::clause::AlignT<TypeTy, IdTy, ExprTy>;
+using Allocate = tomp::clause::AllocateT<TypeTy, IdTy, ExprTy>;
+using Allocator = tomp::clause::AllocatorT<TypeTy, IdTy, ExprTy>;
+using AppendArgs = tomp::clause::AppendArgsT<TypeTy, IdTy, ExprTy>;
 using AtomicDefaultMemOrder =
-    tomp::clause::AtomicDefaultMemOrderT<SymIdent, SymReference>;
-using Collapse = tomp::clause::CollapseT<SymIdent, SymReference>;
-using Copyin = tomp::clause::CopyinT<SymIdent, SymReference>;
-using Copyprivate = tomp::clause::CopyprivateT<SymIdent, SymReference>;
-using Defaultmap = tomp::clause::DefaultmapT<SymIdent, SymReference>;
-using Default = tomp::clause::DefaultT<SymIdent, SymReference>;
-using Depend = tomp::clause::DependT<SymIdent, SymReference>;
-using Device = tomp::clause::DeviceT<SymIdent, SymReference>;
-using DeviceType = tomp::clause::DeviceTypeT<SymIdent, SymReference>;
-using DistSchedule = tomp::clause::DistScheduleT<SymIdent, SymReference>;
-using Enter = tomp::clause::EnterT<SymIdent, SymReference>;
-using Filter = tomp::clause::FilterT<SymIdent, SymReference>;
-using Final = tomp::clause::FinalT<SymIdent, SymReference>;
-using Firstprivate = tomp::clause::FirstprivateT<SymIdent, SymReference>;
-using From = tomp::clause::FromT<SymIdent, SymReference>;
-using Grainsize = tomp::clause::GrainsizeT<SymIdent, SymReference>;
-using HasDeviceAddr = tomp::clause::HasDeviceAddrT<SymIdent, SymReference>;
-using Hint = tomp::clause::HintT<SymIdent, SymReference>;
-using If = tomp::clause::IfT<SymIdent, SymReference>;
-using InReduction = tomp::clause::InReductionT<SymIdent, SymReference>;
-using IsDevicePtr = tomp::clause::IsDevicePtrT<SymIdent, SymReference>;
-using Lastprivate = tomp::clause::LastprivateT<SymIdent, SymReference>;
-using Linear = tomp::clause::LinearT<SymIdent, SymReference>;
-using Link = tomp::clause::LinkT<SymIdent, SymReference>;
-using Map = tomp::clause::MapT<SymIdent, SymReference>;
-using Nocontext = tomp::clause::NocontextT<SymIdent, SymReference>;
-using Nontemporal = tomp::clause::NontemporalT<SymIdent, SymReference>;
-using Novariants = tomp::clause::NovariantsT<SymIdent, SymReference>;
-using NumTasks = tomp::clause::NumTasksT<SymIdent, SymReference>;
-using NumTeams = tomp::clause::NumTeamsT<SymIdent, SymReference>;
-using NumThreads = tomp::clause::NumThreadsT<SymIdent, SymReference>;
-using OmpxDynCgroupMem =
-    tomp::clause::OmpxDynCgroupMemT<SymIdent, SymReference>;
-using Ordered = tomp::clause::OrderedT<SymIdent, SymReference>;
-using Order = tomp::clause::OrderT<SymIdent, SymReference>;
-using Partial = tomp::clause::PartialT<SymIdent, SymReference>;
-using Priority = tomp::clause::PriorityT<SymIdent, SymReference>;
-using Private = tomp::clause::PrivateT<SymIdent, SymReference>;
-using ProcBind = tomp::clause::ProcBindT<SymIdent, SymReference>;
-using Reduction = tomp::clause::ReductionT<SymIdent, SymReference>;
-using Safelen = tomp::clause::SafelenT<SymIdent, SymReference>;
-using Schedule = tomp::clause::ScheduleT<SymIdent, SymReference>;
-using Shared = tomp::clause::SharedT<SymIdent, SymReference>;
-using Simdlen = tomp::clause::SimdlenT<SymIdent, SymReference>;
-using Sizes = tomp::clause::SizesT<SymIdent, SymReference>;
-using TaskReduction = tomp::clause::TaskReductionT<SymIdent, SymReference>;
-using ThreadLimit = tomp::clause::ThreadLimitT<SymIdent, SymReference>;
-using To = tomp::clause::ToT<SymIdent, SymReference>;
-using Uniform = tomp::clause::UniformT<SymIdent, SymReference>;
-using UseDeviceAddr = tomp::clause::UseDeviceAddrT<SymIdent, SymReference>;
-using UseDevicePtr = tomp::clause::UseDevicePtrT<SymIdent, SymReference>;
+    tomp::clause::AtomicDefaultMemOrderT<TypeTy, IdTy, ExprTy>;
+using At = tomp::clause::AtT<TypeTy, IdTy, ExprTy>;
+using Bind = tomp::clause::BindT<TypeTy, IdTy, ExprTy>;
+using Capture = tomp::clause::CaptureT<TypeTy, IdTy, ExprTy>;
+using Collapse = tomp::clause::CollapseT<TypeTy, IdTy, ExprTy>;
+using Compare = tomp::clause::CompareT<TypeTy, IdTy, ExprTy>;
+using Copyin = tomp::clause::CopyinT<TypeTy, IdTy, ExprTy>;
+using Copyprivate = tomp::clause::CopyprivateT<TypeTy, IdTy, ExprTy>;
+using Defaultmap = tomp::clause::DefaultmapT<TypeTy, IdTy, ExprTy>;
+using Default = tomp::clause::DefaultT<TypeTy, IdTy, ExprTy>;
+using Depend = tomp::clause::DependT<TypeTy, IdTy, ExprTy>;
+using Destroy = tomp::clause::DestroyT<TypeTy, IdTy, ExprTy>;
+using Detach = tomp::clause::DetachT<TypeTy, IdTy, ExprTy>;
+using Device = tomp::clause::DeviceT<TypeTy, IdTy, ExprTy>;
+using DeviceType = tomp::clause::DeviceTypeT<TypeTy, IdTy, ExprTy>;
+using DistSchedule = tomp::clause::DistScheduleT<TypeTy, IdTy, ExprTy>;
+using Doacross = tomp::clause::DoacrossT<TypeTy, IdTy, ExprTy>;
+using DynamicAllocators =
+    tomp::clause::DynamicAllocatorsT<TypeTy, IdTy, ExprTy>;
+using Enter = tomp::clause::EnterT<TypeTy, IdTy, ExprTy>;
+using Exclusive = tomp::clause::ExclusiveT<TypeTy, IdTy, ExprTy>;
+using Fail = tomp::clause::FailT<TypeTy, IdTy, ExprTy>;
+using Filter = tomp::clause::FilterT<TypeTy, IdTy, ExprTy>;
+using Final = tomp::clause::FinalT<TypeTy, IdTy, ExprTy>;
+using Firstprivate = tomp::clause::FirstprivateT<TypeTy, IdTy, ExprTy>;
+using From = tomp::clause::FromT<TypeTy, IdTy, ExprTy>;
+using Full = tomp::clause::FullT<TypeTy, IdTy, ExprTy>;
+using Grainsize = tomp::clause::GrainsizeT<TypeTy, IdTy, ExprTy>;
+using HasDeviceAddr = tomp::clause::HasDeviceAddrT<TypeTy, IdTy, ExprTy>;
+using Hint = tomp::clause::HintT<TypeTy, IdTy, ExprTy>;
+using If = tomp::clause::IfT<TypeTy, IdTy, ExprTy>;
+using Inbranch = tomp::clause::InbranchT<TypeTy, IdTy, ExprTy>;
+using Inclusive = tomp::clause::InclusiveT<TypeTy, IdTy, ExprTy>;
+using Indirect = tomp::clause::IndirectT<TypeTy, IdTy, ExprTy>;
+using Init = tomp::clause::InitT<TypeTy, IdTy, ExprTy>;
+using InReduction = tomp::clause::InReductionT<TypeTy, IdTy, ExprTy>;
+using IsDevicePtr = tomp::clause::IsDevicePtrT<TypeTy, IdTy, ExprTy>;
+using Lastprivate = tomp::clause::LastprivateT<TypeTy, IdTy, ExprTy>;
+using Linear = tomp::clause::LinearT<TypeTy, IdTy, ExprTy>;
+using Link = tomp::clause::LinkT<TypeTy, IdTy, ExprTy>;
+using Map = tomp::clause::MapT<TypeTy, IdTy, ExprTy>;
+using Match = tomp::clause::MatchT<TypeTy, IdTy, ExprTy>;
+using Mergeable = tomp::clause::MergeableT<TypeTy, IdTy, ExprTy>;
+using Message = tomp::clause::MessageT<TypeTy, IdTy, ExprTy>;
+using Nocontext = tomp::clause::NocontextT<TypeTy, IdTy, ExprTy>;
+using Nogroup = tomp::clause::NogroupT<TypeTy, IdTy, ExprTy>;
+using Nontemporal = tomp::clause::NontemporalT<TypeTy, IdTy, ExprTy>;
+using Notinbranch = tomp::clause::NotinbranchT<TypeTy, IdTy, ExprTy>;
+using Novariants = tomp::clause::NovariantsT<TypeTy, IdTy, ExprTy>;
+using Nowait = tomp::clause::NowaitT<TypeTy, IdTy, ExprTy>;
+using NumTasks = tomp::clause::NumTasksT<TypeTy, IdTy, ExprTy>;
+using NumTeams = tomp::clause::NumTeamsT<TypeTy, IdTy, ExprTy>;
+using NumThreads = tomp::clause::NumThreadsT<TypeTy, IdTy, ExprTy>;
+using OmpxAttribute = tomp::clause::OmpxAttributeT<TypeTy, IdTy, ExprTy>;
+using OmpxBare = tomp::clause::OmpxBareT<TypeTy, IdTy, ExprTy>;
+using OmpxDynCgroupMem = tomp::clause::OmpxDynCgroupMemT<TypeTy, IdTy, ExprTy>;
+using Ordered = tomp::clause::OrderedT<TypeTy, IdTy, ExprTy>;
+using Order = tomp::clause::OrderT<TypeTy, IdTy, ExprTy>;
+using Partial = tomp::clause::PartialT<TypeTy, IdTy, ExprTy>;
+using Priority = tomp::clause::PriorityT<TypeTy, IdTy, ExprTy>;
+using Private = tomp::clause::PrivateT<TypeTy, IdTy, ExprTy>;
+using ProcBind = tomp::clause::ProcBindT<TypeTy, IdTy, ExprTy>;
+using Read = tomp::clause::ReadT<TypeTy, IdTy, ExprTy>;
+using Reduction = tomp::clause::ReductionT<TypeTy, IdTy, ExprTy>;
+using Relaxed = tomp::clause::RelaxedT<TypeTy, IdTy, ExprTy>;
+using Release = tomp::clause::ReleaseT<TypeTy, IdTy, ExprTy>;
+using ReverseOffload = tomp::clause::ReverseOffloadT<TypeTy, IdTy, ExprTy>;
+using Safelen = tomp::clause::SafelenT<TypeTy, IdTy, ExprTy>;
+using Schedule = tomp::clause::ScheduleT<TypeTy, IdTy, ExprTy>;
+using SeqCst = tomp::clause::SeqCstT<TypeTy, IdTy, ExprTy>;
+using Severity = tomp::clause::SeverityT<TypeTy, IdTy, ExprTy>;
+using Shared = tomp::clause::SharedT<TypeTy, IdTy, ExprTy>;
+using Simdlen = tomp::clause::SimdlenT<TypeTy, IdTy, ExprTy>;
+using Simd = tomp::clause::SimdT<TypeTy, IdTy, ExprTy>;
+using Sizes = tomp::clause::SizesT<TypeTy, IdTy, ExprTy>;
+using TaskReduction = tomp::clause::TaskReductionT<TypeTy, IdTy, ExprTy>;
+using ThreadLimit = tomp::clause::ThreadLimitT<TypeTy, IdTy, ExprTy>;
+using Threads = tomp::clause::ThreadsT<TypeTy, IdTy, ExprTy>;
+using To = tomp::clause::ToT<TypeTy, IdTy, ExprTy>;
+using UnifiedAddress = tomp::clause::UnifiedAddressT<TypeTy, IdTy, ExprTy>;
+using UnifiedSharedMemory =
+    tomp::clause::UnifiedSharedMemoryT<TypeTy, IdTy, ExprTy>;
+using Uniform = tomp::clause::UniformT<TypeTy, IdTy, ExprTy>;
+using Unknown = tomp::clause::UnknownT<TypeTy, IdTy, ExprTy>;
+using Untied = tomp::clause::UntiedT<TypeTy, IdTy, ExprTy>;
+using Update = tomp::clause::UpdateT<TypeTy, IdTy, ExprTy>;
+using UseDeviceAddr = tomp::clause::UseDeviceAddrT<TypeTy, IdTy, ExprTy>;
+using UseDevicePtr = tomp::clause::UseDevicePtrT<TypeTy, IdTy, ExprTy>;
+using UsesAllocators = tomp::clause::UsesAllocatorsT<TypeTy, IdTy, ExprTy>;
+using Use = tomp::clause::UseT<TypeTy, IdTy, ExprTy>;
+using Weak = tomp::clause::WeakT<TypeTy, IdTy, ExprTy>;
+using When = tomp::clause::WhenT<TypeTy, IdTy, ExprTy>;
+using Write = tomp::clause::WriteT<TypeTy, IdTy, ExprTy>;
 } // namespace clause
 
-struct Clause : public tomp::ClauseT<SymIdent, SymReference> {
+struct CancellationConstructType {
+  using EmptyTrait = std::true_type;
+};
+struct Depobj {
+  using EmptyTrait = std::true_type;
+};
+struct Flush {
+  using EmptyTrait = std::true_type;
+};
+struct MemoryOrder {
+  using EmptyTrait = std::true_type;
+};
+struct Threadprivate {
+  using EmptyTrait = std::true_type;
+};
+
+using ClauseBase = tomp::ClauseT<TypeTy, IdTy, ExprTy,
+                                 // Extras...
+                                 CancellationConstructType, Depobj, Flush,
+                                 MemoryOrder, Threadprivate>;
+
+struct Clause : public ClauseBase {
   parser::CharBlock source;
 };
 
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index e1aa7524e930..e114ab9f4548 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -99,7 +99,8 @@ void DataSharingProcessor::collectSymbolsForPrivatization() {
       collectOmpObjectListSymbol(firstPrivateClause->v, privatizedSymbols);
     } else if (const auto &lastPrivateClause =
                    std::get_if<omp::clause::Lastprivate>(&clause.u)) {
-      collectOmpObjectListSymbol(lastPrivateClause->v, privatizedSymbols);
+      const ObjectList &objects = std::get<ObjectList>(lastPrivateClause->t);
+      collectOmpObjectListSymbol(objects, privatizedSymbols);
       hasLastPrivateOp = true;
     } else if (std::get_if<omp::clause::Collapse>(&clause.u)) {
       hasCollapse = true;
@@ -286,12 +287,13 @@ void DataSharingProcessor::collectSymbols(
 }
 
 void DataSharingProcessor::collectDefaultSymbols() {
+  using DataSharingAttribute = omp::clause::Default::DataSharingAttribute;
   for (const omp::Clause &clause : clauses) {
     if (const auto *defaultClause =
             std::get_if<omp::clause::Default>(&clause.u)) {
-      if (defaultClause->v == omp::clause::Default::Type::Private)
+      if (defaultClause->v == DataSharingAttribute::Private)
         collectSymbols(Fortran::semantics::Symbol::Flag::OmpPrivate);
-      else if (defaultClause->v == omp::clause::Default::Type::Firstprivate)
+      else if (defaultClause->v == DataSharingAttribute::Firstprivate)
         collectSymbols(Fortran::semantics::Symbol::Flag::OmpFirstPrivate);
     }
   }
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 02901ed070a7..0cf2a8f97040 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -13,6 +13,7 @@
 #include "flang/Lower/OpenMP.h"
 
 #include "ClauseProcessor.h"
+#include "Clauses.h"
 #include "DataSharingProcessor.h"
 #include "DirectivesCommon.h"
 #include "ReductionProcessor.h"
@@ -555,7 +556,7 @@ genParallelOp(Fortran::lower::AbstractConverter &converter,
   llvm::SmallVector<const Fortran::semantics::Symbol *> reductionSymbols;
 
   ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processIf(clause::If::DirectiveNameModifier::Parallel, ifClauseOperand);
+  cp.processIf(llvm::omp::Directive::OMPD_parallel, ifClauseOperand);
   cp.processNumThreads(stmtCtx, numThreadsClauseOperand);
   cp.processProcBind(procBindKindAttr);
   cp.processDefault();
@@ -726,7 +727,7 @@ genTaskOp(Fortran::lower::AbstractConverter &converter,
       dependOperands;
 
   ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processIf(clause::If::DirectiveNameModifier::Task, ifClauseOperand);
+  cp.processIf(llvm::omp::Directive::OMPD_task, ifClauseOperand);
   cp.processAllocate(allocatorOperands, allocateOperands);
   cp.processDefault();
   cp.processFinal(stmtCtx, finalClauseOperand);
@@ -838,7 +839,7 @@ genTargetDataOp(Fortran::lower::AbstractConverter &converter,
   llvm::SmallVector<const Fortran::semantics::Symbol *> useDeviceSymbols;
 
   ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processIf(clause::If::DirectiveNameModifier::TargetData, ifClauseOperand);
+  cp.processIf(llvm::omp::Directive::OMPD_target_data, ifClauseOperand);
   cp.processDevice(stmtCtx, deviceOperand);
   cp.processUseDevicePtr(devicePtrOperands, useDeviceTypes, useDeviceLocs,
                          useDeviceSymbols);
@@ -883,24 +884,20 @@ static OpTy genTargetEnterExitDataUpdateOp(
   llvm::SmallVector<mlir::Value> mapOperands, dependOperands;
   llvm::SmallVector<mlir::Attribute> dependTypeOperands;
 
-  clause::If::DirectiveNameModifier directiveName;
   // GCC 9.3.0 emits a (probably) bogus warning about an unused variable.
   [[maybe_unused]] llvm::omp::Directive directive;
   if constexpr (std::is_same_v<OpTy, mlir::omp::TargetEnterDataOp>) {
-    directiveName = clause::If::DirectiveNameModifier::TargetEnterData;
     directive = llvm::omp::Directive::OMPD_target_enter_data;
   } else if constexpr (std::is_same_v<OpTy, mlir::omp::TargetExitDataOp>) {
-    directiveName = clause::If::DirectiveNameModifier::TargetExitData;
     directive = llvm::omp::Directive::OMPD_target_exit_data;
   } else if constexpr (std::is_same_v<OpTy, mlir::omp::TargetUpdateOp>) {
-    directiveName = clause::If::DirectiveNameModifier::TargetUpdate;
     directive = llvm::omp::Directive::OMPD_target_update;
   } else {
     return nullptr;
   }
 
   ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processIf(directiveName, ifClauseOperand);
+  cp.processIf(directive, ifClauseOperand);
   cp.processDevice(stmtCtx, deviceOperand);
   cp.processDepend(dependTypeOperands, dependOperands);
   cp.processNowait(nowaitAttr);
@@ -1092,7 +1089,7 @@ genTargetOp(Fortran::lower::AbstractConverter &converter,
   llvm::SmallVector<const Fortran::semantics::Symbol *> mapSymbols;
 
   ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processIf(clause::If::DirectiveNameModifier::Target, ifClauseOperand);
+  cp.processIf(llvm::omp::Directive::OMPD_target, ifClauseOperand);
   cp.processDevice(stmtCtx, deviceOperand);
   cp.processThreadLimit(stmtCtx, threadLimitOperand);
   cp.processDepend(dependTypeOperands, dependOperands);
@@ -1218,7 +1215,7 @@ genTeamsOp(Fortran::lower::AbstractConverter &converter,
   llvm::SmallVector<mlir::Attribute> reductionDeclSymbols;
 
   ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processIf(clause::If::DirectiveNameModifier::Teams, ifClauseOperand);
+  cp.processIf(llvm::omp::Directive::OMPD_teams, ifClauseOperand);
   cp.processAllocate(allocatorOperands, allocateOperands);
   cp.processDefault();
   cp.processNumTeams(stmtCtx, numTeamsClauseOperand);
@@ -1510,7 +1507,7 @@ createSimdLoop(Fortran::lower::AbstractConverter &converter,
   cp.processCollapse(loc, eval, lowerBound, upperBound, step, iv);
   cp.processScheduleChunk(stmtCtx, scheduleChunkClauseOperand);
   cp.processReduction(loc, reductionVars, reductionTypes, reductionDeclSymbols);
-  cp.processIf(clause::If::DirectiveNameModifier::Simd, ifClauseOperand);
+  cp.processIf(llvm::omp::Directive::OMPD_simd, ifClauseOperand);
   cp.processSimdlen(simdlenClauseOperand);
   cp.processSafelen(safelenClauseOperand);
   cp.processTODO<clause::Aligned, clause::Allocate, clause::Linear,
@@ -2360,8 +2357,11 @@ void Fortran::lower::genOpenMPReduction(
   for (const Clause &clause : clauses) {
     if (const auto &reductionClause =
             std::get_if<clause::Reduction>(&clause.u)) {
-      const auto &redOperator{
-          std::get<clause::ReductionOperator>(reductionClause->t)};
+      const auto &redOperatorList{
+          std::get<clause::Reduction::ReductionIdentifiers>(
+              reductionClause->t)};
+      assert(redOperatorList.size() == 1 && "Expecting single operator");
+      const auto &redOperator = redOperatorList.front();
       const auto &objects{std::get<ObjectList>(reductionClause->t)};
       if (const auto *reductionOp =
               std::get_if<clause::DefinedOperator>(&redOperator.u)) {
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
index 2477f635792a..0d05ca5aee65 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
@@ -496,8 +496,10 @@ void ReductionProcessor::addDeclareReduction(
         *reductionSymbols) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   mlir::omp::DeclareReductionOp decl;
-  const auto &redOperator{
-      std::get<omp::clause::ReductionOperator>(reduction.t)};
+  const auto &redOperatorList{
+      std::get<omp::clause::Reduction::ReductionIdentifiers>(reduction.t)};
+  assert(redOperatorList.size() == 1 && "Expecting single operator");
+  const auto &redOperator = redOperatorList.front();
   const auto &objectList{std::get<omp::ObjectList>(reduction.t)};
 
   if (!std::holds_alternative<omp::clause::DefinedOperator>(redOperator.u)) {
diff --git a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
new file mode 100644
index 000000000000..c48b9169c60e
--- /dev/null
+++ b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
@@ -0,0 +1,1123 @@
+//===- ClauseT.h -- clause template definitions ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef FORTRAN_LOWER_OPENMP_CLAUSET_H
+#define FORTRAN_LOWER_OPENMP_CLAUSET_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <algorithm>
+#include <iterator>
+#include <optional>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <variant>
+
+#define ENUM(Name, ...) enum class Name { __VA_ARGS__ }
+#define OPT(x) std::optional<x>
+
+// A number of OpenMP clauses contain values that come from a given set of
+// possibilities. In the IR these are usually represented by enums. Both
+// clang and flang use different types for the enums, and the enum elements
+// representing the same thing may have different values between clang and
+// flang.
+// Since the representation below tries to adhere to the spec, and be source
+// language agnostic, it defines its own enums, independent from any language
+// frontend. As a consequence, when instantiating the templates below,
+// frontend-specific enums need to be translated into the representation
+// used here. The macros below are intended to assist with the conversion.
+
+// Helper macro for enum-class conversion.
+#define CLAUSET_SCOPED_ENUM_MEMBER_CONVERT(Ov, Tv)                             \
+  if (v == OtherEnum::Ov) {                                                    \
+    return ThisEnum::Tv;                                                       \
+  }
+
+// Helper macro for enum (non-class) conversion.
+#define CLAUSET_UNSCOPED_ENUM_MEMBER_CONVERT(Ov, Tv)                           \
+  if (v == Ov) {                                                               \
+    return ThisEnum::Tv;                                                       \
+  }
+
+#define CLAUSET_ENUM_CONVERT(func, OtherE, ThisE, Maps)                        \
+  auto func = [](OtherE v) -> ThisE {                                          \
+    using ThisEnum = ThisE;                                                    \
+    using OtherEnum = OtherE;                                                  \
+    (void)sizeof(OtherEnum); /*Avoid "unused local typedef" warning*/          \
+    Maps;                                                                      \
+    llvm_unreachable("Unexpected value in " #OtherE);                          \
+  }
+
+// Usage:
+//
+// Given two enums,
+//   enum class Other { o1, o2 };
+//   enum class This { t1, t2 };
+// generate conversion function "Func : Other -> This" with
+//   CLAUSET_ENUM_CONVERT(
+//       Func, Other, This,
+//       CLAUSET_ENUM_MEMBER_CONVERT(o1, t1)      // <- No comma
+//       CLAUSET_ENUM_MEMBER_CONVERT(o2, t2)
+//       ...
+//   )
+//
+// Note that the sequence of M(other-value, this-value) is separated
+// with _spaces_, not commas.
+
+namespace detail {
+// Type trait to determine whether T is a specialization of std::variant.
+template <typename T> struct is_variant {
+  static constexpr bool value = false;
+};
+
+template <typename... Ts> struct is_variant<std::variant<Ts...>> {
+  static constexpr bool value = true;
+};
+
+template <typename T> constexpr bool is_variant_v = is_variant<T>::value;
+
+// Helper utility to create a type which is a union of two given variants.
+template <typename...> struct UnionOfTwo;
+
+template <typename... Types1, typename... Types2>
+struct UnionOfTwo<std::variant<Types1...>, std::variant<Types2...>> {
+  using type = std::variant<Types1..., Types2...>;
+};
+} // namespace detail
+
+namespace tomp {
+namespace type {
+
+// Helper utility to create a type which is a union of an arbitrary number
+// of variants.
+template <typename...> struct Union;
+
+template <> struct Union<> {
+  // Legal to define, illegal to instantiate.
+  using type = std::variant<>;
+};
+
+template <typename T, typename... Ts> struct Union<T, Ts...> {
+  static_assert(detail::is_variant_v<T>);
+  using type =
+      typename detail::UnionOfTwo<T, typename Union<Ts...>::type>::type;
+};
+
+template <typename T> using ListT = llvm::SmallVector<T, 0>;
+
+// The ObjectT class represents a variable (as defined in the OpenMP spec).
+//
+// A specialization of ObjectT<Id, Expr> must provide the following definitions:
+// {
+//    using IdTy = Id;
+//    using ExprTy = Expr;
+//
+//    auto id() const -> IdTy {
+//      return the identifier of the object (for use in tests for
+//         presence/absence of the object)
+//    }
+//
+//    auto ref() const -> (e.g. const ExprTy&) {
+//      return the expression accessing (referencing) the object
+//    }
+// }
+//
+// For example, the ObjectT instance created for "var[x+1]" would have
+// the `id()` return the identifier for `var`, and the `ref()` return the
+// representation of the array-access `var[x+1]`.
+//
+// The identity of an object must always be present, i.e. it cannot be
+// nullptr, std::nullopt, etc. The reference is optional.
+//
+// Note: the ObjectT template is not defined. Any user of it is expected to
+// provide their own specialization that conforms to the above requirements.
+template <typename IdType, typename ExprType> struct ObjectT;
+
+template <typename I, typename E> using ObjectListT = ListT<ObjectT<I, E>>;
+
+using DirectiveName = llvm::omp::Directive;
+
+template <typename I, typename E> //
+struct DefinedOperatorT {
+  struct DefinedOpName {
+    using WrapperTrait = std::true_type;
+    ObjectT<I, E> v;
+  };
+  ENUM(IntrinsicOperator, Power, Multiply, Divide, Add, Subtract, Concat, LT,
+       LE, EQ, NE, GE, GT, NOT, AND, OR, EQV, NEQV, Min, Max);
+  using UnionTrait = std::true_type;
+  std::variant<DefinedOpName, IntrinsicOperator> u;
+};
+
+template <typename E> //
+struct RangeT {
+  // range-specification: begin : end[: step]
+  using TupleTrait = std::true_type;
+  std::tuple<E, E, OPT(E)> t;
+};
+
+template <typename TypeType, typename IdType, typename ExprType> //
+struct IteratorSpecifierT {
+  // iterators-specifier: [ iterator-type ] identifier = range-specification
+  using TupleTrait = std::true_type;
+  std::tuple<OPT(TypeType), ObjectT<IdType, ExprType>, RangeT<ExprType>> t;
+};
+
+// Note:
+// For motion or map clauses the OpenMP spec allows a unique mapper modifier.
+// In practice, since these clauses apply to multiple objects, there can be
+// multiple effective mappers applicable to these objects (due to overloads,
+// etc.). Because of that store a list of mappers every time a mapper modifier
+// is allowed. If the mapper list contains a single element, it applies to
+// all objects in the clause, otherwise there should be as many mappers as
+// there are objects.
+template <typename I, typename E> //
+struct MapperT {
+  using MapperIdentifier = ObjectT<I, E>;
+  using WrapperTrait = std::true_type;
+  MapperIdentifier v;
+};
+
+ENUM(MemoryOrder, AcqRel, Acquire, Relaxed, Release, SeqCst);
+ENUM(MotionExpectation, Present);
+ENUM(TaskDependenceType, In, Out, Inout, Mutexinoutset, Inoutset, Depobj);
+
+template <typename I, typename E> //
+struct LoopIterationT {
+  struct Distance {
+    using TupleTrait = std::true_type;
+    std::tuple<DefinedOperatorT<I, E>, E> t;
+  };
+  using TupleTrait = std::true_type;
+  std::tuple<ObjectT<I, E>, OPT(Distance)> t;
+};
+
+template <typename I, typename E> //
+struct ProcedureDesignatorT {
+  using WrapperTrait = std::true_type;
+  ObjectT<I, E> v;
+};
+
+// Note:
+// For reduction clauses the OpenMP spec allows a unique reduction identifier.
+// For reasons analogous to those listed for the MapperT type, clauses that
+// according to the spec contain a reduction identifier will contain a list of
+// reduction identifiers. The same constraints apply: there is either a single
+// identifier that applies to all objects, or there are as many identifiers
+// as there are objects.
+template <typename I, typename E> //
+struct ReductionIdentifierT {
+  using UnionTrait = std::true_type;
+  std::variant<DefinedOperatorT<I, E>, ProcedureDesignatorT<I, E>> u;
+};
+
+template <typename T, typename I, typename E> //
+using IteratorT = ListT<IteratorSpecifierT<T, I, E>>;
+} // namespace type
+
+template <typename T> using ListT = type::ListT<T>;
+
+template <typename I, typename E> using ObjectT = type::ObjectT<I, E>;
+template <typename I, typename E> using ObjectListT = type::ObjectListT<I, E>;
+
+template <typename T, typename I, typename E>
+using IteratorT = type::IteratorT<T, I, E>;
+
+template <
+    typename ContainerTy, typename FunctionTy,
+    typename ElemTy = typename llvm::remove_cvref_t<ContainerTy>::value_type,
+    typename ResultTy = std::invoke_result_t<FunctionTy, ElemTy>>
+ListT<ResultTy> makeList(ContainerTy &&container, FunctionTy &&func) {
+  ListT<ResultTy> v;
+  llvm::transform(container, std::back_inserter(v), func);
+  return v;
+}
+
+namespace clause {
+template <typename T, typename I, typename E> //
+struct AbsentT {
+  using List = ListT<type::DirectiveName>;
+  using WrapperTrait = std::true_type;
+  List v;
+};
+
+template <typename T, typename I, typename E> //
+struct AcqRelT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct AcquireT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct AdjustArgsT {
+  using IncompleteTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct AffinityT {
+  using Iterator = type::IteratorT<T, I, E>;
+  using LocatorList = ObjectListT<I, E>;
+
+  using TupleTrait = std::true_type;
+  std::tuple<OPT(Iterator), LocatorList> t;
+};
+
+template <typename T, typename I, typename E> //
+struct AlignT {
+  using Alignment = E;
+
+  using WrapperTrait = std::true_type;
+  Alignment v;
+};
+
+template <typename T, typename I, typename E> //
+struct AlignedT {
+  using Alignment = E;
+  using List = ObjectListT<I, E>;
+
+  using TupleTrait = std::true_type;
+  std::tuple<OPT(Alignment), List> t;
+};
+
+template <typename T, typename I, typename E> //
+struct AllocatorT;
+
+template <typename T, typename I, typename E> //
+struct AllocateT {
+  using AllocatorSimpleModifier = E;
+  using AllocatorComplexModifier = AllocatorT<T, I, E>;
+  using AlignModifier = AlignT<T, I, E>;
+  using List = ObjectListT<I, E>;
+
+  using TupleTrait = std::true_type;
+  std::tuple<OPT(AllocatorSimpleModifier), OPT(AllocatorComplexModifier),
+             OPT(AlignModifier), List>
+      t;
+};
+
+template <typename T, typename I, typename E> //
+struct AllocatorT {
+  using Allocator = E;
+  using WrapperTrait = std::true_type;
+  Allocator v;
+};
+
+template <typename T, typename I, typename E> //
+struct AppendArgsT {
+  using IncompleteTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct AtT {
+  ENUM(ActionTime, Compilation, Execution);
+  using WrapperTrait = std::true_type;
+  ActionTime v;
+};
+
+template <typename T, typename I, typename E> //
+struct AtomicDefaultMemOrderT {
+  using MemoryOrder = type::MemoryOrder;
+  using WrapperTrait = std::true_type;
+  MemoryOrder v; // Name not provided in spec
+};
+
+template <typename T, typename I, typename E> //
+struct BindT {
+  ENUM(Binding, Teams, Parallel, Thread);
+  using WrapperTrait = std::true_type;
+  Binding v;
+};
+
+template <typename T, typename I, typename E> //
+struct CaptureT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct CollapseT {
+  using N = E;
+  using WrapperTrait = std::true_type;
+  N v;
+};
+
+template <typename T, typename I, typename E> //
+struct CompareT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct ContainsT {
+  using List = ListT<type::DirectiveName>;
+  using WrapperTrait = std::true_type;
+  List v;
+};
+
+template <typename T, typename I, typename E> //
+struct CopyinT {
+  using List = ObjectListT<I, E>;
+  using WrapperTrait = std::true_type;
+  List v;
+};
+
+template <typename T, typename I, typename E> //
+struct CopyprivateT {
+  using List = ObjectListT<I, E>;
+  using WrapperTrait = std::true_type;
+  List v;
+};
+
+template <typename T, typename I, typename E> //
+struct DefaultT {
+  ENUM(DataSharingAttribute, Firstprivate, None, Private, Shared);
+  using WrapperTrait = std::true_type;
+  DataSharingAttribute v;
+};
+
+template <typename T, typename I, typename E> //
+struct DefaultmapT {
+  ENUM(ImplicitBehavior, Alloc, To, From, Tofrom, Firstprivate, None, Default,
+       Present);
+  ENUM(VariableCategory, Scalar, Aggregate, Pointer, Allocatable);
+  using TupleTrait = std::true_type;
+  std::tuple<ImplicitBehavior, OPT(VariableCategory)> t;
+};
+
+template <typename T, typename I, typename E> //
+struct DoacrossT;
+
+template <typename T, typename I, typename E> //
+struct DependT {
+  using Iterator = type::IteratorT<T, I, E>;
+  using LocatorList = ObjectListT<I, E>;
+  using TaskDependenceType = tomp::type::TaskDependenceType;
+
+  struct WithLocators { // Modern form
+    using TupleTrait = std::true_type;
+    // Empty LocatorList means "omp_all_memory".
+    std::tuple<TaskDependenceType, OPT(Iterator), LocatorList> t;
+  };
+
+  using Doacross = DoacrossT<T, I, E>;
+  using UnionTrait = std::true_type;
+  std::variant<Doacross, WithLocators> u; // Doacross form is legacy
+};
+
+template <typename T, typename I, typename E> //
+struct DestroyT {
+  using DestroyVar = ObjectT<I, E>;
+  using WrapperTrait = std::true_type;
+  // DestroyVar can be ommitted in "depobj destroy".
+  OPT(DestroyVar) v;
+};
+
+template <typename T, typename I, typename E> //
+struct DetachT {
+  using EventHandle = ObjectT<I, E>;
+  using WrapperTrait = std::true_type;
+  EventHandle v;
+};
+
+template <typename T, typename I, typename E> //
+struct DeviceT {
+  using DeviceDescription = E;
+  ENUM(DeviceModifier, Ancestor, DeviceNum);
+  using TupleTrait = std::true_type;
+  std::tuple<OPT(DeviceModifier), DeviceDescription> t;
+};
+
+template <typename T, typename I, typename E> //
+struct DeviceTypeT {
+  ENUM(DeviceTypeDescription, Any, Host, Nohost);
+  using WrapperTrait = std::true_type;
+  DeviceTypeDescription v;
+};
+
+template <typename T, typename I, typename E> //
+struct DistScheduleT {
+  ENUM(Kind, Static);
+  using ChunkSize = E;
+  using TupleTrait = std::true_type;
+  std::tuple<Kind, OPT(ChunkSize)> t;
+};
+
+template <typename T, typename I, typename E> //
+struct DoacrossT {
+  using Vector = ListT<type::LoopIterationT<I, E>>;
+  ENUM(DependenceType, Source, Sink);
+  using TupleTrait = std::true_type;
+  // Empty Vector means "omp_cur_iteration"
+  std::tuple<DependenceType, Vector> t;
+};
+
+template <typename T, typename I, typename E> //
+struct DynamicAllocatorsT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct EnterT {
+  using List = ObjectListT<I, E>;
+  using WrapperTrait = std::true_type;
+  List v;
+};
+
+template <typename T, typename I, typename E> //
+struct ExclusiveT {
+  using WrapperTrait = std::true_type;
+  using List = ObjectListT<I, E>;
+  List v;
+};
+
+template <typename T, typename I, typename E> //
+struct FailT {
+  using MemoryOrder = type::MemoryOrder;
+  using WrapperTrait = std::true_type;
+  MemoryOrder v;
+};
+
+template <typename T, typename I, typename E> //
+struct FilterT {
+  using ThreadNum = E;
+  using WrapperTrait = std::true_type;
+  ThreadNum v;
+};
+
+template <typename T, typename I, typename E> //
+struct FinalT {
+  using Finalize = E;
+  using WrapperTrait = std::true_type;
+  Finalize v;
+};
+
+template <typename T, typename I, typename E> //
+struct FirstprivateT {
+  using List = ObjectListT<I, E>;
+  using WrapperTrait = std::true_type;
+  List v;
+};
+
+template <typename T, typename I, typename E> //
+struct FromT {
+  using LocatorList = ObjectListT<I, E>;
+  using Expectation = type::MotionExpectation;
+  using Iterator = type::IteratorT<T, I, E>;
+  // See note at the definition of the MapperT type.
+  using Mappers = ListT<type::MapperT<I, E>>; // Not a spec name
+
+  using TupleTrait = std::true_type;
+  std::tuple<OPT(Expectation), OPT(Mappers), OPT(Iterator), LocatorList> t;
+};
+
+template <typename T, typename I, typename E> //
+struct FullT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct GrainsizeT {
+  ENUM(Prescriptiveness, Strict);
+  using GrainSize = E;
+  using TupleTrait = std::true_type;
+  std::tuple<OPT(Prescriptiveness), GrainSize> t;
+};
+
+template <typename T, typename I, typename E> //
+struct HasDeviceAddrT {
+  using List = ObjectListT<I, E>;
+  using WrapperTrait = std::true_type;
+  List v;
+};
+
+template <typename T, typename I, typename E> //
+struct HintT {
+  using HintExpr = E;
+  using WrapperTrait = std::true_type;
+  HintExpr v;
+};
+
+template <typename T, typename I, typename E> //
+struct HoldsT {
+  using WrapperTrait = std::true_type;
+  E v; // No argument name in spec 5.2
+};
+
+template <typename T, typename I, typename E> //
+struct IfT {
+  using DirectiveNameModifier = type::DirectiveName;
+  using IfExpression = E;
+  using TupleTrait = std::true_type;
+  std::tuple<OPT(DirectiveNameModifier), IfExpression> t;
+};
+
+template <typename T, typename I, typename E> //
+struct InbranchT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct InclusiveT {
+  using List = ObjectListT<I, E>;
+  using WrapperTrait = std::true_type;
+  List v;
+};
+
+template <typename T, typename I, typename E> //
+struct IndirectT {
+  using InvokedByFptr = E;
+  using WrapperTrait = std::true_type;
+  InvokedByFptr v;
+};
+
+template <typename T, typename I, typename E> //
+struct InitT {
+  using ForeignRuntimeId = E;
+  using InteropVar = ObjectT<I, E>;
+  using InteropPreference = ListT<ForeignRuntimeId>;
+  ENUM(InteropType, Target, Targetsync);   // Repeatable
+  using InteropTypes = ListT<InteropType>; // Not a spec name
+
+  using TupleTrait = std::true_type;
+  std::tuple<OPT(InteropPreference), InteropTypes, InteropVar> t;
+};
+
+template <typename T, typename I, typename E> //
+struct InitializerT {
+  using InitializerExpr = E;
+  using WrapperTrait = std::true_type;
+  InitializerExpr v;
+};
+
+template <typename T, typename I, typename E> //
+struct InReductionT {
+  using List = ObjectListT<I, E>;
+  // See note at the definition of the ReductionIdentifierT type.
+  // The name ReductionIdentifiers is not a spec name.
+  using ReductionIdentifiers = ListT<type::ReductionIdentifierT<I, E>>;
+  using TupleTrait = std::true_type;
+  std::tuple<ReductionIdentifiers, List> t;
+};
+
+template <typename T, typename I, typename E> //
+struct IsDevicePtrT {
+  using List = ObjectListT<I, E>;
+  using WrapperTrait = std::true_type;
+  List v;
+};
+
+template <typename T, typename I, typename E> //
+struct LastprivateT {
+  using List = ObjectListT<I, E>;
+  ENUM(LastprivateModifier, Conditional);
+  using TupleTrait = std::true_type;
+  std::tuple<OPT(LastprivateModifier), List> t;
+};
+
+template <typename T, typename I, typename E> //
+struct LinearT {
+  // std::get<type> won't work here due to duplicate types in the tuple.
+  using List = ObjectListT<I, E>;
+  using StepSimpleModifier = E;
+  using StepComplexModifier = E;
+  ENUM(LinearModifier, Ref, Val, Uval);
+
+  using TupleTrait = std::true_type;
+  // Step == nullptr means 1.
+  std::tuple<OPT(StepSimpleModifier), OPT(StepComplexModifier),
+             OPT(LinearModifier), List>
+      t;
+};
+
+template <typename T, typename I, typename E> //
+struct LinkT {
+  using List = ObjectListT<I, E>;
+  using WrapperTrait = std::true_type;
+  List v;
+};
+
+template <typename T, typename I, typename E> //
+struct MapT {
+  using LocatorList = ObjectListT<I, E>;
+  ENUM(MapType, To, From, Tofrom, Alloc, Release, Delete);
+  ENUM(MapTypeModifier, Always, Close, Present, OmpxHold);
+  // See note at the definition of the MapperT type.
+  using Mappers = ListT<type::MapperT<I, E>>; // Not a spec name
+  using Iterator = type::IteratorT<T, I, E>;
+  using MapTypeModifiers = ListT<MapTypeModifier>; // Not a spec name
+
+  using TupleTrait = std::true_type;
+  std::tuple<OPT(MapType), OPT(MapTypeModifiers), OPT(Mappers), OPT(Iterator),
+             LocatorList>
+      t;
+};
+
+template <typename T, typename I, typename E> //
+struct MatchT {
+  using IncompleteTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct MergeableT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct MessageT {
+  using MsgString = E;
+  using WrapperTrait = std::true_type;
+  MsgString v;
+};
+
+template <typename T, typename I, typename E> //
+struct NocontextT {
+  using DoNotUpdateContext = E;
+  using WrapperTrait = std::true_type;
+  DoNotUpdateContext v;
+};
+
+template <typename T, typename I, typename E> //
+struct NogroupT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct NontemporalT {
+  using List = ObjectListT<I, E>;
+  using WrapperTrait = std::true_type;
+  List v;
+};
+
+template <typename T, typename I, typename E> //
+struct NoOpenmpT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct NoOpenmpRoutinesT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct NoParallelismT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct NotinbranchT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct NovariantsT {
+  using DoNotUseVariant = E;
+  using WrapperTrait = std::true_type;
+  DoNotUseVariant v;
+};
+
+template <typename T, typename I, typename E> //
+struct NowaitT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct NumTasksT {
+  using NumTasks = E;
+  ENUM(Prescriptiveness, Strict);
+  using TupleTrait = std::true_type;
+  std::tuple<OPT(Prescriptiveness), NumTasks> t;
+};
+
+template <typename T, typename I, typename E> //
+struct NumTeamsT {
+  using TupleTrait = std::true_type;
+  using LowerBound = E;
+  using UpperBound = E;
+  std::tuple<OPT(LowerBound), UpperBound> t;
+};
+
+template <typename T, typename I, typename E> //
+struct NumThreadsT {
+  using Nthreads = E;
+  using WrapperTrait = std::true_type;
+  Nthreads v;
+};
+
+template <typename T, typename I, typename E> //
+struct OmpxAttributeT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct OmpxBareT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct OmpxDynCgroupMemT {
+  using WrapperTrait = std::true_type;
+  E v;
+};
+
+template <typename T, typename I, typename E> //
+struct OrderT {
+  ENUM(OrderModifier, Reproducible, Unconstrained);
+  ENUM(Ordering, Concurrent);
+  using TupleTrait = std::true_type;
+  std::tuple<OPT(OrderModifier), Ordering> t;
+};
+
+template <typename T, typename I, typename E> //
+struct OrderedT {
+  using N = E;
+  using WrapperTrait = std::true_type;
+  OPT(N) v;
+};
+
+template <typename T, typename I, typename E> //
+struct OtherwiseT {
+  using IncompleteTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct PartialT {
+  using UnrollFactor = E;
+  using WrapperTrait = std::true_type;
+  OPT(UnrollFactor) v;
+};
+
+template <typename T, typename I, typename E> //
+struct PriorityT {
+  using PriorityValue = E;
+  using WrapperTrait = std::true_type;
+  PriorityValue v;
+};
+
+template <typename T, typename I, typename E> //
+struct PrivateT {
+  using List = ObjectListT<I, E>;
+  using WrapperTrait = std::true_type;
+  List v;
+};
+
+template <typename T, typename I, typename E> //
+struct ProcBindT {
+  ENUM(AffinityPolicy, Close, Master, Spread, Primary);
+  using WrapperTrait = std::true_type;
+  AffinityPolicy v;
+};
+
+template <typename T, typename I, typename E> //
+struct ReadT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct ReductionT {
+  using List = ObjectListT<I, E>;
+  // See note at the definition of the ReductionIdentifierT type.
+  // The name ReductionIdentifiers is not a spec name.
+  using ReductionIdentifiers = ListT<type::ReductionIdentifierT<I, E>>;
+  ENUM(ReductionModifier, Default, Inscan, Task);
+  using TupleTrait = std::true_type;
+  std::tuple<ReductionIdentifiers, OPT(ReductionModifier), List> t;
+};
+
+template <typename T, typename I, typename E> //
+struct RelaxedT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct ReleaseT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct ReverseOffloadT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct SafelenT {
+  using Length = E;
+  using WrapperTrait = std::true_type;
+  Length v;
+};
+
+template <typename T, typename I, typename E> //
+struct ScheduleT {
+  ENUM(Kind, Static, Dynamic, Guided, Auto, Runtime);
+  using ChunkSize = E;
+  ENUM(OrderingModifier, Monotonic, Nonmonotonic);
+  ENUM(ChunkModifier, Simd);
+  using TupleTrait = std::true_type;
+  std::tuple<Kind, OPT(OrderingModifier), OPT(ChunkModifier), OPT(ChunkSize)> t;
+};
+
+template <typename T, typename I, typename E> //
+struct SeqCstT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct SeverityT {
+  ENUM(SevLevel, Fatal, Warning);
+  using WrapperTrait = std::true_type;
+  SevLevel v;
+};
+
+template <typename T, typename I, typename E> //
+struct SharedT {
+  using List = ObjectListT<I, E>;
+  using WrapperTrait = std::true_type;
+  List v;
+};
+
+template <typename T, typename I, typename E> //
+struct SimdT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct SimdlenT {
+  using Length = E;
+  using WrapperTrait = std::true_type;
+  Length v;
+};
+
+template <typename T, typename I, typename E> //
+struct SizesT {
+  using SizeList = ListT<E>;
+  using WrapperTrait = std::true_type;
+  SizeList v;
+};
+
+template <typename T, typename I, typename E> //
+struct TaskReductionT {
+  using List = ObjectListT<I, E>;
+  // See note at the definition of the ReductionIdentifierT type.
+  // The name ReductionIdentifiers is not a spec name.
+  using ReductionIdentifiers = ListT<type::ReductionIdentifierT<I, E>>;
+  using TupleTrait = std::true_type;
+  std::tuple<ReductionIdentifiers, List> t;
+};
+
+template <typename T, typename I, typename E> //
+struct ThreadLimitT {
+  using Threadlim = E;
+  using WrapperTrait = std::true_type;
+  Threadlim v;
+};
+
+template <typename T, typename I, typename E> //
+struct ThreadsT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct ToT {
+  using LocatorList = ObjectListT<I, E>;
+  using Expectation = type::MotionExpectation;
+  // See note at the definition of the MapperT type.
+  using Mappers = ListT<type::MapperT<I, E>>; // Not a spec name
+  using Iterator = type::IteratorT<T, I, E>;
+
+  using TupleTrait = std::true_type;
+  std::tuple<OPT(Expectation), OPT(Mappers), OPT(Iterator), LocatorList> t;
+};
+
+template <typename T, typename I, typename E> //
+struct UnifiedAddressT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct UnifiedSharedMemoryT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct UniformT {
+  using ParameterList = ObjectListT<I, E>;
+  using WrapperTrait = std::true_type;
+  ParameterList v;
+};
+
+template <typename T, typename I, typename E> //
+struct UnknownT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct UntiedT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct UpdateT {
+  using TaskDependenceType = tomp::type::TaskDependenceType;
+  using WrapperTrait = std::true_type;
+  OPT(TaskDependenceType) v;
+};
+
+template <typename T, typename I, typename E> //
+struct UseT {
+  using InteropVar = ObjectT<I, E>;
+  using WrapperTrait = std::true_type;
+  InteropVar v;
+};
+
+template <typename T, typename I, typename E> //
+struct UseDeviceAddrT {
+  using List = ObjectListT<I, E>;
+  using WrapperTrait = std::true_type;
+  List v;
+};
+
+template <typename T, typename I, typename E> //
+struct UseDevicePtrT {
+  using List = ObjectListT<I, E>;
+  using WrapperTrait = std::true_type;
+  List v;
+};
+
+template <typename T, typename I, typename E> //
+struct UsesAllocatorsT {
+  using MemSpace = E;
+  using TraitsArray = ObjectT<I, E>;
+  using Allocator = E;
+  using AllocatorSpec =
+      std::tuple<OPT(MemSpace), OPT(TraitsArray), Allocator>; // Not a spec name
+  using Allocators = ListT<AllocatorSpec>;                    // Not a spec name
+  using WrapperTrait = std::true_type;
+  Allocators v;
+};
+
+template <typename T, typename I, typename E> //
+struct WeakT {
+  using EmptyTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct WhenT {
+  using IncompleteTrait = std::true_type;
+};
+
+template <typename T, typename I, typename E> //
+struct WriteT {
+  using EmptyTrait = std::true_type;
+};
+
+// ---
+
+template <typename T, typename I, typename E>
+using ExtensionClausesT =
+    std::variant<OmpxAttributeT<T, I, E>, OmpxBareT<T, I, E>,
+                 OmpxDynCgroupMemT<T, I, E>>;
+
+template <typename T, typename I, typename E>
+using EmptyClausesT = std::variant<
+    AcqRelT<T, I, E>, AcquireT<T, I, E>, CaptureT<T, I, E>, CompareT<T, I, E>,
+    DynamicAllocatorsT<T, I, E>, FullT<T, I, E>, InbranchT<T, I, E>,
+    MergeableT<T, I, E>, NogroupT<T, I, E>, NoOpenmpRoutinesT<T, I, E>,
+    NoOpenmpT<T, I, E>, NoParallelismT<T, I, E>, NotinbranchT<T, I, E>,
+    NowaitT<T, I, E>, ReadT<T, I, E>, RelaxedT<T, I, E>, ReleaseT<T, I, E>,
+    ReverseOffloadT<T, I, E>, SeqCstT<T, I, E>, SimdT<T, I, E>,
+    ThreadsT<T, I, E>, UnifiedAddressT<T, I, E>, UnifiedSharedMemoryT<T, I, E>,
+    UnknownT<T, I, E>, UntiedT<T, I, E>, UseT<T, I, E>, WeakT<T, I, E>,
+    WriteT<T, I, E>>;
+
+template <typename T, typename I, typename E>
+using IncompleteClausesT =
+    std::variant<AdjustArgsT<T, I, E>, AppendArgsT<T, I, E>, MatchT<T, I, E>,
+                 OtherwiseT<T, I, E>, WhenT<T, I, E>>;
+
+template <typename T, typename I, typename E>
+using TupleClausesT =
+    std::variant<AffinityT<T, I, E>, AlignedT<T, I, E>, AllocateT<T, I, E>,
+                 DefaultmapT<T, I, E>, DeviceT<T, I, E>, DistScheduleT<T, I, E>,
+                 DoacrossT<T, I, E>, FromT<T, I, E>, GrainsizeT<T, I, E>,
+                 IfT<T, I, E>, InitT<T, I, E>, InReductionT<T, I, E>,
+                 LastprivateT<T, I, E>, LinearT<T, I, E>, MapT<T, I, E>,
+                 NumTasksT<T, I, E>, OrderT<T, I, E>, ReductionT<T, I, E>,
+                 ScheduleT<T, I, E>, TaskReductionT<T, I, E>, ToT<T, I, E>>;
+
+template <typename T, typename I, typename E>
+using UnionClausesT = std::variant<DependT<T, I, E>>;
+
+template <typename T, typename I, typename E>
+using WrapperClausesT = std::variant<
+    AbsentT<T, I, E>, AlignT<T, I, E>, AllocatorT<T, I, E>,
+    AtomicDefaultMemOrderT<T, I, E>, AtT<T, I, E>, BindT<T, I, E>,
+    CollapseT<T, I, E>, ContainsT<T, I, E>, CopyinT<T, I, E>,
+    CopyprivateT<T, I, E>, DefaultT<T, I, E>, DestroyT<T, I, E>,
+    DetachT<T, I, E>, DeviceTypeT<T, I, E>, EnterT<T, I, E>,
+    ExclusiveT<T, I, E>, FailT<T, I, E>, FilterT<T, I, E>, FinalT<T, I, E>,
+    FirstprivateT<T, I, E>, HasDeviceAddrT<T, I, E>, HintT<T, I, E>,
+    HoldsT<T, I, E>, InclusiveT<T, I, E>, IndirectT<T, I, E>,
+    InitializerT<T, I, E>, IsDevicePtrT<T, I, E>, LinkT<T, I, E>,
+    MessageT<T, I, E>, NocontextT<T, I, E>, NontemporalT<T, I, E>,
+    NovariantsT<T, I, E>, NumTeamsT<T, I, E>, NumThreadsT<T, I, E>,
+    OrderedT<T, I, E>, PartialT<T, I, E>, PriorityT<T, I, E>, PrivateT<T, I, E>,
+    ProcBindT<T, I, E>, SafelenT<T, I, E>, SeverityT<T, I, E>, SharedT<T, I, E>,
+    SimdlenT<T, I, E>, SizesT<T, I, E>, ThreadLimitT<T, I, E>,
+    UniformT<T, I, E>, UpdateT<T, I, E>, UseDeviceAddrT<T, I, E>,
+    UseDevicePtrT<T, I, E>, UsesAllocatorsT<T, I, E>>;
+
+template <typename T, typename I, typename E>
+using UnionOfAllClausesT = typename type::Union< //
+    EmptyClausesT<T, I, E>,                      //
+    ExtensionClausesT<T, I, E>,                  //
+    IncompleteClausesT<T, I, E>,                 //
+    TupleClausesT<T, I, E>,                      //
+    UnionClausesT<T, I, E>,                      //
+    WrapperClausesT<T, I, E>                     //
+    >::type;
+
+} // namespace clause
+
+// The variant wrapper that encapsulates all possible specific clauses.
+// The `Extras` arguments are additional types representing local extensions
+// to the clause set, e.g.
+//
+// using Clause = ClauseT<Type, Id, Expr,
+//                        MyClause1, MyClause2>;
+//
+// The member Clause::u will be a variant containing all specific clauses
+// defined above, plus MyClause1 and MyClause2.
+template <typename TypeType, typename IdType, typename ExprType,
+          typename... Extras>
+struct ClauseT {
+  using TypeTy = TypeType;
+  using IdTy = IdType;
+  using ExprTy = ExprType;
+
+  using VariantTy = typename type::Union<
+      clause::UnionOfAllClausesT<TypeType, IdType, ExprType>,
+      std::variant<Extras...>>::type;
+
+  llvm::omp::Clause id; // The numeric id of the clause
+  using UnionTrait = std::true_type;
+  VariantTy u;
+};
+
+} // namespace tomp
+
+#undef OPT
+#undef ENUM
+
+#endif // FORTRAN_LOWER_OPENMP_CLAUSET_H
-- 
GitLab


From 601d7eab0665ba298d81952da11593124fd893a0 Mon Sep 17 00:00:00 2001
From: Karthika Devi C <quic_kartc@quicinc.com>
Date: Wed, 27 Mar 2024 00:32:27 +0530
Subject: [PATCH 330/404] [polly] Add polly-debug flag to print debug info from
 all parts of polly (#78549)

This flag enable the user to print debug Info from all the passes and
helpers inside polly at once. This will help a novice user as well to
work in polly without explicitly having to know which parts of polly has
actually kicked in and pass them via -debug-only.
---
 polly/include/polly/Support/PollyDebug.h      | 38 ++++++++
 polly/lib/Analysis/DependenceInfo.cpp         | 39 ++++----
 polly/lib/Analysis/PolyhedralInfo.cpp         | 15 +--
 polly/lib/Analysis/PruneUnprofitable.cpp      |  5 +-
 polly/lib/Analysis/ScopBuilder.cpp            | 36 +++----
 polly/lib/Analysis/ScopDetection.cpp          | 27 +++---
 polly/lib/Analysis/ScopInfo.cpp               |  3 +-
 polly/lib/CMakeLists.txt                      |  1 +
 polly/lib/CodeGen/CodeGeneration.cpp          |  5 +-
 polly/lib/CodeGen/IslAst.cpp                  |  7 +-
 polly/lib/Support/PollyDebug.cpp              | 27 ++++++
 polly/lib/Support/SCEVValidator.cpp           | 34 +++----
 polly/lib/Transform/DeLICM.cpp                | 93 ++++++++++---------
 polly/lib/Transform/FlattenAlgo.cpp           | 37 ++++----
 polly/lib/Transform/FlattenSchedule.cpp       | 17 ++--
 polly/lib/Transform/ForwardOpTree.cpp         | 56 +++++------
 polly/lib/Transform/ManualOptimizer.cpp       |  9 +-
 polly/lib/Transform/MatmulOptimizer.cpp       |  5 +-
 polly/lib/Transform/ScheduleOptimizer.cpp     | 31 ++++---
 polly/lib/Transform/ScheduleTreeTransform.cpp | 11 ++-
 polly/lib/Transform/ScopInliner.cpp           | 13 +--
 polly/lib/Transform/Simplify.cpp              | 47 +++++-----
 polly/lib/Transform/ZoneAlgo.cpp              |  7 +-
 polly/test/Support/pollyDebug.ll              | 85 +++++++++++++++++
 24 files changed, 412 insertions(+), 236 deletions(-)
 create mode 100644 polly/include/polly/Support/PollyDebug.h
 create mode 100644 polly/lib/Support/PollyDebug.cpp
 create mode 100644 polly/test/Support/pollyDebug.ll

diff --git a/polly/include/polly/Support/PollyDebug.h b/polly/include/polly/Support/PollyDebug.h
new file mode 100644
index 000000000000..5cea68efc243
--- /dev/null
+++ b/polly/include/polly/Support/PollyDebug.h
@@ -0,0 +1,38 @@
+//===-PollyDebug.h -Provide support for debugging Polly passes-*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Functions to aid printing Debug Info of all polly passes.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef POLLY_DEBUG_H
+#define POLLY_DEBUG_H
+
+#include "llvm/Support/Debug.h"
+
+namespace polly {
+using namespace llvm;
+bool getPollyDebugFlag();
+
+#ifndef NDEBUG
+#define POLLY_DEBUG(X)                                                         \
+  do {                                                                         \
+    if (polly::getPollyDebugFlag()) {                                          \
+      X;                                                                       \
+    } else {                                                                   \
+      DEBUG_WITH_TYPE(DEBUG_TYPE, X);                                          \
+    }                                                                          \
+  } while (0)
+#else
+#define POLLY_DEBUG(X)                                                         \
+  do {                                                                         \
+  } while (false)
+#endif
+
+} // namespace polly
+#endif
diff --git a/polly/lib/Analysis/DependenceInfo.cpp b/polly/lib/Analysis/DependenceInfo.cpp
index 69257c603877..9ee004fbac9e 100644
--- a/polly/lib/Analysis/DependenceInfo.cpp
+++ b/polly/lib/Analysis/DependenceInfo.cpp
@@ -39,6 +39,7 @@
 using namespace polly;
 using namespace llvm;
 
+#include "polly/Support/PollyDebug.h"
 #define DEBUG_TYPE "polly-dependence"
 
 static cl::opt<int> OptComputeOut(
@@ -300,10 +301,10 @@ static __isl_give isl_union_flow *buildFlow(__isl_keep isl_union_map *Snk,
     AI = isl_union_access_info_set_kill(AI, isl_union_map_copy(Kill));
   AI = isl_union_access_info_set_schedule(AI, isl_schedule_copy(Schedule));
   auto Flow = isl_union_access_info_compute_flow(AI);
-  LLVM_DEBUG(if (!Flow) dbgs()
-                 << "last error: "
-                 << isl_ctx_last_error(isl_schedule_get_ctx(Schedule))
-                 << '\n';);
+  POLLY_DEBUG(if (!Flow) dbgs()
+                  << "last error: "
+                  << isl_ctx_last_error(isl_schedule_get_ctx(Schedule))
+                  << '\n';);
   return Flow;
 }
 
@@ -312,18 +313,18 @@ void Dependences::calculateDependences(Scop &S) {
   isl_schedule *Schedule;
   isl_union_set *TaggedStmtDomain;
 
-  LLVM_DEBUG(dbgs() << "Scop: \n" << S << "\n");
+  POLLY_DEBUG(dbgs() << "Scop: \n" << S << "\n");
 
   collectInfo(S, Read, MustWrite, MayWrite, ReductionTagMap, TaggedStmtDomain,
               Level);
 
   bool HasReductions = !isl_union_map_is_empty(ReductionTagMap);
 
-  LLVM_DEBUG(dbgs() << "Read: " << Read << '\n';
-             dbgs() << "MustWrite: " << MustWrite << '\n';
-             dbgs() << "MayWrite: " << MayWrite << '\n';
-             dbgs() << "ReductionTagMap: " << ReductionTagMap << '\n';
-             dbgs() << "TaggedStmtDomain: " << TaggedStmtDomain << '\n';);
+  POLLY_DEBUG(dbgs() << "Read: " << Read << '\n';
+              dbgs() << "MustWrite: " << MustWrite << '\n';
+              dbgs() << "MayWrite: " << MayWrite << '\n';
+              dbgs() << "ReductionTagMap: " << ReductionTagMap << '\n';
+              dbgs() << "TaggedStmtDomain: " << TaggedStmtDomain << '\n';);
 
   Schedule = S.getScheduleTree().release();
 
@@ -360,10 +361,10 @@ void Dependences::calculateDependences(Scop &S) {
     Schedule = isl_schedule_pullback_union_pw_multi_aff(Schedule, Tags);
   }
 
-  LLVM_DEBUG(dbgs() << "Read: " << Read << "\n";
-             dbgs() << "MustWrite: " << MustWrite << "\n";
-             dbgs() << "MayWrite: " << MayWrite << "\n";
-             dbgs() << "Schedule: " << Schedule << "\n");
+  POLLY_DEBUG(dbgs() << "Read: " << Read << "\n";
+              dbgs() << "MustWrite: " << MustWrite << "\n";
+              dbgs() << "MayWrite: " << MayWrite << "\n";
+              dbgs() << "Schedule: " << Schedule << "\n");
 
   isl_union_map *StrictWAW = nullptr;
   {
@@ -504,7 +505,7 @@ void Dependences::calculateDependences(Scop &S) {
       isl_union_map_copy(WAW), isl_union_set_copy(TaggedStmtDomain));
   STMT_WAR =
       isl_union_map_intersect_domain(isl_union_map_copy(WAR), TaggedStmtDomain);
-  LLVM_DEBUG({
+  POLLY_DEBUG({
     dbgs() << "Wrapped Dependences:\n";
     dump();
     dbgs() << "\n";
@@ -553,7 +554,7 @@ void Dependences::calculateDependences(Scop &S) {
   } else
     TC_RED = isl_union_map_empty(isl_union_map_get_space(RED));
 
-  LLVM_DEBUG({
+  POLLY_DEBUG({
     dbgs() << "Final Wrapped Dependences:\n";
     dump();
     dbgs() << "\n";
@@ -603,7 +604,7 @@ void Dependences::calculateDependences(Scop &S) {
   RED = isl_union_map_zip(RED);
   TC_RED = isl_union_map_zip(TC_RED);
 
-  LLVM_DEBUG({
+  POLLY_DEBUG({
     dbgs() << "Zipped Dependences:\n";
     dump();
     dbgs() << "\n";
@@ -615,7 +616,7 @@ void Dependences::calculateDependences(Scop &S) {
   RED = isl_union_set_unwrap(isl_union_map_domain(RED));
   TC_RED = isl_union_set_unwrap(isl_union_map_domain(TC_RED));
 
-  LLVM_DEBUG({
+  POLLY_DEBUG({
     dbgs() << "Unwrapped Dependences:\n";
     dump();
     dbgs() << "\n";
@@ -631,7 +632,7 @@ void Dependences::calculateDependences(Scop &S) {
   RED = isl_union_map_coalesce(RED);
   TC_RED = isl_union_map_coalesce(TC_RED);
 
-  LLVM_DEBUG(dump());
+  POLLY_DEBUG(dump());
 }
 
 bool Dependences::isValidSchedule(Scop &S, isl::schedule NewSched) const {
diff --git a/polly/lib/Analysis/PolyhedralInfo.cpp b/polly/lib/Analysis/PolyhedralInfo.cpp
index 5c77be0a9a1f..8d8e81a9049d 100644
--- a/polly/lib/Analysis/PolyhedralInfo.cpp
+++ b/polly/lib/Analysis/PolyhedralInfo.cpp
@@ -32,6 +32,7 @@
 using namespace llvm;
 using namespace polly;
 
+#include "polly/Support/PollyDebug.h"
 #define DEBUG_TYPE "polyhedral-info"
 
 static cl::opt<bool> CheckParallel("polly-check-parallel",
@@ -77,19 +78,19 @@ bool PolyhedralInfo::checkParallel(Loop *L, isl_pw_aff **MinDepDistPtr) const {
       DI->getDependences(const_cast<Scop *>(S), Dependences::AL_Access);
   if (!D.hasValidDependences())
     return false;
-  LLVM_DEBUG(dbgs() << "Loop :\t" << L->getHeader()->getName() << ":\n");
+  POLLY_DEBUG(dbgs() << "Loop :\t" << L->getHeader()->getName() << ":\n");
 
   isl_union_map *Deps =
       D.getDependences(Dependences::TYPE_RAW | Dependences::TYPE_WAW |
                        Dependences::TYPE_WAR | Dependences::TYPE_RED)
           .release();
 
-  LLVM_DEBUG(dbgs() << "Dependences :\t" << stringFromIslObj(Deps, "null")
-                    << "\n");
+  POLLY_DEBUG(dbgs() << "Dependences :\t" << stringFromIslObj(Deps, "null")
+                     << "\n");
 
   isl_union_map *Schedule = getScheduleForLoop(S, L);
-  LLVM_DEBUG(dbgs() << "Schedule: \t" << stringFromIslObj(Schedule, "null")
-                    << "\n");
+  POLLY_DEBUG(dbgs() << "Schedule: \t" << stringFromIslObj(Schedule, "null")
+                     << "\n");
 
   IsParallel = D.isParallel(Schedule, Deps, MinDepDistPtr);
   isl_union_map_free(Schedule);
@@ -125,14 +126,14 @@ __isl_give isl_union_map *PolyhedralInfo::getScheduleForLoop(const Scop *S,
                                                              Loop *L) const {
   isl_union_map *Schedule = isl_union_map_empty(S->getParamSpace().release());
   int CurrDim = S->getRelativeLoopDepth(L);
-  LLVM_DEBUG(dbgs() << "Relative loop depth:\t" << CurrDim << "\n");
+  POLLY_DEBUG(dbgs() << "Relative loop depth:\t" << CurrDim << "\n");
   assert(CurrDim >= 0 && "Loop in region should have at least depth one");
 
   for (auto &SS : *S) {
     if (L->contains(SS.getSurroundingLoop())) {
 
       unsigned int MaxDim = SS.getNumIterators();
-      LLVM_DEBUG(dbgs() << "Maximum depth of Stmt:\t" << MaxDim << "\n");
+      POLLY_DEBUG(dbgs() << "Maximum depth of Stmt:\t" << MaxDim << "\n");
       isl_map *ScheduleMap = SS.getSchedule().release();
       assert(
           ScheduleMap &&
diff --git a/polly/lib/Analysis/PruneUnprofitable.cpp b/polly/lib/Analysis/PruneUnprofitable.cpp
index db4a3d73dc33..f8469c03fe55 100644
--- a/polly/lib/Analysis/PruneUnprofitable.cpp
+++ b/polly/lib/Analysis/PruneUnprofitable.cpp
@@ -22,6 +22,7 @@
 using namespace llvm;
 using namespace polly;
 
+#include "polly/Support/PollyDebug.h"
 #define DEBUG_TYPE "polly-prune-unprofitable"
 
 namespace {
@@ -57,7 +58,7 @@ static void updateStatistics(Scop &S, bool Pruned) {
 
 static bool runPruneUnprofitable(Scop &S) {
   if (PollyProcessUnprofitable) {
-    LLVM_DEBUG(
+    POLLY_DEBUG(
         dbgs() << "NOTE: -polly-process-unprofitable active, won't prune "
                   "anything\n");
     return false;
@@ -66,7 +67,7 @@ static bool runPruneUnprofitable(Scop &S) {
   ScopsProcessed++;
 
   if (!S.isProfitable(true)) {
-    LLVM_DEBUG(
+    POLLY_DEBUG(
         dbgs() << "SCoP pruned because it probably cannot be optimized in "
                   "a significant way\n");
     S.invalidate(PROFITABLE, DebugLoc());
diff --git a/polly/lib/Analysis/ScopBuilder.cpp b/polly/lib/Analysis/ScopBuilder.cpp
index 0edc41d10641..214e4d360d6b 100644
--- a/polly/lib/Analysis/ScopBuilder.cpp
+++ b/polly/lib/Analysis/ScopBuilder.cpp
@@ -60,6 +60,7 @@
 using namespace llvm;
 using namespace polly;
 
+#include "polly/Support/PollyDebug.h"
 #define DEBUG_TYPE "polly-scops"
 
 STATISTIC(ScopFound, "Number of valid Scops");
@@ -2544,9 +2545,9 @@ bool checkCandidatePairAccesses(MemoryAccess *LoadMA, MemoryAccess *StoreMA,
   isl::map LoadAccs = LoadMA->getAccessRelation();
   isl::map StoreAccs = StoreMA->getAccessRelation();
   bool Valid = LoadAccs.has_equal_space(StoreAccs);
-  LLVM_DEBUG(dbgs() << " == The accessed space below is "
-                    << (Valid ? "" : "not ") << "equal!\n");
-  LLVM_DEBUG(LoadMA->dump(); StoreMA->dump());
+  POLLY_DEBUG(dbgs() << " == The accessed space below is "
+                     << (Valid ? "" : "not ") << "equal!\n");
+  POLLY_DEBUG(LoadMA->dump(); StoreMA->dump());
 
   if (Valid) {
     // Then check if they actually access the same memory.
@@ -2560,8 +2561,8 @@ bool checkCandidatePairAccesses(MemoryAccess *LoadMA, MemoryAccess *StoreMA,
     isl::set InterAccs =
         isl::manage(RS.copy()).intersect(isl::manage(WS.copy()));
     Valid = !InterAccs.is_empty();
-    LLVM_DEBUG(dbgs() << " == The accessed memory is " << (Valid ? "" : "not ")
-                      << "overlapping!\n");
+    POLLY_DEBUG(dbgs() << " == The accessed memory is " << (Valid ? "" : "not ")
+                       << "overlapping!\n");
   }
 
   if (Valid) {
@@ -2571,8 +2572,8 @@ bool checkCandidatePairAccesses(MemoryAccess *LoadMA, MemoryAccess *StoreMA,
     isl::set AllAccs = AllAccsRel.range();
     Valid = !hasIntersectingAccesses(AllAccs, LoadMA, StoreMA, Domain, MemAccs);
 
-    LLVM_DEBUG(dbgs() << " == The accessed memory is " << (Valid ? "not " : "")
-                      << "accessed by other instructions!\n");
+    POLLY_DEBUG(dbgs() << " == The accessed memory is " << (Valid ? "not " : "")
+                       << "accessed by other instructions!\n");
   }
   return Valid;
 }
@@ -3240,8 +3241,8 @@ bool ScopBuilder::buildAliasChecks() {
   // we make the assumed context infeasible.
   scop->invalidate(ALIASING, DebugLoc());
 
-  LLVM_DEBUG(dbgs() << "\n\nNOTE: Run time checks for " << scop->getNameStr()
-                    << " could not be created. This SCoP has been dismissed.");
+  POLLY_DEBUG(dbgs() << "\n\nNOTE: Run time checks for " << scop->getNameStr()
+                     << " could not be created. This SCoP has been dismissed.");
   return false;
 }
 
@@ -3562,7 +3563,7 @@ void ScopBuilder::buildScop(Region &R, AssumptionCache &AC) {
   DenseMap<BasicBlock *, isl::set> InvalidDomainMap;
 
   if (!buildDomains(&R, InvalidDomainMap)) {
-    LLVM_DEBUG(
+    POLLY_DEBUG(
         dbgs() << "Bailing-out because buildDomains encountered problems\n");
     return;
   }
@@ -3582,7 +3583,7 @@ void ScopBuilder::buildScop(Region &R, AssumptionCache &AC) {
   scop->removeStmtNotInDomainMap();
   scop->simplifySCoP(false);
   if (scop->isEmpty()) {
-    LLVM_DEBUG(dbgs() << "Bailing-out because SCoP is empty\n");
+    POLLY_DEBUG(dbgs() << "Bailing-out because SCoP is empty\n");
     return;
   }
 
@@ -3599,7 +3600,8 @@ void ScopBuilder::buildScop(Region &R, AssumptionCache &AC) {
 
   // Check early for a feasible runtime context.
   if (!scop->hasFeasibleRuntimeContext()) {
-    LLVM_DEBUG(dbgs() << "Bailing-out because of unfeasible context (early)\n");
+    POLLY_DEBUG(
+        dbgs() << "Bailing-out because of unfeasible context (early)\n");
     return;
   }
 
@@ -3607,7 +3609,7 @@ void ScopBuilder::buildScop(Region &R, AssumptionCache &AC) {
   // only the runtime context could become infeasible.
   if (!scop->isProfitable(UnprofitableScalarAccs)) {
     scop->invalidate(PROFITABLE, DebugLoc());
-    LLVM_DEBUG(
+    POLLY_DEBUG(
         dbgs() << "Bailing-out because SCoP is not considered profitable\n");
     return;
   }
@@ -3626,7 +3628,7 @@ void ScopBuilder::buildScop(Region &R, AssumptionCache &AC) {
 
   scop->simplifyContexts();
   if (!buildAliasChecks()) {
-    LLVM_DEBUG(dbgs() << "Bailing-out because could not build alias checks\n");
+    POLLY_DEBUG(dbgs() << "Bailing-out because could not build alias checks\n");
     return;
   }
 
@@ -3638,7 +3640,7 @@ void ScopBuilder::buildScop(Region &R, AssumptionCache &AC) {
   // Check late for a feasible runtime context because profitability did not
   // change.
   if (!scop->hasFeasibleRuntimeContext()) {
-    LLVM_DEBUG(dbgs() << "Bailing-out because of unfeasible context (late)\n");
+    POLLY_DEBUG(dbgs() << "Bailing-out because of unfeasible context (late)\n");
     return;
   }
 
@@ -3662,12 +3664,12 @@ ScopBuilder::ScopBuilder(Region *R, AssumptionCache &AC, AAResults &AA,
 
   buildScop(*R, AC);
 
-  LLVM_DEBUG(dbgs() << *scop);
+  POLLY_DEBUG(dbgs() << *scop);
 
   if (!scop->hasFeasibleRuntimeContext()) {
     InfeasibleScops++;
     Msg = "SCoP ends here but was dismissed.";
-    LLVM_DEBUG(dbgs() << "SCoP detected but dismissed\n");
+    POLLY_DEBUG(dbgs() << "SCoP detected but dismissed\n");
     RecordedAssumptions.clear();
     scop.reset();
   } else {
diff --git a/polly/lib/Analysis/ScopDetection.cpp b/polly/lib/Analysis/ScopDetection.cpp
index 938d3f149677..eab7bd83e6a4 100644
--- a/polly/lib/Analysis/ScopDetection.cpp
+++ b/polly/lib/Analysis/ScopDetection.cpp
@@ -91,6 +91,7 @@
 using namespace llvm;
 using namespace polly;
 
+#include "polly/Support/PollyDebug.h"
 #define DEBUG_TYPE "polly-detect"
 
 // This option is set to a very high value, as analyzing such loops increases
@@ -406,8 +407,8 @@ inline bool ScopDetection::invalid(DetectionContext &Context, bool Assert,
     // canUseISLTripCount().
     Log.report(RejectReason);
 
-    LLVM_DEBUG(dbgs() << RejectReason->getMessage());
-    LLVM_DEBUG(dbgs() << "\n");
+    POLLY_DEBUG(dbgs() << RejectReason->getMessage());
+    POLLY_DEBUG(dbgs() << "\n");
   } else {
     assert(!Assert && "Verification of detected scop failed");
   }
@@ -704,8 +705,8 @@ bool ScopDetection::isValidCallInst(CallInst &CI,
     return false;
 
   if (isDebugCall(&CI)) {
-    LLVM_DEBUG(dbgs() << "Allow call to debug function: "
-                      << CalledFunction->getName() << '\n');
+    POLLY_DEBUG(dbgs() << "Allow call to debug function: "
+                       << CalledFunction->getName() << '\n');
     return true;
   }
 
@@ -1486,7 +1487,7 @@ Region *ScopDetection::expandRegion(Region &R) {
   std::unique_ptr<Region> LastValidRegion;
   auto ExpandedRegion = std::unique_ptr<Region>(R.getExpandedRegion());
 
-  LLVM_DEBUG(dbgs() << "\tExpanding " << R.getNameStr() << "\n");
+  POLLY_DEBUG(dbgs() << "\tExpanding " << R.getNameStr() << "\n");
 
   while (ExpandedRegion) {
     BBPair P = getBBPairForRegion(ExpandedRegion.get());
@@ -1495,7 +1496,8 @@ Region *ScopDetection::expandRegion(Region &R) {
                                                /*Verifying=*/false);
     DetectionContext &Context = *Entry.get();
 
-    LLVM_DEBUG(dbgs() << "\t\tTrying " << ExpandedRegion->getNameStr() << "\n");
+    POLLY_DEBUG(dbgs() << "\t\tTrying " << ExpandedRegion->getNameStr()
+                       << "\n");
     // Only expand when we did not collect errors.
 
     if (!Context.Log.hasErrors()) {
@@ -1529,7 +1531,7 @@ Region *ScopDetection::expandRegion(Region &R) {
     }
   }
 
-  LLVM_DEBUG({
+  POLLY_DEBUG({
     if (LastValidRegion)
       dbgs() << "\tto " << LastValidRegion->getNameStr() << "\n";
     else
@@ -1750,10 +1752,11 @@ bool ScopDetection::isProfitableRegion(DetectionContext &Context) const {
 bool ScopDetection::isValidRegion(DetectionContext &Context) {
   Region &CurRegion = Context.CurRegion;
 
-  LLVM_DEBUG(dbgs() << "Checking region: " << CurRegion.getNameStr() << "\n\t");
+  POLLY_DEBUG(dbgs() << "Checking region: " << CurRegion.getNameStr()
+                     << "\n\t");
 
   if (!PollyAllowFullFunction && CurRegion.isTopLevelRegion()) {
-    LLVM_DEBUG(dbgs() << "Top level region is invalid\n");
+    POLLY_DEBUG(dbgs() << "Top level region is invalid\n");
     Context.IsInvalid = true;
     return false;
   }
@@ -1761,14 +1764,14 @@ bool ScopDetection::isValidRegion(DetectionContext &Context) {
   DebugLoc DbgLoc;
   if (CurRegion.getExit() &&
       isa<UnreachableInst>(CurRegion.getExit()->getTerminator())) {
-    LLVM_DEBUG(dbgs() << "Unreachable in exit\n");
+    POLLY_DEBUG(dbgs() << "Unreachable in exit\n");
     return invalid<ReportUnreachableInExit>(Context, /*Assert=*/true,
                                             CurRegion.getExit(), DbgLoc);
   }
 
   if (!OnlyRegion.empty() &&
       !CurRegion.getEntry()->getName().count(OnlyRegion)) {
-    LLVM_DEBUG({
+    POLLY_DEBUG({
       dbgs() << "Region entry does not match -polly-only-region";
       dbgs() << "\n";
     });
@@ -1802,7 +1805,7 @@ bool ScopDetection::isValidRegion(DetectionContext &Context) {
     return invalid<ReportIrreducibleRegion>(Context, /*Assert=*/true,
                                             &CurRegion, DbgLoc);
 
-  LLVM_DEBUG(dbgs() << "OK\n");
+  POLLY_DEBUG(dbgs() << "OK\n");
   return true;
 }
 
diff --git a/polly/lib/Analysis/ScopInfo.cpp b/polly/lib/Analysis/ScopInfo.cpp
index 3e78cc8937fb..fa35fae84ace 100644
--- a/polly/lib/Analysis/ScopInfo.cpp
+++ b/polly/lib/Analysis/ScopInfo.cpp
@@ -73,6 +73,7 @@
 using namespace llvm;
 using namespace polly;
 
+#include "polly/Support/PollyDebug.h"
 #define DEBUG_TYPE "polly-scops"
 
 STATISTIC(AssumptionsAliasing, "Number of aliasing assumptions taken.");
@@ -2042,7 +2043,7 @@ void Scop::intersectDefinedBehavior(isl::set Set, AssumptionSign Sign) {
 }
 
 void Scop::invalidate(AssumptionKind Kind, DebugLoc Loc, BasicBlock *BB) {
-  LLVM_DEBUG(dbgs() << "Invalidate SCoP because of reason " << Kind << "\n");
+  POLLY_DEBUG(dbgs() << "Invalidate SCoP because of reason " << Kind << "\n");
   addAssumption(Kind, isl::set::empty(getParamSpace()), Loc, AS_ASSUMPTION, BB);
 }
 
diff --git a/polly/lib/CMakeLists.txt b/polly/lib/CMakeLists.txt
index 9780f24d5d36..4557878e515e 100644
--- a/polly/lib/CMakeLists.txt
+++ b/polly/lib/CMakeLists.txt
@@ -62,6 +62,7 @@ add_llvm_pass_plugin(Polly
   CodeGen/PerfMonitor.cpp
   Exchange/JSONExporter.cpp
   Support/GICHelper.cpp
+  Support/PollyDebug.cpp
   Support/SCEVAffinator.cpp
   Support/SCEVValidator.cpp
   Support/RegisterPasses.cpp
diff --git a/polly/lib/CodeGen/CodeGeneration.cpp b/polly/lib/CodeGen/CodeGeneration.cpp
index 3792d995fd99..c4ca3f2e0f0f 100644
--- a/polly/lib/CodeGen/CodeGeneration.cpp
+++ b/polly/lib/CodeGen/CodeGeneration.cpp
@@ -47,6 +47,7 @@
 using namespace llvm;
 using namespace polly;
 
+#include "polly/Support/PollyDebug.h"
 #define DEBUG_TYPE "polly-codegen"
 
 static cl::opt<bool> Verify("polly-codegen-verify",
@@ -86,7 +87,7 @@ static void verifyGeneratedFunction(Scop &S, Function &F, IslAstInfo &AI) {
   if (!Verify || !verifyFunction(F, &errs()))
     return;
 
-  LLVM_DEBUG({
+  POLLY_DEBUG({
     errs() << "== ISL Codegen created an invalid function ==\n\n== The "
               "SCoP ==\n";
     errs() << S;
@@ -183,7 +184,7 @@ static bool generateCode(Scop &S, IslAstInfo &AI, LoopInfo &LI,
   // DependenceInfo or IslAstInfo around.
   IslAst &Ast = AI.getIslAst();
   if (Ast.getSharedIslCtx() != S.getSharedIslCtx()) {
-    LLVM_DEBUG(dbgs() << "Got an IstAst for a different Scop/isl_ctx\n");
+    POLLY_DEBUG(dbgs() << "Got an IstAst for a different Scop/isl_ctx\n");
     return false;
   }
 
diff --git a/polly/lib/CodeGen/IslAst.cpp b/polly/lib/CodeGen/IslAst.cpp
index fe2b5d9104d7..142a19d5e069 100644
--- a/polly/lib/CodeGen/IslAst.cpp
+++ b/polly/lib/CodeGen/IslAst.cpp
@@ -52,6 +52,7 @@
 #include <cassert>
 #include <cstdlib>
 
+#include "polly/Support/PollyDebug.h"
 #define DEBUG_TYPE "polly-ast"
 
 using namespace llvm;
@@ -643,14 +644,14 @@ static std::unique_ptr<IslAstInfo> runIslAst(
   const Dependences &D = GetDeps(Dependences::AL_Statement);
 
   if (D.getSharedIslCtx() != Scop.getSharedIslCtx()) {
-    LLVM_DEBUG(
+    POLLY_DEBUG(
         dbgs() << "Got dependence analysis for different SCoP/isl_ctx\n");
     return {};
   }
 
   std::unique_ptr<IslAstInfo> Ast = std::make_unique<IslAstInfo>(Scop, D);
 
-  LLVM_DEBUG({
+  POLLY_DEBUG({
     if (Ast)
       Ast->print(dbgs());
   });
@@ -751,7 +752,7 @@ void IslAstInfo::print(raw_ostream &OS) {
   P = isl_ast_node_print(RootNode.get(), P, Options);
   AstStr = isl_printer_get_str(P);
 
-  LLVM_DEBUG({
+  POLLY_DEBUG({
     dbgs() << S.getContextStr() << "\n";
     dbgs() << stringFromIslObj(S.getScheduleTree(), "null");
   });
diff --git a/polly/lib/Support/PollyDebug.cpp b/polly/lib/Support/PollyDebug.cpp
new file mode 100644
index 000000000000..9dcd8ed03694
--- /dev/null
+++ b/polly/lib/Support/PollyDebug.cpp
@@ -0,0 +1,27 @@
+//===-PollyDebug.cpp -Provide support for debugging Polly passes-*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Functions to aid printing Debug Info of all polly passes.
+//
+//===----------------------------------------------------------------------===//
+
+#include "polly/Support/PollyDebug.h"
+#include "llvm/Support/CommandLine.h"
+
+using namespace polly;
+using namespace llvm;
+
+bool PollyDebugFlag;
+bool polly::getPollyDebugFlag() { return PollyDebugFlag; }
+
+// -debug - Command line option to enable the DEBUG statements in the passes.
+// This flag may only be enabled in debug builds.
+static cl::opt<bool, true>
+    PollyDebug("polly-debug",
+               cl::desc("Enable debug output for only polly passes."),
+               cl::Hidden, cl::location(PollyDebugFlag), cl::ZeroOrMore);
diff --git a/polly/lib/Support/SCEVValidator.cpp b/polly/lib/Support/SCEVValidator.cpp
index e3d9818597ea..5bb82624ed78 100644
--- a/polly/lib/Support/SCEVValidator.cpp
+++ b/polly/lib/Support/SCEVValidator.cpp
@@ -9,6 +9,7 @@
 using namespace llvm;
 using namespace polly;
 
+#include "polly/Support/PollyDebug.h"
 #define DEBUG_TYPE "polly-scev-validator"
 
 namespace SCEVType {
@@ -136,7 +137,7 @@ public:
 
   ValidatorResult visitVScale(const SCEVVScale *VScale) {
     // We do not support VScale constants.
-    LLVM_DEBUG(dbgs() << "INVALID: VScale is not supported");
+    POLLY_DEBUG(dbgs() << "INVALID: VScale is not supported");
     return ValidatorResult(SCEVType::INVALID);
   }
 
@@ -203,7 +204,7 @@ public:
       }
 
       if ((Op.isIV() || Op.isPARAM()) && !Return.isINT()) {
-        LLVM_DEBUG(
+        POLLY_DEBUG(
             dbgs() << "INVALID: More than one non-int operand in MulExpr\n"
                    << "\tExpr: " << *Expr << "\n"
                    << "\tPrevious expression type: " << Return << "\n"
@@ -224,7 +225,7 @@ public:
 
   ValidatorResult visitAddRecExpr(const SCEVAddRecExpr *Expr) {
     if (!Expr->isAffine()) {
-      LLVM_DEBUG(dbgs() << "INVALID: AddRec is not affine");
+      POLLY_DEBUG(dbgs() << "INVALID: AddRec is not affine");
       return ValidatorResult(SCEVType::INVALID);
     }
 
@@ -239,7 +240,7 @@ public:
 
     auto *L = Expr->getLoop();
     if (R->contains(L) && (!Scope || !L->contains(Scope))) {
-      LLVM_DEBUG(
+      POLLY_DEBUG(
           dbgs() << "INVALID: Loop of AddRec expression boxed in an a "
                     "non-affine subregion or has a non-synthesizable exit "
                     "value.");
@@ -253,8 +254,8 @@ public:
         return Result;
       }
 
-      LLVM_DEBUG(dbgs() << "INVALID: AddRec within scop has non-int"
-                           "recurrence part");
+      POLLY_DEBUG(dbgs() << "INVALID: AddRec within scop has non-int"
+                            "recurrence part");
       return ValidatorResult(SCEVType::INVALID);
     }
 
@@ -314,7 +315,7 @@ public:
       ValidatorResult Op = visit(Expr->getOperand(i));
 
       if (!Op.isConstant()) {
-        LLVM_DEBUG(dbgs() << "INVALID: UMaxExpr has a non-constant operand");
+        POLLY_DEBUG(dbgs() << "INVALID: UMaxExpr has a non-constant operand");
         return ValidatorResult(SCEVType::INVALID);
       }
     }
@@ -329,7 +330,7 @@ public:
       ValidatorResult Op = visit(Expr->getOperand(i));
 
       if (!Op.isConstant()) {
-        LLVM_DEBUG(dbgs() << "INVALID: UMinExpr has a non-constant operand");
+        POLLY_DEBUG(dbgs() << "INVALID: UMinExpr has a non-constant operand");
         return ValidatorResult(SCEVType::INVALID);
       }
     }
@@ -344,7 +345,7 @@ public:
       ValidatorResult Op = visit(Expr->getOperand(i));
 
       if (!Op.isConstant()) {
-        LLVM_DEBUG(
+        POLLY_DEBUG(
             dbgs()
             << "INVALID: SCEVSequentialUMinExpr has a non-constant operand");
         return ValidatorResult(SCEVType::INVALID);
@@ -356,8 +357,8 @@ public:
 
   ValidatorResult visitGenericInst(Instruction *I, const SCEV *S) {
     if (R->contains(I)) {
-      LLVM_DEBUG(dbgs() << "INVALID: UnknownExpr references an instruction "
-                           "within the region\n");
+      POLLY_DEBUG(dbgs() << "INVALID: UnknownExpr references an instruction "
+                            "within the region\n");
       return ValidatorResult(SCEVType::INVALID);
     }
 
@@ -393,7 +394,7 @@ public:
     if (LHS.isConstant() && RHS.isConstant())
       return ValidatorResult(SCEVType::PARAM, DivExpr);
 
-    LLVM_DEBUG(
+    POLLY_DEBUG(
         dbgs() << "INVALID: unsigned division of non-constant expressions");
     return ValidatorResult(SCEVType::INVALID);
   }
@@ -434,12 +435,13 @@ public:
     Value *V = Expr->getValue();
 
     if (!Expr->getType()->isIntegerTy() && !Expr->getType()->isPointerTy()) {
-      LLVM_DEBUG(dbgs() << "INVALID: UnknownExpr is not an integer or pointer");
+      POLLY_DEBUG(
+          dbgs() << "INVALID: UnknownExpr is not an integer or pointer");
       return ValidatorResult(SCEVType::INVALID);
     }
 
     if (isa<UndefValue>(V)) {
-      LLVM_DEBUG(dbgs() << "INVALID: UnknownExpr references an undef value");
+      POLLY_DEBUG(dbgs() << "INVALID: UnknownExpr references an undef value");
       return ValidatorResult(SCEVType::INVALID);
     }
 
@@ -601,7 +603,7 @@ bool polly::isAffineExpr(const Region *R, llvm::Loop *Scope, const SCEV *Expr,
     return false;
 
   SCEVValidator Validator(R, Scope, SE, ILS);
-  LLVM_DEBUG({
+  POLLY_DEBUG({
     dbgs() << "\n";
     dbgs() << "Expr: " << *Expr << "\n";
     dbgs() << "Region: " << R->getNameStr() << "\n";
@@ -610,7 +612,7 @@ bool polly::isAffineExpr(const Region *R, llvm::Loop *Scope, const SCEV *Expr,
 
   ValidatorResult Result = Validator.visit(Expr);
 
-  LLVM_DEBUG({
+  POLLY_DEBUG({
     if (Result.isValid())
       dbgs() << "VALID\n";
     dbgs() << "\n";
diff --git a/polly/lib/Transform/DeLICM.cpp b/polly/lib/Transform/DeLICM.cpp
index dae5e79639f7..8afa63724ade 100644
--- a/polly/lib/Transform/DeLICM.cpp
+++ b/polly/lib/Transform/DeLICM.cpp
@@ -26,6 +26,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/InitializePasses.h"
 
+#include "polly/Support/PollyDebug.h"
 #define DEBUG_TYPE "polly-delicm"
 
 using namespace polly;
@@ -547,7 +548,7 @@ private:
   /// @see Knowledge::isConflicting
   bool isConflicting(const Knowledge &Proposed) {
     raw_ostream *OS = nullptr;
-    LLVM_DEBUG(OS = &llvm::dbgs());
+    POLLY_DEBUG(OS = &llvm::dbgs());
     return Knowledge::isConflicting(Zone, Proposed, OS, 4);
   }
 
@@ -559,7 +560,7 @@ private:
     if (SAI->isValueKind()) {
       auto *MA = S->getValueDef(SAI);
       if (!MA) {
-        LLVM_DEBUG(
+        POLLY_DEBUG(
             dbgs()
             << "    Reject because value is read-only within the scop\n");
         return false;
@@ -576,7 +577,7 @@ private:
         auto UserInst = cast<Instruction>(User);
 
         if (!S->contains(UserInst)) {
-          LLVM_DEBUG(dbgs() << "    Reject because value is escaping\n");
+          POLLY_DEBUG(dbgs() << "    Reject because value is escaping\n");
           return false;
         }
       }
@@ -593,9 +594,9 @@ private:
       auto PHI = cast<PHINode>(MA->getAccessInstruction());
       for (auto Incoming : PHI->blocks()) {
         if (!S->contains(Incoming)) {
-          LLVM_DEBUG(dbgs()
-                     << "    Reject because at least one incoming block is "
-                        "not in the scop region\n");
+          POLLY_DEBUG(dbgs()
+                      << "    Reject because at least one incoming block is "
+                         "not in the scop region\n");
           return false;
         }
       }
@@ -603,7 +604,7 @@ private:
       return true;
     }
 
-    LLVM_DEBUG(dbgs() << "    Reject ExitPHI or other non-value\n");
+    POLLY_DEBUG(dbgs() << "    Reject ExitPHI or other non-value\n");
     return false;
   }
 
@@ -686,12 +687,12 @@ private:
     // { DomainDef[] -> Element[] }
     auto DefTarget = TargetElt.apply_domain(DefSched.reverse());
     simplify(DefTarget);
-    LLVM_DEBUG(dbgs() << "    Def Mapping: " << DefTarget << '\n');
+    POLLY_DEBUG(dbgs() << "    Def Mapping: " << DefTarget << '\n');
 
     auto OrigDomain = getDomainFor(DefMA);
     auto MappedDomain = DefTarget.domain();
     if (!OrigDomain.is_subset(MappedDomain)) {
-      LLVM_DEBUG(
+      POLLY_DEBUG(
           dbgs()
           << "    Reject because mapping does not encompass all instances\n");
       return false;
@@ -704,7 +705,7 @@ private:
     isl::union_map DefUses;
 
     std::tie(DefUses, Lifetime) = computeValueUses(SAI);
-    LLVM_DEBUG(dbgs() << "    Lifetime: " << Lifetime << '\n');
+    POLLY_DEBUG(dbgs() << "    Lifetime: " << Lifetime << '\n');
 
     /// { [Element[] -> Zone[]] }
     auto EltZone = Lifetime.apply_domain(DefTarget).wrap();
@@ -858,12 +859,12 @@ private:
     // { DomainRead[] -> Element[] }
     auto PHITarget = PHISched.apply_range(TargetElt);
     simplify(PHITarget);
-    LLVM_DEBUG(dbgs() << "    Mapping: " << PHITarget << '\n');
+    POLLY_DEBUG(dbgs() << "    Mapping: " << PHITarget << '\n');
 
     auto OrigDomain = getDomainFor(PHIRead);
     auto MappedDomain = PHITarget.domain();
     if (!OrigDomain.is_subset(MappedDomain)) {
-      LLVM_DEBUG(
+      POLLY_DEBUG(
           dbgs()
           << "    Reject because mapping does not encompass all instances\n");
       return false;
@@ -872,7 +873,7 @@ private:
     // { DomainRead[] -> DomainWrite[] }
     auto PerPHIWrites = computePerPHI(SAI);
     if (PerPHIWrites.is_null()) {
-      LLVM_DEBUG(
+      POLLY_DEBUG(
           dbgs() << "    Reject because cannot determine incoming values\n");
       return false;
     }
@@ -894,17 +895,17 @@ private:
     auto ExpandedWritesDom = WritesTarget.domain();
     if (!DelicmPartialWrites &&
         !UniverseWritesDom.is_subset(ExpandedWritesDom)) {
-      LLVM_DEBUG(
+      POLLY_DEBUG(
           dbgs() << "    Reject because did not find PHI write mapping for "
                     "all instances\n");
       if (DelicmOverapproximateWrites)
-        LLVM_DEBUG(dbgs() << "      Relevant Mapping:    "
-                          << RelevantWritesTarget << '\n');
-      LLVM_DEBUG(dbgs() << "      Deduced Mapping:     " << WritesTarget
-                        << '\n');
-      LLVM_DEBUG(dbgs() << "      Missing instances:    "
-                        << UniverseWritesDom.subtract(ExpandedWritesDom)
-                        << '\n');
+        POLLY_DEBUG(dbgs() << "      Relevant Mapping:    "
+                           << RelevantWritesTarget << '\n');
+      POLLY_DEBUG(dbgs() << "      Deduced Mapping:     " << WritesTarget
+                         << '\n');
+      POLLY_DEBUG(dbgs() << "      Missing instances:    "
+                         << UniverseWritesDom.subtract(ExpandedWritesDom)
+                         << '\n');
       return false;
     }
 
@@ -916,7 +917,7 @@ private:
     // { DomainRead[] -> Zone[] }
     auto Lifetime = betweenScatter(PerPHIWriteScatter, PHISched, false, true);
     simplify(Lifetime);
-    LLVM_DEBUG(dbgs() << "    Lifetime: " << Lifetime << "\n");
+    POLLY_DEBUG(dbgs() << "    Lifetime: " << Lifetime << "\n");
 
     // { DomainWrite[] -> Zone[] }
     auto WriteLifetime = isl::union_map(Lifetime).apply_domain(PerPHIWrites);
@@ -1029,7 +1030,7 @@ private:
     // Use the target store's write location as a suggestion to map scalars to.
     auto EltTarget = Target.apply_range(TargetAccRel);
     simplify(EltTarget);
-    LLVM_DEBUG(dbgs() << "    Target mapping is " << EltTarget << '\n');
+    POLLY_DEBUG(dbgs() << "    Target mapping is " << EltTarget << '\n');
 
     // Stack of elements not yet processed.
     SmallVector<MemoryAccess *, 16> Worklist;
@@ -1067,8 +1068,8 @@ private:
       if (Closed.count(SAI))
         continue;
       Closed.insert(SAI);
-      LLVM_DEBUG(dbgs() << "\n    Trying to map " << MA << " (SAI: " << SAI
-                        << ")\n");
+      POLLY_DEBUG(dbgs() << "\n    Trying to map " << MA << " (SAI: " << SAI
+                         << ")\n");
 
       // Skip non-mappable scalars.
       if (!isMappable(SAI))
@@ -1076,7 +1077,7 @@ private:
 
       auto MASize = DL.getTypeAllocSize(MA->getAccessValue()->getType());
       if (MASize > StoreSize) {
-        LLVM_DEBUG(
+        POLLY_DEBUG(
             dbgs() << "    Reject because storage size is insufficient\n");
         continue;
       }
@@ -1212,7 +1213,7 @@ public:
              "The only reason that these things have not been computed should "
              "be if the max-operations limit hit");
       DeLICMOutOfQuota++;
-      LLVM_DEBUG(dbgs() << "DeLICM analysis exceeded max_operations\n");
+      POLLY_DEBUG(dbgs() << "DeLICM analysis exceeded max_operations\n");
       DebugLoc Begin, End;
       getDebugLocations(getBBPairForRegion(&S->getRegion()), Begin, End);
       OptimizationRemarkAnalysis R(DEBUG_TYPE, "OutOfQuota", Begin,
@@ -1223,7 +1224,7 @@ public:
     }
 
     Zone = OriginalZone = Knowledge({}, EltUnused, EltKnown, EltWritten);
-    LLVM_DEBUG(dbgs() << "Computed Zone:\n"; OriginalZone.print(dbgs(), 4));
+    POLLY_DEBUG(dbgs() << "Computed Zone:\n"; OriginalZone.print(dbgs(), 4));
 
     assert(Zone.isUsable() && OriginalZone.isUsable());
     return true;
@@ -1245,8 +1246,8 @@ public:
           continue;
 
         if (MA->isMayWrite()) {
-          LLVM_DEBUG(dbgs() << "Access " << MA
-                            << " pruned because it is a MAY_WRITE\n");
+          POLLY_DEBUG(dbgs() << "Access " << MA
+                             << " pruned because it is a MAY_WRITE\n");
           OptimizationRemarkMissed R(DEBUG_TYPE, "TargetMayWrite",
                                      MA->getAccessInstruction());
           R << "Skipped possible mapping target because it is not an "
@@ -1256,8 +1257,8 @@ public:
         }
 
         if (Stmt.getNumIterators() == 0) {
-          LLVM_DEBUG(dbgs() << "Access " << MA
-                            << " pruned because it is not in a loop\n");
+          POLLY_DEBUG(dbgs() << "Access " << MA
+                             << " pruned because it is not in a loop\n");
           OptimizationRemarkMissed R(DEBUG_TYPE, "WriteNotInLoop",
                                      MA->getAccessInstruction());
           R << "skipped possible mapping target because it is not in a loop";
@@ -1266,9 +1267,9 @@ public:
         }
 
         if (isScalarAccess(MA)) {
-          LLVM_DEBUG(dbgs()
-                     << "Access " << MA
-                     << " pruned because it writes only a single element\n");
+          POLLY_DEBUG(dbgs()
+                      << "Access " << MA
+                      << " pruned because it writes only a single element\n");
           OptimizationRemarkMissed R(DEBUG_TYPE, "ScalarWrite",
                                      MA->getAccessInstruction());
           R << "skipped possible mapping target because the memory location "
@@ -1278,8 +1279,8 @@ public:
         }
 
         if (!isa<StoreInst>(MA->getAccessInstruction())) {
-          LLVM_DEBUG(dbgs() << "Access " << MA
-                            << " pruned because it is not a StoreInst\n");
+          POLLY_DEBUG(dbgs() << "Access " << MA
+                             << " pruned because it is not a StoreInst\n");
           OptimizationRemarkMissed R(DEBUG_TYPE, "NotAStore",
                                      MA->getAccessInstruction());
           R << "skipped possible mapping target because non-store instructions "
@@ -1301,9 +1302,9 @@ public:
         // arguments.
         isl::union_map AccRel = MA->getLatestAccessRelation();
         if (!AccRel.is_single_valued().is_true()) {
-          LLVM_DEBUG(dbgs() << "Access " << MA
-                            << " is incompatible because it writes multiple "
-                               "elements per instance\n");
+          POLLY_DEBUG(dbgs() << "Access " << MA
+                             << " is incompatible because it writes multiple "
+                                "elements per instance\n");
           OptimizationRemarkMissed R(DEBUG_TYPE, "NonFunctionalAccRel",
                                      MA->getAccessInstruction());
           R << "skipped possible mapping target because it writes more than "
@@ -1314,7 +1315,7 @@ public:
 
         isl::union_set TouchedElts = AccRel.range();
         if (!TouchedElts.is_subset(CompatibleElts)) {
-          LLVM_DEBUG(
+          POLLY_DEBUG(
               dbgs()
               << "Access " << MA
               << " is incompatible because it touches incompatible elements\n");
@@ -1328,7 +1329,7 @@ public:
 
         assert(isCompatibleAccess(MA));
         NumberOfCompatibleTargets++;
-        LLVM_DEBUG(dbgs() << "Analyzing target access " << MA << "\n");
+        POLLY_DEBUG(dbgs() << "Analyzing target access " << MA << "\n");
         if (collapseScalarsToStore(MA))
           Modified = true;
       }
@@ -1361,15 +1362,15 @@ static std::unique_ptr<DeLICMImpl> collapseToUnused(Scop &S, LoopInfo &LI) {
   std::unique_ptr<DeLICMImpl> Impl = std::make_unique<DeLICMImpl>(&S, &LI);
 
   if (!Impl->computeZone()) {
-    LLVM_DEBUG(dbgs() << "Abort because cannot reliably compute lifetimes\n");
+    POLLY_DEBUG(dbgs() << "Abort because cannot reliably compute lifetimes\n");
     return Impl;
   }
 
-  LLVM_DEBUG(dbgs() << "Collapsing scalars to unused array elements...\n");
+  POLLY_DEBUG(dbgs() << "Collapsing scalars to unused array elements...\n");
   Impl->greedyCollapse();
 
-  LLVM_DEBUG(dbgs() << "\nFinal Scop:\n");
-  LLVM_DEBUG(dbgs() << S);
+  POLLY_DEBUG(dbgs() << "\nFinal Scop:\n");
+  POLLY_DEBUG(dbgs() << S);
 
   return Impl;
 }
diff --git a/polly/lib/Transform/FlattenAlgo.cpp b/polly/lib/Transform/FlattenAlgo.cpp
index f8ed332348ab..27a699e5ea59 100644
--- a/polly/lib/Transform/FlattenAlgo.cpp
+++ b/polly/lib/Transform/FlattenAlgo.cpp
@@ -14,6 +14,7 @@
 #include "polly/FlattenAlgo.h"
 #include "polly/Support/ISLOStream.h"
 #include "polly/Support/ISLTools.h"
+#include "polly/Support/PollyDebug.h"
 #include "llvm/Support/Debug.h"
 #define DEBUG_TYPE "polly-flatten-algo"
 
@@ -171,7 +172,7 @@ isl::union_map tryFlattenSequence(isl::union_map Schedule) {
 
   // Would cause an infinite loop.
   if (!isDimBoundedByConstant(ScatterSet, 0)) {
-    LLVM_DEBUG(dbgs() << "Abort; dimension is not of fixed size\n");
+    POLLY_DEBUG(dbgs() << "Abort; dimension is not of fixed size\n");
     return {};
   }
 
@@ -182,8 +183,8 @@ isl::union_map tryFlattenSequence(isl::union_map Schedule) {
   auto Counter = isl::pw_aff(isl::local_space(ParamSpace.set_from_params()));
 
   while (!ScatterSet.is_empty()) {
-    LLVM_DEBUG(dbgs() << "Next counter:\n  " << Counter << "\n");
-    LLVM_DEBUG(dbgs() << "Remaining scatter set:\n  " << ScatterSet << "\n");
+    POLLY_DEBUG(dbgs() << "Next counter:\n  " << Counter << "\n");
+    POLLY_DEBUG(dbgs() << "Remaining scatter set:\n  " << ScatterSet << "\n");
     auto ThisSet = ScatterSet.project_out(isl::dim::set, 1, Dims - 1);
     auto ThisFirst = ThisSet.lexmin();
     auto ScatterFirst = ThisFirst.add_dims(isl::dim::set, Dims - 1);
@@ -199,11 +200,11 @@ isl::union_map tryFlattenSequence(isl::union_map Schedule) {
     auto RemainingSubSchedule = scheduleProjectOut(SubSchedule, 0, 1);
 
     auto FirstSubScatter = isl::set(FirstSubSchedule.range());
-    LLVM_DEBUG(dbgs() << "Next step in sequence is:\n  " << FirstSubScatter
-                      << "\n");
+    POLLY_DEBUG(dbgs() << "Next step in sequence is:\n  " << FirstSubScatter
+                       << "\n");
 
     if (!isDimBoundedByParameter(FirstSubScatter, 0)) {
-      LLVM_DEBUG(dbgs() << "Abort; sequence step is not bounded\n");
+      POLLY_DEBUG(dbgs() << "Abort; sequence step is not bounded\n");
       return {};
     }
 
@@ -236,8 +237,8 @@ isl::union_map tryFlattenSequence(isl::union_map Schedule) {
     Counter = Counter.add(PartLen);
   }
 
-  LLVM_DEBUG(dbgs() << "Sequence-flatten result is:\n  " << NewSchedule
-                    << "\n");
+  POLLY_DEBUG(dbgs() << "Sequence-flatten result is:\n  " << NewSchedule
+                     << "\n");
   return NewSchedule;
 }
 
@@ -266,20 +267,20 @@ isl::union_map tryFlattenLoop(isl::union_map Schedule) {
   SubExtent = SubExtent.project_out(isl::dim::set, 1, SubDims - 1);
 
   if (!isDimBoundedByConstant(SubExtent, 0)) {
-    LLVM_DEBUG(dbgs() << "Abort; dimension not bounded by constant\n");
+    POLLY_DEBUG(dbgs() << "Abort; dimension not bounded by constant\n");
     return {};
   }
 
   auto Min = SubExtent.dim_min(0);
-  LLVM_DEBUG(dbgs() << "Min bound:\n  " << Min << "\n");
+  POLLY_DEBUG(dbgs() << "Min bound:\n  " << Min << "\n");
   auto MinVal = getConstant(Min, false, true);
   auto Max = SubExtent.dim_max(0);
-  LLVM_DEBUG(dbgs() << "Max bound:\n  " << Max << "\n");
+  POLLY_DEBUG(dbgs() << "Max bound:\n  " << Max << "\n");
   auto MaxVal = getConstant(Max, true, false);
 
   if (MinVal.is_null() || MaxVal.is_null() || MinVal.is_nan() ||
       MaxVal.is_nan()) {
-    LLVM_DEBUG(dbgs() << "Abort; dimension bounds could not be determined\n");
+    POLLY_DEBUG(dbgs() << "Abort; dimension bounds could not be determined\n");
     return {};
   }
 
@@ -297,15 +298,15 @@ isl::union_map tryFlattenLoop(isl::union_map Schedule) {
   auto IndexMap = isl::union_map::from(Index);
 
   auto Result = IndexMap.flat_range_product(RemainingSubSchedule);
-  LLVM_DEBUG(dbgs() << "Loop-flatten result is:\n  " << Result << "\n");
+  POLLY_DEBUG(dbgs() << "Loop-flatten result is:\n  " << Result << "\n");
   return Result;
 }
 } // anonymous namespace
 
 isl::union_map polly::flattenSchedule(isl::union_map Schedule) {
   unsigned Dims = getNumScatterDims(Schedule);
-  LLVM_DEBUG(dbgs() << "Recursive schedule to process:\n  " << Schedule
-                    << "\n");
+  POLLY_DEBUG(dbgs() << "Recursive schedule to process:\n  " << Schedule
+                     << "\n");
 
   // Base case; no dimensions left
   if (Dims == 0) {
@@ -319,20 +320,20 @@ isl::union_map polly::flattenSchedule(isl::union_map Schedule) {
 
   // Fixed dimension; no need to preserve variabledness.
   if (!isVariableDim(Schedule)) {
-    LLVM_DEBUG(dbgs() << "Fixed dimension; try sequence flattening\n");
+    POLLY_DEBUG(dbgs() << "Fixed dimension; try sequence flattening\n");
     auto NewScheduleSequence = tryFlattenSequence(Schedule);
     if (!NewScheduleSequence.is_null())
       return NewScheduleSequence;
   }
 
   // Constant stride
-  LLVM_DEBUG(dbgs() << "Try loop flattening\n");
+  POLLY_DEBUG(dbgs() << "Try loop flattening\n");
   auto NewScheduleLoop = tryFlattenLoop(Schedule);
   if (!NewScheduleLoop.is_null())
     return NewScheduleLoop;
 
   // Try again without loop condition (may blow up the number of pieces!!)
-  LLVM_DEBUG(dbgs() << "Try sequence flattening again\n");
+  POLLY_DEBUG(dbgs() << "Try sequence flattening again\n");
   auto NewScheduleSequence = tryFlattenSequence(Schedule);
   if (!NewScheduleSequence.is_null())
     return NewScheduleSequence;
diff --git a/polly/lib/Transform/FlattenSchedule.cpp b/polly/lib/Transform/FlattenSchedule.cpp
index 87bf642ba0d9..f514ef359ba0 100644
--- a/polly/lib/Transform/FlattenSchedule.cpp
+++ b/polly/lib/Transform/FlattenSchedule.cpp
@@ -18,6 +18,7 @@
 #include "polly/ScopPass.h"
 #include "polly/Support/ISLOStream.h"
 #include "polly/Support/ISLTools.h"
+#include "polly/Support/PollyDebug.h"
 #define DEBUG_TYPE "polly-flatten-schedule"
 
 using namespace polly;
@@ -57,23 +58,23 @@ public:
     // OldSchedule.
     IslCtx = S.getSharedIslCtx();
 
-    LLVM_DEBUG(dbgs() << "Going to flatten old schedule:\n");
+    POLLY_DEBUG(dbgs() << "Going to flatten old schedule:\n");
     OldSchedule = S.getSchedule();
-    LLVM_DEBUG(printSchedule(dbgs(), OldSchedule, 2));
+    POLLY_DEBUG(printSchedule(dbgs(), OldSchedule, 2));
 
     auto Domains = S.getDomains();
     auto RestrictedOldSchedule = OldSchedule.intersect_domain(Domains);
-    LLVM_DEBUG(dbgs() << "Old schedule with domains:\n");
-    LLVM_DEBUG(printSchedule(dbgs(), RestrictedOldSchedule, 2));
+    POLLY_DEBUG(dbgs() << "Old schedule with domains:\n");
+    POLLY_DEBUG(printSchedule(dbgs(), RestrictedOldSchedule, 2));
 
     auto NewSchedule = flattenSchedule(RestrictedOldSchedule);
 
-    LLVM_DEBUG(dbgs() << "Flattened new schedule:\n");
-    LLVM_DEBUG(printSchedule(dbgs(), NewSchedule, 2));
+    POLLY_DEBUG(dbgs() << "Flattened new schedule:\n");
+    POLLY_DEBUG(printSchedule(dbgs(), NewSchedule, 2));
 
     NewSchedule = NewSchedule.gist_domain(Domains);
-    LLVM_DEBUG(dbgs() << "Gisted, flattened new schedule:\n");
-    LLVM_DEBUG(printSchedule(dbgs(), NewSchedule, 2));
+    POLLY_DEBUG(dbgs() << "Gisted, flattened new schedule:\n");
+    POLLY_DEBUG(printSchedule(dbgs(), NewSchedule, 2));
 
     S.setSchedule(NewSchedule);
     return false;
diff --git a/polly/lib/Transform/ForwardOpTree.cpp b/polly/lib/Transform/ForwardOpTree.cpp
index 5e6de2e182a5..e9be6c9cdcc2 100644
--- a/polly/lib/Transform/ForwardOpTree.cpp
+++ b/polly/lib/Transform/ForwardOpTree.cpp
@@ -40,6 +40,7 @@
 #include <cassert>
 #include <memory>
 
+#include "polly/Support/PollyDebug.h"
 #define DEBUG_TYPE "polly-optree"
 
 using namespace llvm;
@@ -171,7 +172,7 @@ struct ForwardingAction {
     Result.Decision =
         IsProfitable ? FD_CanForwardProfitably : FD_CanForwardLeaf;
     Result.Execute = [=]() {
-      LLVM_DEBUG(dbgs() << "    trivially forwarded: " << *Val << "\n");
+      POLLY_DEBUG(dbgs() << "    trivially forwarded: " << *Val << "\n");
       return true;
     };
     return Result;
@@ -368,12 +369,12 @@ public:
       Known = {};
       Translator = {};
       NormalizeMap = {};
-      LLVM_DEBUG(dbgs() << "Known analysis exceeded max_operations\n");
+      POLLY_DEBUG(dbgs() << "Known analysis exceeded max_operations\n");
       return false;
     }
 
     KnownAnalyzed++;
-    LLVM_DEBUG(dbgs() << "All known: " << Known << "\n");
+    POLLY_DEBUG(dbgs() << "All known: " << Known << "\n");
 
     return true;
   }
@@ -490,7 +491,7 @@ public:
       //   do not add another MemoryAccess.
       auto ExecAction = [this, TargetStmt, LI, Access]() -> bool {
         TargetStmt->prependInstruction(LI);
-        LLVM_DEBUG(
+        POLLY_DEBUG(
             dbgs() << "    forwarded known load with preexisting MemoryAccess"
                    << Access << "\n");
         (void)Access;
@@ -528,10 +529,10 @@ public:
     if (SameVal.is_null())
       return ForwardingAction::notApplicable();
 
-    LLVM_DEBUG(dbgs() << "      expected values where " << TargetExpectedVal
-                      << "\n");
-    LLVM_DEBUG(dbgs() << "      candidate elements where " << Candidates
-                      << "\n");
+    POLLY_DEBUG(dbgs() << "      expected values where " << TargetExpectedVal
+                       << "\n");
+    POLLY_DEBUG(dbgs() << "      candidate elements where " << Candidates
+                       << "\n");
 
     // { ValInst[] }
     isl::space ValInstSpace = ExpectedVal.get_space().range();
@@ -568,8 +569,8 @@ public:
 
       // { [TargetDomain[] -> Value[]] -> [DefDomain[] -> Value] }
       LocalTranslator = DefToTarget.reverse().product(ValToVal);
-      LLVM_DEBUG(dbgs() << "      local translator is " << LocalTranslator
-                        << "\n");
+      POLLY_DEBUG(dbgs() << "      local translator is " << LocalTranslator
+                         << "\n");
 
       if (LocalTranslator.is_null())
         return ForwardingAction::notApplicable();
@@ -579,8 +580,8 @@ public:
                        LocalTranslator]() -> bool {
       TargetStmt->prependInstruction(LI);
       MemoryAccess *Access = makeReadArrayAccess(TargetStmt, LI, SameVal);
-      LLVM_DEBUG(dbgs() << "    forwarded known load with new MemoryAccess"
-                        << Access << "\n");
+      POLLY_DEBUG(dbgs() << "    forwarded known load with new MemoryAccess"
+                         << Access << "\n");
       (void)Access;
 
       if (!LocalTranslator.is_null())
@@ -643,8 +644,8 @@ public:
         Access = TargetStmt->ensureValueRead(Inst);
       Access->setNewAccessRelation(SameVal);
 
-      LLVM_DEBUG(dbgs() << "    forwarded known content of " << *Inst
-                        << " which is " << SameVal << "\n");
+      POLLY_DEBUG(dbgs() << "    forwarded known content of " << *Inst
+                         << " which is " << SameVal << "\n");
       TotalReloads++;
       NumReloads++;
       return false;
@@ -712,8 +713,8 @@ public:
       // instruction using them.
       TargetStmt->prependInstruction(UseInst);
 
-      LLVM_DEBUG(dbgs() << "    forwarded speculable instruction: " << *UseInst
-                        << "\n");
+      POLLY_DEBUG(dbgs() << "    forwarded speculable instruction: " << *UseInst
+                         << "\n");
       NumInstructionsCopied++;
       TotalInstructionsCopied++;
       return true;
@@ -765,7 +766,7 @@ public:
       if (TargetUse.getKind() == VirtualUse::Synthesizable)
         return ForwardingAction::triviallyForwardable(false, UseVal);
 
-      LLVM_DEBUG(
+      POLLY_DEBUG(
           dbgs() << "    Synthesizable would not be synthesizable anymore: "
                  << *UseVal << "\n");
       return ForwardingAction::cannotForward();
@@ -779,8 +780,8 @@ public:
       auto ExecAction = [this, TargetStmt, UseVal]() {
         TargetStmt->ensureValueRead(UseVal);
 
-        LLVM_DEBUG(dbgs() << "    forwarded read-only value " << *UseVal
-                          << "\n");
+        POLLY_DEBUG(dbgs() << "    forwarded read-only value " << *UseVal
+                           << "\n");
         NumReadOnlyCopied++;
         TotalReadOnlyCopied++;
 
@@ -830,7 +831,8 @@ public:
 
       // When no method is found to forward the operand tree, we effectively
       // cannot handle it.
-      LLVM_DEBUG(dbgs() << "    Cannot forward instruction: " << *Inst << "\n");
+      POLLY_DEBUG(dbgs() << "    Cannot forward instruction: " << *Inst
+                         << "\n");
       return ForwardingAction::cannotForward();
     }
 
@@ -945,7 +947,7 @@ public:
   /// Try to forward an operand tree rooted in @p RA.
   bool tryForwardTree(MemoryAccess *RA) {
     assert(RA->isLatestScalarKind());
-    LLVM_DEBUG(dbgs() << "Trying to forward operand tree " << RA << "...\n");
+    POLLY_DEBUG(dbgs() << "Trying to forward operand tree " << RA << "...\n");
 
     ScopStmt *Stmt = RA->getStatement();
     Loop *InLoop = Stmt->getSurroundingLoop();
@@ -1036,22 +1038,22 @@ static std::unique_ptr<ForwardOpTreeImpl> runForwardOpTree(Scop &S,
     Impl = std::make_unique<ForwardOpTreeImpl>(&S, &LI, MaxOpGuard);
 
     if (AnalyzeKnown) {
-      LLVM_DEBUG(dbgs() << "Prepare forwarders...\n");
+      POLLY_DEBUG(dbgs() << "Prepare forwarders...\n");
       Impl->computeKnownValues();
     }
 
-    LLVM_DEBUG(dbgs() << "Forwarding operand trees...\n");
+    POLLY_DEBUG(dbgs() << "Forwarding operand trees...\n");
     Impl->forwardOperandTrees();
 
     if (MaxOpGuard.hasQuotaExceeded()) {
-      LLVM_DEBUG(dbgs() << "Not all operations completed because of "
-                           "max_operations exceeded\n");
+      POLLY_DEBUG(dbgs() << "Not all operations completed because of "
+                            "max_operations exceeded\n");
       KnownOutOfQuota++;
     }
   }
 
-  LLVM_DEBUG(dbgs() << "\nFinal Scop:\n");
-  LLVM_DEBUG(dbgs() << S);
+  POLLY_DEBUG(dbgs() << "\nFinal Scop:\n");
+  POLLY_DEBUG(dbgs() << S);
 
   // Update statistics
   Scop::ScopStatistics ScopStats = S.getStatistics();
diff --git a/polly/lib/Transform/ManualOptimizer.cpp b/polly/lib/Transform/ManualOptimizer.cpp
index 264491b7577b..0e330f207fbc 100644
--- a/polly/lib/Transform/ManualOptimizer.cpp
+++ b/polly/lib/Transform/ManualOptimizer.cpp
@@ -22,6 +22,7 @@
 #include "llvm/Transforms/Utils/LoopUtils.h"
 #include <optional>
 
+#include "polly/Support/PollyDebug.h"
 #define DEBUG_TYPE "polly-opt-manual"
 
 using namespace polly;
@@ -159,13 +160,13 @@ private:
       return Result;
 
     LLVMContext &Ctx = LoopMD->getContext();
-    LLVM_DEBUG(dbgs() << "Dependency violation detected\n");
+    POLLY_DEBUG(dbgs() << "Dependency violation detected\n");
 
     DebugLoc TransformLoc = findTransformationDebugLoc(LoopMD, DebugLocAttr);
 
     if (IgnoreDepcheck) {
-      LLVM_DEBUG(dbgs() << "Still accepting transformation due to "
-                           "-polly-pragma-ignore-depcheck\n");
+      POLLY_DEBUG(dbgs() << "Still accepting transformation due to "
+                            "-polly-pragma-ignore-depcheck\n");
       if (ORE) {
         ORE->emit(
             OptimizationRemark(DEBUG_TYPE, RemarkName, TransformLoc, CodeRegion)
@@ -177,7 +178,7 @@ private:
       return Result;
     }
 
-    LLVM_DEBUG(dbgs() << "Rolling back transformation\n");
+    POLLY_DEBUG(dbgs() << "Rolling back transformation\n");
 
     if (ORE) {
       ORE->emit(DiagnosticInfoOptimizationFailure(DEBUG_TYPE, RemarkName,
diff --git a/polly/lib/Transform/MatmulOptimizer.cpp b/polly/lib/Transform/MatmulOptimizer.cpp
index 05578bd9ed11..51ae5a778e4f 100644
--- a/polly/lib/Transform/MatmulOptimizer.cpp
+++ b/polly/lib/Transform/MatmulOptimizer.cpp
@@ -42,6 +42,7 @@
 #include <string>
 #include <vector>
 
+#include "polly/Support/PollyDebug.h"
 #define DEBUG_TYPE "polly-opt-isl"
 
 using namespace llvm;
@@ -1825,10 +1826,10 @@ polly::tryOptimizeMatMulPattern(isl::schedule_node Node,
                                 const Dependences *D) {
   TCInfoTy TCI;
   if (PMBasedTCOpts && isTCPattern(Node, D, TCI))
-    LLVM_DEBUG(dbgs() << "The tensor contraction pattern was detected\n");
+    POLLY_DEBUG(dbgs() << "The tensor contraction pattern was detected\n");
   MatMulInfoTy MMI;
   if (PMBasedMMMOpts && isMatrMultPattern(Node, D, MMI)) {
-    LLVM_DEBUG(dbgs() << "The matrix multiplication pattern was detected\n");
+    POLLY_DEBUG(dbgs() << "The matrix multiplication pattern was detected\n");
     return optimizeMatMulPattern(Node, TTI, MMI);
   }
   return {};
diff --git a/polly/lib/Transform/ScheduleOptimizer.cpp b/polly/lib/Transform/ScheduleOptimizer.cpp
index 5a0ea3b40675..55d51982d90e 100644
--- a/polly/lib/Transform/ScheduleOptimizer.cpp
+++ b/polly/lib/Transform/ScheduleOptimizer.cpp
@@ -69,6 +69,7 @@ class Loop;
 class Module;
 } // namespace llvm
 
+#include "polly/Support/PollyDebug.h"
 #define DEBUG_TYPE "polly-opt-isl"
 
 static cl::opt<std::string>
@@ -730,14 +731,14 @@ static void runIslScheduleOptimizer(
   // Schedule without optimizations.
   isl::schedule Schedule = S.getScheduleTree();
   walkScheduleTreeForStatistics(S.getScheduleTree(), 0);
-  LLVM_DEBUG(printSchedule(dbgs(), Schedule, "Original schedule tree"));
+  POLLY_DEBUG(printSchedule(dbgs(), Schedule, "Original schedule tree"));
 
   bool HasUserTransformation = false;
   if (PragmaBasedOpts) {
     isl::schedule ManuallyTransformed = applyManualTransformations(
         &S, Schedule, GetDeps(Dependences::AL_Statement), ORE);
     if (ManuallyTransformed.is_null()) {
-      LLVM_DEBUG(dbgs() << "Error during manual optimization\n");
+      POLLY_DEBUG(dbgs() << "Error during manual optimization\n");
       return;
     }
 
@@ -745,7 +746,7 @@ static void runIslScheduleOptimizer(
       // User transformations have precedence over other transformations.
       HasUserTransformation = true;
       Schedule = std::move(ManuallyTransformed);
-      LLVM_DEBUG(
+      POLLY_DEBUG(
           printSchedule(dbgs(), Schedule, "After manual transformations"));
     }
   }
@@ -755,18 +756,18 @@ static void runIslScheduleOptimizer(
   // TODO: Detect disabled heuristics and no user-directed transformation
   // metadata earlier in ScopDetection.
   if (!HasUserTransformation && S.hasDisableHeuristicsHint()) {
-    LLVM_DEBUG(dbgs() << "Heuristic optimizations disabled by metadata\n");
+    POLLY_DEBUG(dbgs() << "Heuristic optimizations disabled by metadata\n");
     return;
   }
 
   // Get dependency analysis.
   const Dependences &D = GetDeps(Dependences::AL_Statement);
   if (D.getSharedIslCtx() != S.getSharedIslCtx()) {
-    LLVM_DEBUG(dbgs() << "DependenceInfo for another SCoP/isl_ctx\n");
+    POLLY_DEBUG(dbgs() << "DependenceInfo for another SCoP/isl_ctx\n");
     return;
   }
   if (!D.hasValidDependences()) {
-    LLVM_DEBUG(dbgs() << "Dependency information not available\n");
+    POLLY_DEBUG(dbgs() << "Dependency information not available\n");
     return;
   }
 
@@ -776,9 +777,9 @@ static void runIslScheduleOptimizer(
   // are added by the rescheduling analyzer. Therefore, disabling the
   // rescheduler implicitly also disables these optimizations.
   if (!EnableReschedule) {
-    LLVM_DEBUG(dbgs() << "Skipping rescheduling due to command line option\n");
+    POLLY_DEBUG(dbgs() << "Skipping rescheduling due to command line option\n");
   } else if (HasUserTransformation) {
-    LLVM_DEBUG(
+    POLLY_DEBUG(
         dbgs() << "Skipping rescheduling due to manual transformation\n");
   } else {
     // Build input data.
@@ -824,10 +825,10 @@ static void runIslScheduleOptimizer(
              "or 'no'. Falling back to default: 'yes'\n";
     }
 
-    LLVM_DEBUG(dbgs() << "\n\nCompute schedule from: ");
-    LLVM_DEBUG(dbgs() << "Domain := " << Domain << ";\n");
-    LLVM_DEBUG(dbgs() << "Proximity := " << Proximity << ";\n");
-    LLVM_DEBUG(dbgs() << "Validity := " << Validity << ";\n");
+    POLLY_DEBUG(dbgs() << "\n\nCompute schedule from: ");
+    POLLY_DEBUG(dbgs() << "Domain := " << Domain << ";\n");
+    POLLY_DEBUG(dbgs() << "Proximity := " << Proximity << ";\n");
+    POLLY_DEBUG(dbgs() << "Validity := " << Validity << ";\n");
 
     int IslMaximizeBands;
     if (MaximizeBandDepth == "yes") {
@@ -873,14 +874,14 @@ static void runIslScheduleOptimizer(
       Schedule = SC.compute_schedule();
 
       if (MaxOpGuard.hasQuotaExceeded())
-        LLVM_DEBUG(
+        POLLY_DEBUG(
             dbgs() << "Schedule optimizer calculation exceeds ISL quota\n");
     }
 
     isl_options_set_on_error(Ctx, OnErrorStatus);
 
     ScopsRescheduled++;
-    LLVM_DEBUG(printSchedule(dbgs(), Schedule, "After rescheduling"));
+    POLLY_DEBUG(printSchedule(dbgs(), Schedule, "After rescheduling"));
   }
 
   walkScheduleTreeForStatistics(Schedule, 1);
@@ -908,7 +909,7 @@ static void runIslScheduleOptimizer(
   if (OAI.PatternOpts || OAI.Postopts || OAI.Prevect) {
     Schedule = ScheduleTreeOptimizer::optimizeSchedule(Schedule, &OAI);
     Schedule = hoistExtensionNodes(Schedule);
-    LLVM_DEBUG(printSchedule(dbgs(), Schedule, "After post-optimizations"));
+    POLLY_DEBUG(printSchedule(dbgs(), Schedule, "After post-optimizations"));
     walkScheduleTreeForStatistics(Schedule, 2);
   }
 
diff --git a/polly/lib/Transform/ScheduleTreeTransform.cpp b/polly/lib/Transform/ScheduleTreeTransform.cpp
index e42b3d1c2460..f0684de825d2 100644
--- a/polly/lib/Transform/ScheduleTreeTransform.cpp
+++ b/polly/lib/Transform/ScheduleTreeTransform.cpp
@@ -21,6 +21,7 @@
 #include "llvm/IR/Metadata.h"
 #include "llvm/Transforms/Utils/UnrollLoop.h"
 
+#include "polly/Support/PollyDebug.h"
 #define DEBUG_TYPE "polly-opt-isl"
 
 using namespace polly;
@@ -599,7 +600,7 @@ public:
     if (Nest.size() <= 1)
       return getBase().visitBand(Band);
 
-    LLVM_DEBUG({
+    POLLY_DEBUG({
       dbgs() << "Found loops to collapse between\n";
       dumpIslObj(RootBand, dbgs());
       dbgs() << "and\n";
@@ -644,7 +645,7 @@ public:
 };
 
 static isl::schedule collapseBands(isl::schedule Sched) {
-  LLVM_DEBUG(dbgs() << "Collapse bands in schedule\n");
+  POLLY_DEBUG(dbgs() << "Collapse bands in schedule\n");
   BandCollapseRewriter Rewriter;
   return Rewriter.visit(Sched);
 }
@@ -773,7 +774,7 @@ static isl::schedule tryGreedyFuse(isl::schedule_node_band LHS,
   if (!canFuseOutermost(LHS, RHS, Deps))
     return {};
 
-  LLVM_DEBUG({
+  POLLY_DEBUG({
     dbgs() << "Found loops for greedy fusion:\n";
     dumpIslObj(LHS, dbgs());
     dbgs() << "and\n";
@@ -1228,12 +1229,12 @@ isl::schedule polly::applyMaxFission(isl::schedule_node BandToFission) {
 
 isl::schedule polly::applyGreedyFusion(isl::schedule Sched,
                                        const isl::union_map &Deps) {
-  LLVM_DEBUG(dbgs() << "Greedy loop fusion\n");
+  POLLY_DEBUG(dbgs() << "Greedy loop fusion\n");
 
   GreedyFusionRewriter Rewriter;
   isl::schedule Result = Rewriter.visit(Sched, Deps);
   if (!Rewriter.AnyChange) {
-    LLVM_DEBUG(dbgs() << "Found nothing to fuse\n");
+    POLLY_DEBUG(dbgs() << "Found nothing to fuse\n");
     return Sched;
   }
 
diff --git a/polly/lib/Transform/ScopInliner.cpp b/polly/lib/Transform/ScopInliner.cpp
index ca61407d2587..b78206c1e40b 100644
--- a/polly/lib/Transform/ScopInliner.cpp
+++ b/polly/lib/Transform/ScopInliner.cpp
@@ -21,6 +21,7 @@
 #include "llvm/Passes/PassBuilder.h"
 #include "llvm/Transforms/IPO/AlwaysInliner.h"
 
+#include "polly/Support/PollyDebug.h"
 #define DEBUG_TYPE "polly-scop-inliner"
 
 using namespace llvm;
@@ -60,8 +61,8 @@ public:
     if (!F)
       return false;
     if (F->isDeclaration()) {
-      LLVM_DEBUG(dbgs() << "Skipping " << F->getName()
-                        << "because it is a declaration.\n");
+      POLLY_DEBUG(dbgs() << "Skipping " << F->getName()
+                         << "because it is a declaration.\n");
       return false;
     }
 
@@ -86,8 +87,8 @@ public:
 
     bool Changed = false;
     if (HasScopAsTopLevelRegion) {
-      LLVM_DEBUG(dbgs() << "Skipping " << F->getName()
-                        << " has scop as top level region");
+      POLLY_DEBUG(dbgs() << "Skipping " << F->getName()
+                         << " has scop as top level region");
       F->addFnAttr(llvm::Attribute::AlwaysInline);
 
       ModulePassManager MPM;
@@ -98,8 +99,8 @@ public:
       if (!PA.areAllPreserved())
         Changed = true;
     } else {
-      LLVM_DEBUG(dbgs() << F->getName()
-                        << " does NOT have scop as top level region\n");
+      POLLY_DEBUG(dbgs() << F->getName()
+                         << " does NOT have scop as top level region\n");
     }
 
     return Changed;
diff --git a/polly/lib/Transform/Simplify.cpp b/polly/lib/Transform/Simplify.cpp
index 41a155a41de8..75e91cd1c031 100644
--- a/polly/lib/Transform/Simplify.cpp
+++ b/polly/lib/Transform/Simplify.cpp
@@ -22,6 +22,7 @@
 #include "llvm/Support/Debug.h"
 #include <optional>
 
+#include "polly/Support/PollyDebug.h"
 #define DEBUG_TYPE "polly-simplify"
 
 using namespace llvm;
@@ -237,8 +238,8 @@ void SimplifyImpl::removeEmptyDomainStmts() {
 
   assert(NumStmtsBefore >= S->getSize());
   EmptyDomainsRemoved = NumStmtsBefore - S->getSize();
-  LLVM_DEBUG(dbgs() << "Removed " << EmptyDomainsRemoved << " (of "
-                    << NumStmtsBefore << ") statements with empty domains \n");
+  POLLY_DEBUG(dbgs() << "Removed " << EmptyDomainsRemoved << " (of "
+                     << NumStmtsBefore << ") statements with empty domains \n");
   TotalEmptyDomainsRemoved[CallNo] += EmptyDomainsRemoved;
 }
 
@@ -280,8 +281,8 @@ void SimplifyImpl::removeOverwrites() {
       // If all of a write's elements are overwritten, remove it.
       isl::union_map AccRelUnion = AccRel;
       if (AccRelUnion.is_subset(WillBeOverwritten)) {
-        LLVM_DEBUG(dbgs() << "Removing " << MA
-                          << " which will be overwritten anyway\n");
+        POLLY_DEBUG(dbgs() << "Removing " << MA
+                           << " which will be overwritten anyway\n");
 
         Stmt.removeSingleMemoryAccess(MA);
         OverwritesRemoved++;
@@ -532,9 +533,9 @@ void SimplifyImpl::removeRedundantWrites() {
           isl::map AccRelStoredVal = isl::map::from_domain_and_range(
               AccRelWrapped, makeValueSet(StoredVal));
           if (isl::union_map(AccRelStoredVal).is_subset(Known)) {
-            LLVM_DEBUG(dbgs() << "Cleanup of " << MA << ":\n");
-            LLVM_DEBUG(dbgs() << "      Scalar: " << *StoredVal << "\n");
-            LLVM_DEBUG(dbgs() << "      AccRel: " << AccRel << "\n");
+            POLLY_DEBUG(dbgs() << "Cleanup of " << MA << ":\n");
+            POLLY_DEBUG(dbgs() << "      Scalar: " << *StoredVal << "\n");
+            POLLY_DEBUG(dbgs() << "      AccRel: " << AccRel << "\n");
 
             Stmt.removeSingleMemoryAccess(MA);
 
@@ -576,8 +577,8 @@ void SimplifyImpl::removeUnnecessaryStmts() {
   S->simplifySCoP(true);
   assert(NumStmtsBefore >= S->getSize());
   StmtsRemoved = NumStmtsBefore - S->getSize();
-  LLVM_DEBUG(dbgs() << "Removed " << StmtsRemoved << " (of " << NumStmtsBefore
-                    << ") statements\n");
+  POLLY_DEBUG(dbgs() << "Removed " << StmtsRemoved << " (of " << NumStmtsBefore
+                     << ") statements\n");
   TotalStmtsRemoved[CallNo] += StmtsRemoved;
 }
 
@@ -595,7 +596,7 @@ void SimplifyImpl::removeEmptyPartialAccesses() {
       if (!AccRel.is_empty().is_true())
         continue;
 
-      LLVM_DEBUG(
+      POLLY_DEBUG(
           dbgs() << "Removing " << MA
                  << " because it's a partial access that never occurs\n");
       DeferredRemove.push_back(MA);
@@ -628,8 +629,8 @@ void SimplifyImpl::markAndSweep(LoopInfo *LI) {
   for (MemoryAccess *MA : AllMAs) {
     if (UsedMA.count(MA))
       continue;
-    LLVM_DEBUG(dbgs() << "Removing " << MA
-                      << " because its value is not used\n");
+    POLLY_DEBUG(dbgs() << "Removing " << MA
+                       << " because its value is not used\n");
     ScopStmt *Stmt = MA->getStatement();
     Stmt->removeSingleMemoryAccess(MA);
 
@@ -650,8 +651,8 @@ void SimplifyImpl::markAndSweep(LoopInfo *LI) {
     for (Instruction *Inst : AllInsts) {
       auto It = UsedInsts.find({&Stmt, Inst});
       if (It == UsedInsts.end()) {
-        LLVM_DEBUG(dbgs() << "Removing "; Inst->print(dbgs());
-                   dbgs() << " because it is not used\n");
+        POLLY_DEBUG(dbgs() << "Removing "; Inst->print(dbgs());
+                    dbgs() << " because it is not used\n");
         DeadInstructionsRemoved++;
         TotalDeadInstructionsRemoved[CallNo]++;
         continue;
@@ -708,31 +709,31 @@ void SimplifyImpl::run(Scop &S, LoopInfo *LI) {
   this->S = &S;
   ScopsProcessed[CallNo]++;
 
-  LLVM_DEBUG(dbgs() << "Removing statements that are never executed...\n");
+  POLLY_DEBUG(dbgs() << "Removing statements that are never executed...\n");
   removeEmptyDomainStmts();
 
-  LLVM_DEBUG(dbgs() << "Removing partial writes that never happen...\n");
+  POLLY_DEBUG(dbgs() << "Removing partial writes that never happen...\n");
   removeEmptyPartialAccesses();
 
-  LLVM_DEBUG(dbgs() << "Removing overwrites...\n");
+  POLLY_DEBUG(dbgs() << "Removing overwrites...\n");
   removeOverwrites();
 
-  LLVM_DEBUG(dbgs() << "Coalesce partial writes...\n");
+  POLLY_DEBUG(dbgs() << "Coalesce partial writes...\n");
   coalesceWrites();
 
-  LLVM_DEBUG(dbgs() << "Removing redundant writes...\n");
+  POLLY_DEBUG(dbgs() << "Removing redundant writes...\n");
   removeRedundantWrites();
 
-  LLVM_DEBUG(dbgs() << "Cleanup unused accesses...\n");
+  POLLY_DEBUG(dbgs() << "Cleanup unused accesses...\n");
   markAndSweep(LI);
 
-  LLVM_DEBUG(dbgs() << "Removing statements without side effects...\n");
+  POLLY_DEBUG(dbgs() << "Removing statements without side effects...\n");
   removeUnnecessaryStmts();
 
   if (isModified())
     ScopsModified[CallNo]++;
-  LLVM_DEBUG(dbgs() << "\nFinal Scop:\n");
-  LLVM_DEBUG(dbgs() << S);
+  POLLY_DEBUG(dbgs() << "\nFinal Scop:\n");
+  POLLY_DEBUG(dbgs() << S);
 
   auto ScopStats = S.getStatistics();
   NumValueWrites[CallNo] += ScopStats.NumValueWrites;
diff --git a/polly/lib/Transform/ZoneAlgo.cpp b/polly/lib/Transform/ZoneAlgo.cpp
index 4c86891d2cf7..a114a241d87a 100644
--- a/polly/lib/Transform/ZoneAlgo.cpp
+++ b/polly/lib/Transform/ZoneAlgo.cpp
@@ -156,6 +156,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Support/raw_ostream.h"
 
+#include "polly/Support/PollyDebug.h"
 #define DEBUG_TYPE "polly-zone"
 
 STATISTIC(NumIncompatibleArrays, "Number of not zone-analyzable arrays");
@@ -342,7 +343,7 @@ void ZoneAlgorithm::collectIncompatibleElts(ScopStmt *Stmt,
     if (MA->isRead()) {
       // Reject load after store to same location.
       if (!Stores.is_disjoint(AccRel)) {
-        LLVM_DEBUG(
+        POLLY_DEBUG(
             dbgs() << "Load after store of same element in same statement\n");
         OptimizationRemarkMissed R(PassName, "LoadAfterStore",
                                    MA->getAccessInstruction());
@@ -362,7 +363,7 @@ void ZoneAlgorithm::collectIncompatibleElts(ScopStmt *Stmt,
     // In region statements the order is less clear, eg. the load and store
     // might be in a boxed loop.
     if (Stmt->isRegionStmt() && !Loads.is_disjoint(AccRel)) {
-      LLVM_DEBUG(dbgs() << "WRITE in non-affine subregion not supported\n");
+      POLLY_DEBUG(dbgs() << "WRITE in non-affine subregion not supported\n");
       OptimizationRemarkMissed R(PassName, "StoreInSubregion",
                                  MA->getAccessInstruction());
       R << "store is in a non-affine subregion";
@@ -373,7 +374,7 @@ void ZoneAlgorithm::collectIncompatibleElts(ScopStmt *Stmt,
 
     // Do not allow more than one store to the same location.
     if (!Stores.is_disjoint(AccRel) && !onlySameValueWrites(Stmt)) {
-      LLVM_DEBUG(dbgs() << "WRITE after WRITE to same element\n");
+      POLLY_DEBUG(dbgs() << "WRITE after WRITE to same element\n");
       OptimizationRemarkMissed R(PassName, "StoreAfterStore",
                                  MA->getAccessInstruction());
       R << "store after store of same element in same statement";
diff --git a/polly/test/Support/pollyDebug.ll b/polly/test/Support/pollyDebug.ll
new file mode 100644
index 000000000000..ada079023b6c
--- /dev/null
+++ b/polly/test/Support/pollyDebug.ll
@@ -0,0 +1,85 @@
+; Test if "polly-debug" flag enables debug prints from different parts of polly
+; RUN: opt %loadNPMPolly -O3 -polly -polly-debug --disable-output < %s 2>&1 | FileCheck %s
+;
+; REQUIRES: asserts
+
+; void callee(int n, double A[], int i) {
+;   for (int j = 0; j < n; j += 1)
+;     A[i+j] = 42.0;
+; }
+;
+; void caller(int n, double A[]) {
+;   for (int i = 0; i < n; i += 1)
+;     callee(n, A, i);
+; }
+
+
+%unrelated_type = type { i32 }
+
+define internal void @callee(i32 %n, ptr noalias nonnull %A, i32 %i) #0 {
+entry:
+  br label %for
+
+for:
+  %j = phi i32 [0, %entry], [%j.inc, %inc]
+  %j.cmp = icmp slt i32 %j, %n
+  br i1 %j.cmp, label %body, label %exit
+
+    body:
+      %idx = add i32 %i, %j
+      %arrayidx = getelementptr inbounds double, ptr %A, i32 %idx
+      store double 42.0, ptr %arrayidx
+      br label %inc
+
+inc:
+  %j.inc = add nuw nsw i32 %j, 1
+  br label %for
+
+exit:
+  br label %return
+
+return:
+  ret void
+}
+
+
+define void @caller(i32 %n, ptr noalias nonnull %A) #0 {
+entry:
+  br label %for
+
+for:
+  %i = phi i32 [0, %entry], [%j.inc, %inc]
+  %i.cmp = icmp slt i32 %i, %n
+  br i1 %i.cmp, label %body, label %exit
+
+    body:
+      call void @callee(i32 %n, ptr %A, i32 %i)
+      br label %inc
+
+inc:
+  %j.inc = add nuw nsw i32 %i, 1
+  br label %for
+
+exit:
+  br label %return
+
+return:
+  ret void
+}
+
+
+declare void @unrelated_decl()
+
+
+attributes #0 = { noinline }
+
+!llvm.ident = !{!8}
+!8 = !{!"xyxxy"}
+
+; CHECK: Checking region: entry => <Function Return>
+; CHECK: Removing statements that are never executed...
+; CHECK: Final Scop:
+; CHECK: Forwarding operand trees...
+; CHECK: Final Scop:
+; CHECK: Collapsing scalars to unused array elements...
+; CHECK: Final Scop:
-- 
GitLab


From 0e5c504d3d80eb1488483a15269c2155f9e04bd7 Mon Sep 17 00:00:00 2001
From: Emil Pedersen <3mille.prenom.nom@gmail.com>
Date: Tue, 26 Mar 2024 12:09:22 -0700
Subject: [PATCH 331/404] [DebugInfo] [SelectionDAG] Fix handling of duplicate
 dbg values (#86598)

Before this fix, a duplicate llvm.dbg.value intrinsic referring to an
argument, after an alloca, would be generated with `$noreg`, losing
debug information. Instead, we silently drop the second debug info, so
it doesn't break the first one.

rdar://125375717
---
 .../SelectionDAG/SelectionDAGBuilder.cpp      |  5 +-
 .../X86/dbg-value-funcarg-duplicates.ll       | 65 +++++++++++++++++++
 2 files changed, 69 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/DebugInfo/X86/dbg-value-funcarg-duplicates.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index f99ab55a14ba..5db32472d197 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6007,12 +6007,15 @@ bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
     // incorrect hoisting of the DBG_VALUE to the function entry).
     // Notice that we allow one dbg.value per IR level argument, to accommodate
     // for the situation with fragments above.
+    // If there is no node for the value being handled, we return true to skip
+    // the normal generation of debug info, as it would kill existing debug
+    // info for the parameter in case of duplicates.
     if (VariableIsFunctionInputArg) {
       unsigned ArgNo = Arg->getArgNo();
       if (ArgNo >= FuncInfo.DescribedArgs.size())
         FuncInfo.DescribedArgs.resize(ArgNo + 1, false);
       else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))
-        return false;
+        return !NodeMap[V].getNode();
       FuncInfo.DescribedArgs.set(ArgNo);
     }
   }
diff --git a/llvm/test/DebugInfo/X86/dbg-value-funcarg-duplicates.ll b/llvm/test/DebugInfo/X86/dbg-value-funcarg-duplicates.ll
new file mode 100644
index 000000000000..e61b7256345c
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/dbg-value-funcarg-duplicates.ll
@@ -0,0 +1,65 @@
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -start-after=codegenprepare -stop-before=finalize-isel -o - %s -experimental-debug-variable-locations=false | FileCheck %s
+
+; Input to this test was created by reducing a Swift file using bugpoint
+
+; CHECK-DAG: ![[LHS:.*]] = !DILocalVariable(name: "lhs"
+
+define hidden i64 @"_wideDivide42"(ptr %0, ptr %1, ptr %2, i64 %3, i64 %4, i64 %5, i64 %6, i64 %7, i64 %8) local_unnamed_addr !dbg !16 {
+; CHECK-LABEL: name:            _wideDivide42
+; CHECK-NOT:  DBG_VALUE
+; CHECK:      DBG_VALUE $rcx, $noreg, ![[LHS]], !DIExpression(DW_OP_LLVM_fragment, 0, 64)
+; CHECK-NEXT: DBG_VALUE $r8, $noreg, ![[LHS]], !DIExpression(DW_OP_LLVM_fragment, 64, 64)
+; CHECK-NEXT: DBG_VALUE $r9, $noreg, ![[LHS]], !DIExpression(DW_OP_LLVM_fragment, 128, 64)
+; CHECK-NEXT: DBG_VALUE %fixed-stack.{{.+}}, ![[LHS]], !DIExpression(DW_OP_LLVM_fragment, 192, 64)
+; The duplicates should be removed:
+; CHECK-NOT:  DBG_VALUE
+
+entry:
+  %9 = alloca i64, align 8
+  call void @llvm.dbg.value(metadata i64 %3, metadata !24, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 64)), !dbg !67
+  call void @llvm.dbg.value(metadata i64 %4, metadata !24, metadata !DIExpression(DW_OP_LLVM_fragment, 64, 64)), !dbg !67
+  call void @llvm.dbg.value(metadata i64 %3, metadata !24, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 64)), !dbg !67
+  call void @llvm.dbg.value(metadata i64 %4, metadata !24, metadata !DIExpression(DW_OP_LLVM_fragment, 64, 64)), !dbg !67
+  call void @llvm.dbg.value(metadata i64 %5, metadata !24, metadata !DIExpression(DW_OP_LLVM_fragment, 128, 64)), !dbg !67
+  call void @llvm.dbg.value(metadata i64 %6, metadata !24, metadata !DIExpression(DW_OP_LLVM_fragment, 192, 64)), !dbg !67
+  br i1 poison, label %11, label %10, !dbg !68
+
+10:                                               ; preds = %entry
+  tail call void asm sideeffect "", "n"(i32 7) #7
+  unreachable
+
+11:                                               ; preds = %entry
+  tail call void @abort()
+  unreachable
+}
+
+declare void @abort()
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+attributes #7 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!13}
+!llvm.linker.options = !{!14, !15}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_Swift, file: !1, producer: "Swift", isOptimized: true, runtimeVersion: 6, emissionKind: FullDebug)
+!1 = !DIFile(filename: "Int128.swift", directory: "")
+!13 = !{i32 2, !"Debug Info Version", i32 3}
+!14 = !{!"-lswiftCore"}
+!15 = !{!"-lobjc"}
+!16 = distinct !DISubprogram(name: "_wideDivide42", scope: !0, file: !1, line: 222, type: !17, scopeLine: 222, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !23)
+!17 = !DISubroutineType(types: !18)
+!18 = !{!19, !20, !20, !20, !20, !20, !20}
+!19 = !DICompositeType(tag: DW_TAG_structure_type, name: "4 x UInt64", flags: DIFlagFwdDecl, runtimeLang: DW_LANG_Swift)
+!20 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "UInt64", scope: !1, file: !1, size: 64, elements: !22, runtimeLang: DW_LANG_Swift)
+!22 = !{}
+!23 = !{!24, !27}
+!24 = !DILocalVariable(name: "lhs", arg: 1, scope: !16, file: !1, line: 223, type: !25, flags: DIFlagArtificial)
+!25 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !26)
+!26 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "2 x 2 x UInt64", file: !1, size: 256, elements: !22, runtimeLang: DW_LANG_Swift)
+!27 = !DILocalVariable(name: "rhs", arg: 2, scope: !16, file: !1, line: 223, type: !28, flags: DIFlagArtificial)
+!28 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !29)
+!29 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "2 x UInt64", file: !1, size: 128, elements: !22, runtimeLang: DW_LANG_Swift)
+!67 = !DILocation(line: 0, scope: !16)
+!68 = !DILocation(line: 225, column: 9, scope: !16)
-- 
GitLab


From 28ddbd4a863c4c1b6c36deb6edf305ee93412cd4 Mon Sep 17 00:00:00 2001
From: Chris B <chris.bieneman@me.com>
Date: Tue, 26 Mar 2024 14:15:56 -0500
Subject: [PATCH 332/404] [NFC] Refactor ConstantArrayType size storage
 (#85716)

In PR #79382, I need to add a new type that derives from
ConstantArrayType. This means that ConstantArrayType can no longer use
`llvm::TrailingObjects` to store the trailing optional Expr*.

This change refactors ConstantArrayType to store a 60-bit integer and
4-bits for the integer size in bytes. This replaces the APInt field
previously in the type but preserves enough information to recreate it
where needed.

To reduce the number of places where the APInt is re-constructed I've
also added some helper methods to the ConstantArrayType to allow some
common use cases that operate on either the stored small integer or the
APInt as appropriate.

Resolves #85124.
---
 clang/include/clang/AST/Type.h                | 108 ++++++++++++++----
 clang/lib/AST/ASTContext.cpp                  |  21 ++--
 clang/lib/AST/Decl.cpp                        |   2 +-
 clang/lib/AST/ExprConstant.cpp                |  31 +++--
 clang/lib/AST/Interp/ByteCodeExprGen.cpp      |   8 +-
 clang/lib/AST/Interp/EvaluationResult.cpp     |   2 +-
 clang/lib/AST/Interp/Program.cpp              |   2 +-
 clang/lib/AST/JSONNodeDumper.cpp              |   2 +-
 clang/lib/AST/MicrosoftMangle.cpp             |   6 +-
 clang/lib/AST/ScanfFormatString.cpp           |   4 +-
 clang/lib/AST/Type.cpp                        |  29 +++--
 clang/lib/AST/TypePrinter.cpp                 |   2 +-
 clang/lib/Analysis/CFG.cpp                    |   4 +-
 clang/lib/Analysis/UnsafeBufferUsage.cpp      |   8 +-
 clang/lib/CodeGen/ABIInfo.cpp                 |   4 +-
 clang/lib/CodeGen/ABIInfoImpl.cpp             |   4 +-
 clang/lib/CodeGen/CGCall.cpp                  |   6 +-
 clang/lib/CodeGen/CGDebugInfo.cpp             |   2 +-
 clang/lib/CodeGen/CGExprCXX.cpp               |   6 +-
 clang/lib/CodeGen/CGExprConstant.cpp          |   8 +-
 clang/lib/CodeGen/CGObjCMac.cpp               |   6 +-
 clang/lib/CodeGen/CGOpenMPRuntime.cpp         |   6 +-
 clang/lib/CodeGen/CodeGenFunction.cpp         |   7 +-
 clang/lib/CodeGen/CodeGenModule.cpp           |   2 +-
 clang/lib/CodeGen/CodeGenTypes.cpp            |   2 +-
 clang/lib/CodeGen/SwiftCallingConv.cpp        |   2 +-
 clang/lib/CodeGen/Targets/ARM.cpp             |   2 +-
 clang/lib/CodeGen/Targets/LoongArch.cpp       |   2 +-
 clang/lib/CodeGen/Targets/RISCV.cpp           |   2 +-
 clang/lib/CodeGen/Targets/X86.cpp             |   4 +-
 clang/lib/Sema/SemaChecking.cpp               |   8 +-
 clang/lib/Sema/SemaDecl.cpp                   |   2 +-
 clang/lib/Sema/SemaDeclCXX.cpp                |   2 +-
 clang/lib/Sema/SemaExpr.cpp                   |   5 +-
 clang/lib/Sema/SemaExprCXX.cpp                |   2 +-
 clang/lib/Sema/SemaInit.cpp                   |  21 ++--
 clang/lib/Sema/SemaOpenMP.cpp                 |   6 +-
 clang/lib/Sema/SemaSYCL.cpp                   |   2 +-
 .../Checkers/CastSizeChecker.cpp              |   2 +-
 .../Checkers/PaddingChecker.cpp               |   2 +-
 clang/lib/StaticAnalyzer/Core/MemRegion.cpp   |   2 +-
 clang/lib/StaticAnalyzer/Core/RegionStore.cpp |   8 +-
 42 files changed, 207 insertions(+), 149 deletions(-)

diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index b3bf23227a47..5d8dde37e769 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -1690,7 +1690,10 @@ protected:
 
     /// Whether we have a stored size expression.
     LLVM_PREFERRED_TYPE(bool)
-    unsigned HasStoredSizeExpr : 1;
+    unsigned HasExternalSize : 1;
+
+    LLVM_PREFERRED_TYPE(unsigned)
+    unsigned SizeWidth : 5;
   };
 
   class BuiltinTypeBitfields {
@@ -3338,35 +3341,93 @@ public:
 /// Represents the canonical version of C arrays with a specified constant size.
 /// For example, the canonical type for 'int A[4 + 4*100]' is a
 /// ConstantArrayType where the element type is 'int' and the size is 404.
-class ConstantArrayType final
-    : public ArrayType,
-      private llvm::TrailingObjects<ConstantArrayType, const Expr *> {
+class ConstantArrayType final : public ArrayType {
   friend class ASTContext; // ASTContext creates these.
-  friend TrailingObjects;
 
-  llvm::APInt Size; // Allows us to unique the type.
+  struct ExternalSize {
+    ExternalSize(const llvm::APInt &Sz, const Expr *SE)
+        : Size(Sz), SizeExpr(SE) {}
+    llvm::APInt Size; // Allows us to unique the type.
+    const Expr *SizeExpr;
+  };
 
-  ConstantArrayType(QualType et, QualType can, const llvm::APInt &size,
-                    const Expr *sz, ArraySizeModifier sm, unsigned tq)
-      : ArrayType(ConstantArray, et, can, sm, tq, sz), Size(size) {
-    ConstantArrayTypeBits.HasStoredSizeExpr = sz != nullptr;
-    if (ConstantArrayTypeBits.HasStoredSizeExpr) {
-      assert(!can.isNull() && "canonical constant array should not have size");
-      *getTrailingObjects<const Expr*>() = sz;
-    }
+  union {
+    uint64_t Size;
+    ExternalSize *SizePtr;
+  };
+
+  ConstantArrayType(QualType Et, QualType Can, uint64_t Width, uint64_t Sz,
+                    ArraySizeModifier SM, unsigned TQ)
+      : ArrayType(ConstantArray, Et, Can, SM, TQ, nullptr), Size(Sz) {
+    ConstantArrayTypeBits.HasExternalSize = false;
+    ConstantArrayTypeBits.SizeWidth = Width / 8;
+    // The in-structure size stores the size in bytes rather than bits so we
+    // drop the three least significant bits since they're always zero anyways.
+    assert(Width < 0xFF && "Type width in bits must be less than 8 bits");
   }
 
-  unsigned numTrailingObjects(OverloadToken<const Expr*>) const {
-    return ConstantArrayTypeBits.HasStoredSizeExpr;
+  ConstantArrayType(QualType Et, QualType Can, ExternalSize *SzPtr,
+                    ArraySizeModifier SM, unsigned TQ)
+      : ArrayType(ConstantArray, Et, Can, SM, TQ, SzPtr->SizeExpr),
+        SizePtr(SzPtr) {
+    ConstantArrayTypeBits.HasExternalSize = true;
+    ConstantArrayTypeBits.SizeWidth = 0;
+
+    assert((SzPtr->SizeExpr == nullptr || !Can.isNull()) &&
+           "canonical constant array should not have size expression");
   }
 
+  static ConstantArrayType *Create(const ASTContext &Ctx, QualType ET,
+                                   QualType Can, const llvm::APInt &Sz,
+                                   const Expr *SzExpr, ArraySizeModifier SzMod,
+                                   unsigned Qual);
+
 public:
-  const llvm::APInt &getSize() const { return Size; }
+  /// Return the constant array size as an APInt.
+  llvm::APInt getSize() const {
+    return ConstantArrayTypeBits.HasExternalSize
+               ? SizePtr->Size
+               : llvm::APInt(ConstantArrayTypeBits.SizeWidth * 8, Size);
+  }
+
+  /// Return the bit width of the size type.
+  unsigned getSizeBitWidth() const {
+    return ConstantArrayTypeBits.HasExternalSize
+               ? SizePtr->Size.getBitWidth()
+               : static_cast<unsigned>(ConstantArrayTypeBits.SizeWidth * 8);
+  }
+
+  /// Return true if the size is zero.
+  bool isZeroSize() const {
+    return ConstantArrayTypeBits.HasExternalSize ? SizePtr->Size.isZero()
+                                                 : 0 == Size;
+  }
+
+  /// Return the size zero-extended as a uint64_t.
+  uint64_t getZExtSize() const {
+    return ConstantArrayTypeBits.HasExternalSize ? SizePtr->Size.getZExtValue()
+                                                 : Size;
+  }
+
+  /// Return the size sign-extended as a uint64_t.
+  int64_t getSExtSize() const {
+    return ConstantArrayTypeBits.HasExternalSize ? SizePtr->Size.getSExtValue()
+                                                 : static_cast<int64_t>(Size);
+  }
+
+  /// Return the size zero-extended to uint64_t or UINT64_MAX if the value is
+  /// larger than UINT64_MAX.
+  uint64_t getLimitedSize() const {
+    return ConstantArrayTypeBits.HasExternalSize
+               ? SizePtr->Size.getLimitedValue()
+               : Size;
+  }
+
+  /// Return a pointer to the size expression.
   const Expr *getSizeExpr() const {
-    return ConstantArrayTypeBits.HasStoredSizeExpr
-               ? *getTrailingObjects<const Expr *>()
-               : nullptr;
+    return ConstantArrayTypeBits.HasExternalSize ? SizePtr->SizeExpr : nullptr;
   }
+
   bool isSugared() const { return false; }
   QualType desugar() const { return QualType(this, 0); }
 
@@ -3383,14 +3444,13 @@ public:
   static unsigned getMaxSizeBits(const ASTContext &Context);
 
   void Profile(llvm::FoldingSetNodeID &ID, const ASTContext &Ctx) {
-    Profile(ID, Ctx, getElementType(), getSize(), getSizeExpr(),
+    Profile(ID, Ctx, getElementType(), getZExtSize(), getSizeExpr(),
             getSizeModifier(), getIndexTypeCVRQualifiers());
   }
 
   static void Profile(llvm::FoldingSetNodeID &ID, const ASTContext &Ctx,
-                      QualType ET, const llvm::APInt &ArraySize,
-                      const Expr *SizeExpr, ArraySizeModifier SizeMod,
-                      unsigned TypeQuals);
+                      QualType ET, uint64_t ArraySize, const Expr *SizeExpr,
+                      ArraySizeModifier SizeMod, unsigned TypeQuals);
 
   static bool classof(const Type *T) {
     return T->getTypeClass() == ConstantArray;
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 0b5f20a572a7..20a5ecc99e44 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -1766,7 +1766,7 @@ TypeInfoChars
 static getConstantArrayInfoInChars(const ASTContext &Context,
                                    const ConstantArrayType *CAT) {
   TypeInfoChars EltInfo = Context.getTypeInfoInChars(CAT->getElementType());
-  uint64_t Size = CAT->getSize().getZExtValue();
+  uint64_t Size = CAT->getZExtSize();
   assert((Size == 0 || static_cast<uint64_t>(EltInfo.Width.getQuantity()) <=
               (uint64_t)(-1)/Size) &&
          "Overflow in array type char size evaluation");
@@ -1910,7 +1910,7 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
     // Model non-constant sized arrays as size zero, but track the alignment.
     uint64_t Size = 0;
     if (const auto *CAT = dyn_cast<ConstantArrayType>(T))
-      Size = CAT->getSize().getZExtValue();
+      Size = CAT->getZExtSize();
 
     TypeInfo EltInfo = getTypeInfo(cast<ArrayType>(T)->getElementType());
     assert((Size == 0 || EltInfo.Width <= (uint64_t)(-1) / Size) &&
@@ -3560,8 +3560,8 @@ QualType ASTContext::getConstantArrayType(QualType EltTy,
   ArySize = ArySize.zextOrTrunc(Target->getMaxPointerWidth());
 
   llvm::FoldingSetNodeID ID;
-  ConstantArrayType::Profile(ID, *this, EltTy, ArySize, SizeExpr, ASM,
-                             IndexTypeQuals);
+  ConstantArrayType::Profile(ID, *this, EltTy, ArySize.getZExtValue(), SizeExpr,
+                             ASM, IndexTypeQuals);
 
   void *InsertPos = nullptr;
   if (ConstantArrayType *ATP =
@@ -3585,11 +3585,8 @@ QualType ASTContext::getConstantArrayType(QualType EltTy,
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
 
-  void *Mem = Allocate(
-      ConstantArrayType::totalSizeToAlloc<const Expr *>(SizeExpr ? 1 : 0),
-      alignof(ConstantArrayType));
-  auto *New = new (Mem)
-    ConstantArrayType(EltTy, Canon, ArySize, SizeExpr, ASM, IndexTypeQuals);
+  auto *New = ConstantArrayType::Create(*this, EltTy, Canon, ArySize, SizeExpr,
+                                        ASM, IndexTypeQuals);
   ConstantArrayTypes.InsertNode(New, InsertPos);
   Types.push_back(New);
   return QualType(New, 0);
@@ -7051,7 +7048,7 @@ uint64_t
 ASTContext::getConstantArrayElementCount(const ConstantArrayType *CA)  const {
   uint64_t ElementCount = 1;
   do {
-    ElementCount *= CA->getSize().getZExtValue();
+    ElementCount *= CA->getZExtSize();
     CA = dyn_cast_or_null<ConstantArrayType>(
       CA->getElementType()->getAsArrayTypeUnsafe());
   } while (CA);
@@ -8374,7 +8371,7 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string &S,
       S += '[';
 
       if (const auto *CAT = dyn_cast<ConstantArrayType>(AT))
-        S += llvm::utostr(CAT->getSize().getZExtValue());
+        S += llvm::utostr(CAT->getZExtSize());
       else {
         //Variable length arrays are encoded as a regular array with 0 elements.
         assert((isa<VariableArrayType>(AT) || isa<IncompleteArrayType>(AT)) &&
@@ -10808,7 +10805,7 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS, bool OfBlockPointer,
   {
     const ConstantArrayType* LCAT = getAsConstantArrayType(LHS);
     const ConstantArrayType* RCAT = getAsConstantArrayType(RHS);
-    if (LCAT && RCAT && RCAT->getSize() != LCAT->getSize())
+    if (LCAT && RCAT && RCAT->getZExtSize() != LCAT->getZExtSize())
       return {};
 
     QualType LHSElem = getAsArrayType(LHS)->getElementType();
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index 95900afdd2c5..131f82985e90 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -2840,7 +2840,7 @@ bool VarDecl::hasFlexibleArrayInit(const ASTContext &Ctx) const {
   auto InitTy = Ctx.getAsConstantArrayType(FlexibleInit->getType());
   if (!InitTy)
     return false;
-  return InitTy->getSize() != 0;
+  return !InitTy->isZeroSize();
 }
 
 CharUnits VarDecl::getFlexibleArrayInitChars(const ASTContext &Ctx) const {
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 592d43597dc1..c5bdff0df06b 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -209,7 +209,7 @@ namespace {
         IsArray = true;
 
         if (auto *CAT = dyn_cast<ConstantArrayType>(AT)) {
-          ArraySize = CAT->getSize().getZExtValue();
+          ArraySize = CAT->getZExtSize();
         } else {
           assert(I == 0 && "unexpected unsized array designator");
           FirstEntryIsUnsizedArray = true;
@@ -401,7 +401,7 @@ namespace {
       // This is a most-derived object.
       MostDerivedType = CAT->getElementType();
       MostDerivedIsArrayElement = true;
-      MostDerivedArraySize = CAT->getSize().getZExtValue();
+      MostDerivedArraySize = CAT->getZExtSize();
       MostDerivedPathLength = Entries.size();
     }
     /// Update this designator to refer to the first element within the array of
@@ -3476,7 +3476,7 @@ static void expandStringLiteral(EvalInfo &Info, const StringLiteral *S,
   QualType CharType = CAT->getElementType();
   assert(CharType->isIntegerType() && "unexpected character type");
 
-  unsigned Elts = CAT->getSize().getZExtValue();
+  unsigned Elts = CAT->getZExtSize();
   Result = APValue(APValue::UninitArray(),
                    std::min(S->getLength(), Elts), Elts);
   APSInt Value(Info.Ctx.getTypeSize(CharType),
@@ -3619,7 +3619,7 @@ static bool CheckArraySize(EvalInfo &Info, const ConstantArrayType *CAT,
                            SourceLocation CallLoc = {}) {
   return Info.CheckArraySize(
       CAT->getSizeExpr() ? CAT->getSizeExpr()->getBeginLoc() : CallLoc,
-      CAT->getNumAddressingBits(Info.Ctx), CAT->getSize().getZExtValue(),
+      CAT->getNumAddressingBits(Info.Ctx), CAT->getZExtSize(),
       /*Diag=*/true);
 }
 
@@ -4908,7 +4908,7 @@ static bool handleDefaultInitValue(QualType T, APValue &Result) {
 
   if (auto *AT =
           dyn_cast_or_null<ConstantArrayType>(T->getAsArrayTypeUnsafe())) {
-    Result = APValue(APValue::UninitArray(), 0, AT->getSize().getZExtValue());
+    Result = APValue(APValue::UninitArray(), 0, AT->getZExtSize());
     if (Result.hasArrayFiller())
       Success &=
           handleDefaultInitValue(AT->getElementType(), Result.getArrayFiller());
@@ -6595,7 +6595,7 @@ static bool HandleDestructionImpl(EvalInfo &Info, SourceRange CallRange,
 
   // For arrays, destroy elements right-to-left.
   if (const ConstantArrayType *CAT = Info.Ctx.getAsConstantArrayType(T)) {
-    uint64_t Size = CAT->getSize().getZExtValue();
+    uint64_t Size = CAT->getZExtSize();
     QualType ElemT = CAT->getElementType();
 
     if (!CheckArraySize(Info, CAT, CallRange.getBegin()))
@@ -7396,7 +7396,7 @@ class BufferToAPValueConverter {
   }
 
   std::optional<APValue> visit(const ConstantArrayType *Ty, CharUnits Offset) {
-    size_t Size = Ty->getSize().getLimitedValue();
+    size_t Size = Ty->getLimitedSize();
     CharUnits ElementWidth = Info.Ctx.getTypeSizeInChars(Ty->getElementType());
 
     APValue ArrayValue(APValue::UninitArray(), Size, Size);
@@ -9951,7 +9951,7 @@ bool PointerExprEvaluator::VisitCXXNewExpr(const CXXNewExpr *E) {
       assert(CAT && "unexpected type for array initializer");
 
       unsigned Bits =
-          std::max(CAT->getSize().getBitWidth(), ArrayBound.getBitWidth());
+          std::max(CAT->getSizeBitWidth(), ArrayBound.getBitWidth());
       llvm::APInt InitBound = CAT->getSize().zext(Bits);
       llvm::APInt AllocBound = ArrayBound.zext(Bits);
       if (InitBound.ugt(AllocBound)) {
@@ -10410,7 +10410,7 @@ bool RecordExprEvaluator::VisitCXXParenListOrInitListExpr(
 
     if (Field->getType()->isIncompleteArrayType()) {
       if (auto *CAT = Info.Ctx.getAsConstantArrayType(Init->getType())) {
-        if (!CAT->getSize().isZero()) {
+        if (!CAT->isZeroSize()) {
           // Bail out for now. This might sort of "work", but the rest of the
           // code isn't really prepared to handle it.
           Info.FFDiag(Init, diag::note_constexpr_unsupported_flexible_array);
@@ -10554,7 +10554,7 @@ bool RecordExprEvaluator::VisitCXXStdInitializerListExpr(
     // End pointer.
     if (!HandleLValueArrayAdjustment(Info, E, Array,
                                      ArrayType->getElementType(),
-                                     ArrayType->getSize().getZExtValue()))
+                                     ArrayType->getZExtSize()))
       return false;
     Array.moveInto(Result.getStructField(1));
   } else if (Info.Ctx.hasSameType(Field->getType(), Info.Ctx.getSizeType()))
@@ -10996,8 +10996,7 @@ namespace {
         return Error(E);
       }
 
-      Result = APValue(APValue::UninitArray(), 0,
-                       CAT->getSize().getZExtValue());
+      Result = APValue(APValue::UninitArray(), 0, CAT->getZExtSize());
       if (!Result.hasArrayFiller())
         return true;
 
@@ -11122,7 +11121,7 @@ bool ArrayExprEvaluator::VisitCXXParenListOrInitListExpr(
     Filler = Result.getArrayFiller();
 
   unsigned NumEltsToInit = Args.size();
-  unsigned NumElts = CAT->getSize().getZExtValue();
+  unsigned NumElts = CAT->getZExtSize();
 
   // If the initializer might depend on the array index, run it for each
   // array element.
@@ -11180,7 +11179,7 @@ bool ArrayExprEvaluator::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E) {
 
   auto *CAT = cast<ConstantArrayType>(E->getType()->castAsArrayTypeUnsafe());
 
-  uint64_t Elements = CAT->getSize().getZExtValue();
+  uint64_t Elements = CAT->getZExtSize();
   Result = APValue(APValue::UninitArray(), Elements, Elements);
 
   LValue Subobject = This;
@@ -11225,7 +11224,7 @@ bool ArrayExprEvaluator::VisitCXXConstructExpr(const CXXConstructExpr *E,
   bool HadZeroInit = Value->hasValue();
 
   if (const ConstantArrayType *CAT = Info.Ctx.getAsConstantArrayType(Type)) {
-    unsigned FinalSize = CAT->getSize().getZExtValue();
+    unsigned FinalSize = CAT->getZExtSize();
 
     // Preserve the array filler if we had prior zero-initialization.
     APValue Filler =
@@ -11940,7 +11939,7 @@ static bool isDesignatorAtObjectEnd(const ASTContext &Ctx, const LValue &LVal) {
         return true;
       const auto *CAT = cast<ConstantArrayType>(Ctx.getAsArrayType(BaseType));
       uint64_t Index = Entry.getAsArrayIndex();
-      if (Index + 1 != CAT->getSize())
+      if (Index + 1 != CAT->getZExtSize())
         return false;
       BaseType = CAT->getElementType();
     } else if (BaseType->isAnyComplexType()) {
diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
index 73831eefba45..46182809810b 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
@@ -819,7 +819,7 @@ bool ByteCodeExprGen<Emitter>::VisitImplicitValueInitExpr(const ImplicitValueIni
     const ArrayType *AT = QT->getAsArrayTypeUnsafe();
     assert(AT);
     const auto *CAT = cast<ConstantArrayType>(AT);
-    size_t NumElems = CAT->getSize().getZExtValue();
+    size_t NumElems = CAT->getZExtSize();
     PrimType ElemT = classifyPrim(CAT->getElementType());
 
     for (size_t I = 0; I != NumElems; ++I) {
@@ -992,7 +992,7 @@ bool ByteCodeExprGen<Emitter>::VisitInitListExpr(const InitListExpr *E) {
     if (const Expr *Filler = E->getArrayFiller()) {
       const ConstantArrayType *CAT =
           Ctx.getASTContext().getAsConstantArrayType(E->getType());
-      uint64_t NumElems = CAT->getSize().getZExtValue();
+      uint64_t NumElems = CAT->getZExtSize();
 
       for (; ElementIndex != NumElems; ++ElementIndex) {
         if (!this->visitArrayElemInit(ElementIndex, Filler))
@@ -1318,7 +1318,7 @@ bool ByteCodeExprGen<Emitter>::VisitStringLiteral(const StringLiteral *E) {
 
   // If the initializer string is too long, a diagnostic has already been
   // emitted. Read only the array length from the string literal.
-  unsigned ArraySize = CAT->getSize().getZExtValue();
+  unsigned ArraySize = CAT->getZExtSize();
   unsigned N = std::min(ArraySize, E->getLength());
   size_t CharWidth = E->getCharByteWidth();
 
@@ -1919,7 +1919,7 @@ bool ByteCodeExprGen<Emitter>::VisitCXXConstructExpr(
     const ConstantArrayType *CAT =
         Ctx.getASTContext().getAsConstantArrayType(E->getType());
     assert(CAT);
-    size_t NumElems = CAT->getSize().getZExtValue();
+    size_t NumElems = CAT->getZExtSize();
     const Function *Func = getFunction(E->getConstructor());
     if (!Func || !Func->isConstexpr())
       return false;
diff --git a/clang/lib/AST/Interp/EvaluationResult.cpp b/clang/lib/AST/Interp/EvaluationResult.cpp
index 07b28d07326f..d567b551f7f6 100644
--- a/clang/lib/AST/Interp/EvaluationResult.cpp
+++ b/clang/lib/AST/Interp/EvaluationResult.cpp
@@ -66,7 +66,7 @@ static bool CheckArrayInitialized(InterpState &S, SourceLocation Loc,
                                   const Pointer &BasePtr,
                                   const ConstantArrayType *CAT) {
   bool Result = true;
-  size_t NumElems = CAT->getSize().getZExtValue();
+  size_t NumElems = CAT->getZExtSize();
   QualType ElemType = CAT->getElementType();
 
   if (ElemType->isRecordType()) {
diff --git a/clang/lib/AST/Interp/Program.cpp b/clang/lib/AST/Interp/Program.cpp
index da6f72c62115..25e938e01503 100644
--- a/clang/lib/AST/Interp/Program.cpp
+++ b/clang/lib/AST/Interp/Program.cpp
@@ -355,7 +355,7 @@ Descriptor *Program::createDescriptor(const DeclTy &D, const Type *Ty,
     QualType ElemTy = ArrayType->getElementType();
     // Array of well-known bounds.
     if (auto CAT = dyn_cast<ConstantArrayType>(ArrayType)) {
-      size_t NumElems = CAT->getSize().getZExtValue();
+      size_t NumElems = CAT->getZExtSize();
       if (std::optional<PrimType> T = Ctx.classify(ElemTy)) {
         // Arrays of primitives.
         unsigned ElemSize = primSize(*T);
diff --git a/clang/lib/AST/JSONNodeDumper.cpp b/clang/lib/AST/JSONNodeDumper.cpp
index e27d44fc2ffe..5861d5a7ea0d 100644
--- a/clang/lib/AST/JSONNodeDumper.cpp
+++ b/clang/lib/AST/JSONNodeDumper.cpp
@@ -695,7 +695,7 @@ void JSONNodeDumper::VisitArrayType(const ArrayType *AT) {
 void JSONNodeDumper::VisitConstantArrayType(const ConstantArrayType *CAT) {
   // FIXME: this should use ZExt instead of SExt, but JSON doesn't allow a
   // narrowing conversion to int64_t so it cannot be expressed.
-  JOS.attribute("size", CAT->getSize().getSExtValue());
+  JOS.attribute("size", CAT->getSExtSize());
   VisitArrayType(CAT);
 }
 
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index cf9c2093a8f6..addc3140546a 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -4023,10 +4023,8 @@ void MicrosoftMangleContextImpl::mangleStringLiteral(const StringLiteral *SL,
   // char bar[42] = "foobar";
   // Where it is truncated or zero-padded to fit the array. This is the length
   // used for mangling, and any trailing null-bytes also need to be mangled.
-  unsigned StringLength = getASTContext()
-                              .getAsConstantArrayType(SL->getType())
-                              ->getSize()
-                              .getZExtValue();
+  unsigned StringLength =
+      getASTContext().getAsConstantArrayType(SL->getType())->getZExtSize();
   unsigned StringByteLength = StringLength * SL->getCharByteWidth();
 
   // <char-type>: The "kind" of string literal is encoded into the mangled name.
diff --git a/clang/lib/AST/ScanfFormatString.cpp b/clang/lib/AST/ScanfFormatString.cpp
index 64c430e623b5..7ee21c8c6195 100644
--- a/clang/lib/AST/ScanfFormatString.cpp
+++ b/clang/lib/AST/ScanfFormatString.cpp
@@ -448,9 +448,7 @@ bool ScanfSpecifier::fixType(QualType QT, QualType RawQT,
     if (const ConstantArrayType *CAT = Ctx.getAsConstantArrayType(RawQT)) {
       if (CAT->getSizeModifier() == ArraySizeModifier::Normal)
         FieldWidth = OptionalAmount(OptionalAmount::Constant,
-                                    CAT->getSize().getZExtValue() - 1,
-                                    "", 0, false);
-
+                                    CAT->getZExtSize() - 1, "", 0, false);
     }
     return true;
   }
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index c6fe90ba5e29..d2ffb23845ac 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -159,6 +159,22 @@ ArrayType::ArrayType(TypeClass tc, QualType et, QualType can,
   ArrayTypeBits.SizeModifier = llvm::to_underlying(sm);
 }
 
+ConstantArrayType *
+ConstantArrayType::Create(const ASTContext &Ctx, QualType ET, QualType Can,
+                          const llvm::APInt &Sz, const Expr *SzExpr,
+                          ArraySizeModifier SzMod, unsigned Qual) {
+  bool NeedsExternalSize = SzExpr != nullptr || Sz.ugt(0x0FFFFFFFFFFFFFFF) ||
+                           Sz.getBitWidth() > 0xFF;
+  if (!NeedsExternalSize)
+    return new (Ctx, alignof(ConstantArrayType)) ConstantArrayType(
+        ET, Can, Sz.getBitWidth(), Sz.getZExtValue(), SzMod, Qual);
+
+  auto *SzPtr = new (Ctx, alignof(ConstantArrayType::ExternalSize))
+      ConstantArrayType::ExternalSize(Sz, SzExpr);
+  return new (Ctx, alignof(ConstantArrayType))
+      ConstantArrayType(ET, Can, SzPtr, SzMod, Qual);
+}
+
 unsigned ConstantArrayType::getNumAddressingBits(const ASTContext &Context,
                                                  QualType ElementType,
                                                const llvm::APInt &NumElements) {
@@ -213,11 +229,10 @@ unsigned ConstantArrayType::getMaxSizeBits(const ASTContext &Context) {
 
 void ConstantArrayType::Profile(llvm::FoldingSetNodeID &ID,
                                 const ASTContext &Context, QualType ET,
-                                const llvm::APInt &ArraySize,
-                                const Expr *SizeExpr, ArraySizeModifier SizeMod,
-                                unsigned TypeQuals) {
+                                uint64_t ArraySize, const Expr *SizeExpr,
+                                ArraySizeModifier SizeMod, unsigned TypeQuals) {
   ID.AddPointer(ET.getAsOpaquePtr());
-  ID.AddInteger(ArraySize.getZExtValue());
+  ID.AddInteger(ArraySize);
   ID.AddInteger(llvm::to_underlying(SizeMod));
   ID.AddInteger(TypeQuals);
   ID.AddBoolean(SizeExpr != nullptr);
@@ -452,12 +467,8 @@ QualType QualType::getSingleStepDesugaredTypeImpl(QualType type,
 // Check that no type class has a non-trival destructor. Types are
 // allocated with the BumpPtrAllocator from ASTContext and therefore
 // their destructor is not executed.
-//
-// FIXME: ConstantArrayType is not trivially destructible because of its
-// APInt member. It should be replaced in favor of ASTContext allocation.
 #define TYPE(CLASS, BASE)                                                      \
-  static_assert(std::is_trivially_destructible<CLASS##Type>::value ||          \
-                    std::is_same<CLASS##Type, ConstantArrayType>::value,       \
+  static_assert(std::is_trivially_destructible<CLASS##Type>::value,            \
                 #CLASS "Type should be trivially destructible!");
 #include "clang/AST/TypeNodes.inc"
 
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index d0ba6066fa01..f176d043d525 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -538,7 +538,7 @@ void TypePrinter::printConstantArrayAfter(const ConstantArrayType *T,
   if (T->getSizeModifier() == ArraySizeModifier::Static)
     OS << "static ";
 
-  OS << T->getSize().getZExtValue() << ']';
+  OS << T->getZExtSize() << ']';
   printAfter(T->getElementType(), OS);
 }
 
diff --git a/clang/lib/Analysis/CFG.cpp b/clang/lib/Analysis/CFG.cpp
index de70cbbf6cdb..64e6155de090 100644
--- a/clang/lib/Analysis/CFG.cpp
+++ b/clang/lib/Analysis/CFG.cpp
@@ -2039,7 +2039,7 @@ void CFGBuilder::addImplicitDtorsForDestructor(const CXXDestructorDecl *DD) {
     QualType QT = FI->getType();
     // It may be a multidimensional array.
     while (const ConstantArrayType *AT = Context->getAsConstantArrayType(QT)) {
-      if (AT->getSize() == 0)
+      if (AT->isZeroSize())
         break;
       QT = AT->getElementType();
     }
@@ -2133,7 +2133,7 @@ bool CFGBuilder::hasTrivialDestructor(const VarDecl *VD) const {
 
   // Check for constant size array. Set type to array element type.
   while (const ConstantArrayType *AT = Context->getAsConstantArrayType(QT)) {
-    if (AT->getSize() == 0)
+    if (AT->isZeroSize())
       return true;
     QT = AT->getElementType();
   }
diff --git a/clang/lib/Analysis/UnsafeBufferUsage.cpp b/clang/lib/Analysis/UnsafeBufferUsage.cpp
index e1ff0d92f6b2..e03fe1b68300 100644
--- a/clang/lib/Analysis/UnsafeBufferUsage.cpp
+++ b/clang/lib/Analysis/UnsafeBufferUsage.cpp
@@ -403,10 +403,11 @@ AST_MATCHER(CXXConstructExpr, isSafeSpanTwoParamConstruct) {
   QualType Arg0Ty = Arg0->IgnoreImplicit()->getType();
 
   if (Arg0Ty->isConstantArrayType()) {
-    const APInt &ConstArrSize = cast<ConstantArrayType>(Arg0Ty)->getSize();
+    const APSInt ConstArrSize =
+        APSInt(cast<ConstantArrayType>(Arg0Ty)->getSize());
 
     // Check form 4:
-    return Arg1CV && APSInt::compareValues(APSInt(ConstArrSize), *Arg1CV) == 0;
+    return Arg1CV && APSInt::compareValues(ConstArrSize, *Arg1CV) == 0;
   }
   return false;
 }
@@ -429,14 +430,13 @@ AST_MATCHER(ArraySubscriptExpr, isSafeArraySubscript) {
       BaseDRE->getDecl()->getType());
   if (!CATy)
     return false;
-  const APInt ArrSize = CATy->getSize();
 
   if (const auto *IdxLit = dyn_cast<IntegerLiteral>(Node.getIdx())) {
     const APInt ArrIdx = IdxLit->getValue();
     // FIXME: ArrIdx.isNegative() we could immediately emit an error as that's a
     // bug
     if (ArrIdx.isNonNegative() &&
-        ArrIdx.getLimitedValue() < ArrSize.getLimitedValue())
+        ArrIdx.getLimitedValue() < CATy->getLimitedSize())
       return true;
   }
 
diff --git a/clang/lib/CodeGen/ABIInfo.cpp b/clang/lib/CodeGen/ABIInfo.cpp
index efcff958ce54..acaae9f8c3d8 100644
--- a/clang/lib/CodeGen/ABIInfo.cpp
+++ b/clang/lib/CodeGen/ABIInfo.cpp
@@ -61,7 +61,7 @@ bool ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate() const {
 bool ABIInfo::isHomogeneousAggregate(QualType Ty, const Type *&Base,
                                      uint64_t &Members) const {
   if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
-    uint64_t NElements = AT->getSize().getZExtValue();
+    uint64_t NElements = AT->getZExtSize();
     if (NElements == 0)
       return false;
     if (!isHomogeneousAggregate(AT->getElementType(), Base, Members))
@@ -98,7 +98,7 @@ bool ABIInfo::isHomogeneousAggregate(QualType Ty, const Type *&Base,
       QualType FT = FD->getType();
       while (const ConstantArrayType *AT =
              getContext().getAsConstantArrayType(FT)) {
-        if (AT->getSize().getZExtValue() == 0)
+        if (AT->isZeroSize())
           return false;
         FT = AT->getElementType();
       }
diff --git a/clang/lib/CodeGen/ABIInfoImpl.cpp b/clang/lib/CodeGen/ABIInfoImpl.cpp
index 2b20d5a13346..dd59101ecc81 100644
--- a/clang/lib/CodeGen/ABIInfoImpl.cpp
+++ b/clang/lib/CodeGen/ABIInfoImpl.cpp
@@ -257,7 +257,7 @@ bool CodeGen::isEmptyField(ASTContext &Context, const FieldDecl *FD,
   bool WasArray = false;
   if (AllowArrays)
     while (const ConstantArrayType *AT = Context.getAsConstantArrayType(FT)) {
-      if (AT->getSize() == 0)
+      if (AT->isZeroSize())
         return true;
       FT = AT->getElementType();
       // The [[no_unique_address]] special case below does not apply to
@@ -352,7 +352,7 @@ const Type *CodeGen::isSingleElementStruct(QualType T, ASTContext &Context) {
 
     // Treat single element arrays as the element.
     while (const ConstantArrayType *AT = Context.getAsConstantArrayType(FT)) {
-      if (AT->getSize().getZExtValue() != 1)
+      if (AT->getZExtSize() != 1)
         break;
       FT = AT->getElementType();
     }
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index a28d7888715d..475d96b0e87d 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -933,8 +933,8 @@ struct NoExpansion : TypeExpansion {
 static std::unique_ptr<TypeExpansion>
 getTypeExpansion(QualType Ty, const ASTContext &Context) {
   if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
-    return std::make_unique<ConstantArrayExpansion>(
-        AT->getElementType(), AT->getSize().getZExtValue());
+    return std::make_unique<ConstantArrayExpansion>(AT->getElementType(),
+                                                    AT->getZExtSize());
   }
   if (const RecordType *RT = Ty->getAs<RecordType>()) {
     SmallVector<const CXXBaseSpecifier *, 1> Bases;
@@ -3086,7 +3086,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
               llvm::Align Alignment =
                   CGM.getNaturalTypeAlignment(ETy).getAsAlign();
               AI->addAttrs(llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr(Alignment));
-              uint64_t ArrSize = ArrTy->getSize().getZExtValue();
+              uint64_t ArrSize = ArrTy->getZExtSize();
               if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
                   ArrSize) {
                 llvm::AttrBuilder Attrs(getLLVMContext());
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 7453ed14aef4..0e20de2005b2 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -3239,7 +3239,7 @@ llvm::DIType *CGDebugInfo::CreateType(const ArrayType *Ty, llvm::DIFile *Unit) {
     //   };
     int64_t Count = -1; // Count == -1 is an unbounded array.
     if (const auto *CAT = dyn_cast<ConstantArrayType>(Ty))
-      Count = CAT->getSize().getZExtValue();
+      Count = CAT->getZExtSize();
     else if (const auto *VAT = dyn_cast<VariableArrayType>(Ty)) {
       if (Expr *Size = VAT->getSizeExpr()) {
         Expr::EvalResult Result;
diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp
index 2adbef6d5512..35da0f1a89bc 100644
--- a/clang/lib/CodeGen/CGExprCXX.cpp
+++ b/clang/lib/CodeGen/CGExprCXX.cpp
@@ -1073,8 +1073,7 @@ void CodeGenFunction::EmitNewArrayInitializer(
       // Move past these elements.
       InitListElements =
           cast<ConstantArrayType>(Init->getType()->getAsArrayTypeUnsafe())
-              ->getSize()
-              .getZExtValue();
+              ->getZExtSize();
       CurPtr = Builder.CreateConstInBoundsGEP(
           CurPtr, InitListElements, "string.init.end");
 
@@ -1591,8 +1590,7 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
         isa<StringLiteral>(IgnoreParen) || isa<ObjCEncodeExpr>(IgnoreParen)) {
       minElements =
           cast<ConstantArrayType>(Init->getType()->getAsArrayTypeUnsafe())
-              ->getSize()
-              .getZExtValue();
+              ->getZExtSize();
     } else if (ILE || CPLIE) {
       minElements = ILE ? ILE->getNumInits() : CPLIE->getInitExprs().size();
     }
diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index 75286dceb13a..67a3cdc77042 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -656,7 +656,7 @@ static bool EmitDesignatedInitUpdater(ConstantEmitter &Emitter,
   }
 
   unsigned NumElementsToUpdate =
-      FillC ? CAT->getSize().getZExtValue() : Updater->getNumInits();
+      FillC ? CAT->getZExtSize() : Updater->getNumInits();
   for (unsigned I = 0; I != NumElementsToUpdate; ++I, Offset += ElemSize) {
     Expr *Init = nullptr;
     if (I < Updater->getNumInits())
@@ -1249,7 +1249,7 @@ public:
     auto *CAT = CGM.getContext().getAsConstantArrayType(ILE->getType());
     assert(CAT && "can't emit array init for non-constant-bound array");
     unsigned NumInitElements = ILE->getNumInits();
-    unsigned NumElements = CAT->getSize().getZExtValue();
+    unsigned NumElements = CAT->getZExtSize();
 
     // Initialising an array requires us to automatically
     // initialise any elements that have not been initialised explicitly
@@ -1374,7 +1374,7 @@ public:
 
     // Resize the string to the right size, adding zeros at the end, or
     // truncating as needed.
-    Str.resize(CAT->getSize().getZExtValue(), '\0');
+    Str.resize(CAT->getZExtSize(), '\0');
     return llvm::ConstantDataArray::getString(VMContext, Str, false);
   }
 
@@ -2382,7 +2382,7 @@ llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) {
 
     llvm::Constant *Element =
       ConstantEmitter::emitNullForMemory(*this, ElementTy);
-    unsigned NumElements = CAT->getSize().getZExtValue();
+    unsigned NumElements = CAT->getZExtSize();
     SmallVector<llvm::Constant *, 8> Array(NumElements, Element);
     return llvm::ConstantArray::get(ATy, Array);
   }
diff --git a/clang/lib/CodeGen/CGObjCMac.cpp b/clang/lib/CodeGen/CGObjCMac.cpp
index e815e097e1fb..ed8d7b9a065d 100644
--- a/clang/lib/CodeGen/CGObjCMac.cpp
+++ b/clang/lib/CodeGen/CGObjCMac.cpp
@@ -2501,12 +2501,12 @@ void CGObjCCommonMac::BuildRCRecordLayout(const llvm::StructLayout *RecLayout,
 
     if (const ArrayType *Array = CGM.getContext().getAsArrayType(FQT)) {
       auto *CArray = cast<ConstantArrayType>(Array);
-      uint64_t ElCount = CArray->getSize().getZExtValue();
+      uint64_t ElCount = CArray->getZExtSize();
       assert(CArray && "only array with known element size is supported");
       FQT = CArray->getElementType();
       while (const ArrayType *Array = CGM.getContext().getAsArrayType(FQT)) {
         auto *CArray = cast<ConstantArrayType>(Array);
-        ElCount *= CArray->getSize().getZExtValue();
+        ElCount *= CArray->getZExtSize();
         FQT = CArray->getElementType();
       }
       if (FQT->isRecordType() && ElCount) {
@@ -5326,7 +5326,7 @@ void IvarLayoutBuilder::visitField(const FieldDecl *field,
   }
   // Unlike incomplete arrays, constant arrays can be nested.
   while (auto arrayType = CGM.getContext().getAsConstantArrayType(fieldType)) {
-    numElts *= arrayType->getSize().getZExtValue();
+    numElts *= arrayType->getZExtSize();
     fieldType = arrayType->getElementType();
   }
 
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index e8a68dbcc687..00e395c2a207 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -6796,7 +6796,7 @@ private:
                              OASE->getBase()->IgnoreParenImpCasts())
                              .getCanonicalType();
       if (const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.getTypePtr()))
-        return ATy->getSize().getSExtValue() != 1;
+        return ATy->getSExtSize() != 1;
       // If we don't have a constant dimension length, we have to consider
       // the current section as having any size, so it is not necessarily
       // unitary. If it happen to be unity size, that's user fault.
@@ -7546,8 +7546,8 @@ private:
       // it.
       if (DimSizes.size() < Components.size() - 1) {
         if (CAT)
-          DimSizes.push_back(llvm::ConstantInt::get(
-              CGF.Int64Ty, CAT->getSize().getZExtValue()));
+          DimSizes.push_back(
+              llvm::ConstantInt::get(CGF.Int64Ty, CAT->getZExtSize()));
         else if (VAT)
           DimSizes.push_back(CGF.Builder.CreateIntCast(
               CGF.EmitScalarExpr(VAT->getSizeExpr()), CGF.Int64Ty,
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index fad26c43da3d..f2ebaf767452 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -2189,8 +2189,8 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
     dyn_cast<llvm::ArrayType>(addr.getElementType());
   while (llvmArrayType) {
     assert(isa<ConstantArrayType>(arrayType));
-    assert(cast<ConstantArrayType>(arrayType)->getSize().getZExtValue()
-             == llvmArrayType->getNumElements());
+    assert(cast<ConstantArrayType>(arrayType)->getZExtSize() ==
+           llvmArrayType->getNumElements());
 
     gepIndices.push_back(zero);
     countFromCLAs *= llvmArrayType->getNumElements();
@@ -2208,8 +2208,7 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
     // as some other type (probably a packed struct). Compute the array
     // size, and just emit the 'begin' expression as a bitcast.
     while (arrayType) {
-      countFromCLAs *=
-          cast<ConstantArrayType>(arrayType)->getSize().getZExtValue();
+      countFromCLAs *= cast<ConstantArrayType>(arrayType)->getZExtSize();
       eltType = arrayType->getElementType();
       arrayType = getContext().getAsArrayType(eltType);
     }
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index bc7d7ac56111..e3ed5e90f2d3 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -6281,7 +6281,7 @@ CodeGenModule::GetConstantArrayFromStringLiteral(const StringLiteral *E) {
     // Resize the string to the right size, which is indicated by its type.
     const ConstantArrayType *CAT = Context.getAsConstantArrayType(E->getType());
     assert(CAT && "String literal not of constant array type!");
-    Str.resize(CAT->getSize().getZExtValue());
+    Str.resize(CAT->getZExtSize());
     return llvm::ConstantDataArray::getString(VMContext, Str, false);
   }
 
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index a6b51bfef876..afadc29ab1b0 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -601,7 +601,7 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
       EltTy = llvm::Type::getInt8Ty(getLLVMContext());
     }
 
-    ResultType = llvm::ArrayType::get(EltTy, A->getSize().getZExtValue());
+    ResultType = llvm::ArrayType::get(EltTy, A->getZExtSize());
     break;
   }
   case Type::ExtVector:
diff --git a/clang/lib/CodeGen/SwiftCallingConv.cpp b/clang/lib/CodeGen/SwiftCallingConv.cpp
index 16fbf52a517d..ab2e2bd0b306 100644
--- a/clang/lib/CodeGen/SwiftCallingConv.cpp
+++ b/clang/lib/CodeGen/SwiftCallingConv.cpp
@@ -78,7 +78,7 @@ void SwiftAggLowering::addTypedData(QualType type, CharUnits begin) {
 
     QualType eltType = arrayType->getElementType();
     auto eltSize = CGM.getContext().getTypeSizeInChars(eltType);
-    for (uint64_t i = 0, e = arrayType->getSize().getZExtValue(); i != e; ++i) {
+    for (uint64_t i = 0, e = arrayType->getZExtSize(); i != e; ++i) {
       addTypedData(eltType, begin + i * eltSize);
     }
 
diff --git a/clang/lib/CodeGen/Targets/ARM.cpp b/clang/lib/CodeGen/Targets/ARM.cpp
index 5d42e6286e52..885d9c77d0e7 100644
--- a/clang/lib/CodeGen/Targets/ARM.cpp
+++ b/clang/lib/CodeGen/Targets/ARM.cpp
@@ -671,7 +671,7 @@ bool ARMABIInfo::isIllegalVectorType(QualType Ty) const {
 /// Return true if a type contains any 16-bit floating point vectors
 bool ARMABIInfo::containsAnyFP16Vectors(QualType Ty) const {
   if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
-    uint64_t NElements = AT->getSize().getZExtValue();
+    uint64_t NElements = AT->getZExtSize();
     if (NElements == 0)
       return false;
     return containsAnyFP16Vectors(AT->getElementType());
diff --git a/clang/lib/CodeGen/Targets/LoongArch.cpp b/clang/lib/CodeGen/Targets/LoongArch.cpp
index 63b9a1fdb988..3f01d9ad90f1 100644
--- a/clang/lib/CodeGen/Targets/LoongArch.cpp
+++ b/clang/lib/CodeGen/Targets/LoongArch.cpp
@@ -146,7 +146,7 @@ bool LoongArchABIInfo::detectFARsEligibleStructHelper(
   }
 
   if (const ConstantArrayType *ATy = getContext().getAsConstantArrayType(Ty)) {
-    uint64_t ArraySize = ATy->getSize().getZExtValue();
+    uint64_t ArraySize = ATy->getZExtSize();
     QualType EltTy = ATy->getElementType();
     // Non-zero-length arrays of empty records make the struct ineligible to be
     // passed via FARs in C++.
diff --git a/clang/lib/CodeGen/Targets/RISCV.cpp b/clang/lib/CodeGen/Targets/RISCV.cpp
index 9a79424c4612..7b32c7972356 100644
--- a/clang/lib/CodeGen/Targets/RISCV.cpp
+++ b/clang/lib/CodeGen/Targets/RISCV.cpp
@@ -152,7 +152,7 @@ bool RISCVABIInfo::detectFPCCEligibleStructHelper(QualType Ty, CharUnits CurOff,
   }
 
   if (const ConstantArrayType *ATy = getContext().getAsConstantArrayType(Ty)) {
-    uint64_t ArraySize = ATy->getSize().getZExtValue();
+    uint64_t ArraySize = ATy->getZExtSize();
     QualType EltTy = ATy->getElementType();
     // Non-zero-length arrays of empty records make the struct ineligible for
     // the FP calling convention in C++.
diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index 6931768147a9..1146a851a771 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -1993,7 +1993,7 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
     // this, but it isn't worth it and would be harder to verify.
     Current = NoClass;
     uint64_t EltSize = getContext().getTypeSize(AT->getElementType());
-    uint64_t ArraySize = AT->getSize().getZExtValue();
+    uint64_t ArraySize = AT->getZExtSize();
 
     // The only case a 256-bit wide vector could be used is when the array
     // contains a single 256-bit element. Since Lo and Hi logic isn't extended
@@ -2295,7 +2295,7 @@ static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
 
   if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
     unsigned EltSize = (unsigned)Context.getTypeSize(AT->getElementType());
-    unsigned NumElts = (unsigned)AT->getSize().getZExtValue();
+    unsigned NumElts = (unsigned)AT->getZExtSize();
 
     // Check each element to see if the element overlaps with the queried range.
     for (unsigned i = 0; i != NumElts; ++i) {
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index b4e87b618032..084495813309 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -1098,7 +1098,7 @@ static bool ProcessFormatStringLiteral(const Expr *FormatExpr,
     const ConstantArrayType *T =
         Context.getAsConstantArrayType(Format->getType());
     assert(T && "String literal not of constant array type!");
-    size_t TypeSize = T->getSize().getZExtValue();
+    size_t TypeSize = T->getZExtSize();
     // In case there's a null byte somewhere.
     StrLen = std::min(std::max(TypeSize, size_t(1)) - 1, FormatStrRef.find(0));
     return true;
@@ -13044,7 +13044,7 @@ static void CheckFormatString(
   const ConstantArrayType *T =
     S.Context.getAsConstantArrayType(FExpr->getType());
   assert(T && "String literal not of constant array type!");
-  size_t TypeSize = T->getSize().getZExtValue();
+  size_t TypeSize = T->getZExtSize();
   size_t StrLen = std::min(std::max(TypeSize, size_t(1)) - 1, StrRef.size());
   const unsigned numDataArgs = Args.size() - firstDataArg;
 
@@ -13104,7 +13104,7 @@ bool Sema::FormatStringHasSArg(const StringLiteral *FExpr) {
   // Account for cases where the string literal is truncated in a declaration.
   const ConstantArrayType *T = Context.getAsConstantArrayType(FExpr->getType());
   assert(T && "String literal not of constant array type!");
-  size_t TypeSize = T->getSize().getZExtValue();
+  size_t TypeSize = T->getZExtSize();
   size_t StrLen = std::min(std::max(TypeSize, size_t(1)) - 1, StrRef.size());
   return analyze_format_string::ParseFormatStringHasSArg(Str, Str + StrLen,
                                                          getLangOpts(),
@@ -14067,7 +14067,7 @@ static bool isConstantSizeArrayWithMoreThanOneElement(QualType Ty,
   // Only handle constant-sized or VLAs, but not flexible members.
   if (const ConstantArrayType *CAT = Context.getAsConstantArrayType(Ty)) {
     // Only issue the FIXIT for arrays of size > 1.
-    if (CAT->getSize().getSExtValue() <= 1)
+    if (CAT->getZExtSize() <= 1)
       return false;
   } else if (!Ty->isVariableArrayType()) {
     return false;
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index b907fee01d5f..66aad2592cb3 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -8837,7 +8837,7 @@ void Sema::CheckVariableDeclarationType(VarDecl *NewVD) {
       return;
     }
     const auto *ATy = dyn_cast<ConstantArrayType>(T.getTypePtr());
-    if (!ATy || ATy->getSize().getSExtValue() != 0) {
+    if (!ATy || ATy->getZExtSize() != 0) {
       Diag(NewVD->getLocation(),
            diag::err_typecheck_wasm_table_must_have_zero_length);
       NewVD->setInvalidDecl();
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index c63f2bdc5f83..e9fecaea84b0 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -5282,7 +5282,7 @@ static bool isIncompleteOrZeroLengthArrayType(ASTContext &Context, QualType T) {
     return true;
 
   while (const ConstantArrayType *ArrayT = Context.getAsConstantArrayType(T)) {
-    if (!ArrayT->getSize())
+    if (ArrayT->isZeroSize())
       return true;
 
     T = ArrayT->getElementType();
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 5f03b9814282..091fc3e4836b 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -6857,9 +6857,8 @@ Sema::CheckStaticArrayArgument(SourceLocation CallLoc,
                                              ArgCAT->getElementType())) {
     if (ArgCAT->getSize().ult(CAT->getSize())) {
       Diag(CallLoc, diag::warn_static_array_too_small)
-          << ArgExpr->getSourceRange()
-          << (unsigned)ArgCAT->getSize().getZExtValue()
-          << (unsigned)CAT->getSize().getZExtValue() << 0;
+          << ArgExpr->getSourceRange() << (unsigned)ArgCAT->getZExtSize()
+          << (unsigned)CAT->getZExtSize() << 0;
       DiagnoseCalleeStaticArrayParam(*this, Param);
     }
     return;
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index c34a40fa7c81..51c8e04bee8c 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -6083,7 +6083,7 @@ static uint64_t EvaluateArrayTypeTrait(Sema &Self, ArrayTypeTrait ATT,
 
       if (Matched && T->isArrayType()) {
         if (const ConstantArrayType *CAT = Self.Context.getAsConstantArrayType(T))
-          return CAT->getSize().getLimitedValue();
+          return CAT->getLimitedSize();
       }
     }
     return 0;
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index aa470adb30b4..2b4805d62d07 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -213,7 +213,7 @@ static void CheckStringInit(Expr *Str, QualType &DeclT, const ArrayType *AT,
   // Get the length of the string as parsed.
   auto *ConstantArrayTy =
       cast<ConstantArrayType>(Str->getType()->getAsArrayTypeUnsafe());
-  uint64_t StrLength = ConstantArrayTy->getSize().getZExtValue();
+  uint64_t StrLength = ConstantArrayTy->getZExtSize();
 
   if (CheckC23ConstexprInit)
     if (const StringLiteral *SL = dyn_cast<StringLiteral>(Str->IgnoreParens()))
@@ -246,14 +246,13 @@ static void CheckStringInit(Expr *Str, QualType &DeclT, const ArrayType *AT,
     }
 
     // [dcl.init.string]p2
-    if (StrLength > CAT->getSize().getZExtValue())
+    if (StrLength > CAT->getZExtSize())
       S.Diag(Str->getBeginLoc(),
              diag::err_initializer_string_for_char_array_too_long)
-          << CAT->getSize().getZExtValue() << StrLength
-          << Str->getSourceRange();
+          << CAT->getZExtSize() << StrLength << Str->getSourceRange();
   } else {
     // C99 6.7.8p14.
-    if (StrLength-1 > CAT->getSize().getZExtValue())
+    if (StrLength - 1 > CAT->getZExtSize())
       S.Diag(Str->getBeginLoc(),
              diag::ext_initializer_string_for_char_array_too_long)
           << Str->getSourceRange();
@@ -879,7 +878,7 @@ InitListChecker::FillInEmptyInitializations(const InitializedEntity &Entity,
   if (const ArrayType *AType = SemaRef.Context.getAsArrayType(ILE->getType())) {
     ElementType = AType->getElementType();
     if (const auto *CAType = dyn_cast<ConstantArrayType>(AType))
-      NumElements = CAType->getSize().getZExtValue();
+      NumElements = CAType->getZExtSize();
     // For an array new with an unknown bound, ask for one additional element
     // in order to populate the array filler.
     if (Entity.isVariableLengthArrayNew())
@@ -1016,7 +1015,7 @@ int InitListChecker::numArrayElements(QualType DeclType) {
   int maxElements = 0x7FFFFFFF;
   if (const ConstantArrayType *CAT =
         SemaRef.Context.getAsConstantArrayType(DeclType)) {
-    maxElements = static_cast<int>(CAT->getSize().getZExtValue());
+    maxElements = static_cast<int>(CAT->getZExtSize());
   }
   return maxElements;
 }
@@ -3101,7 +3100,7 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
       // Get the length of the string.
       uint64_t StrLen = SL->getLength();
       if (cast<ConstantArrayType>(AT)->getSize().ult(StrLen))
-        StrLen = cast<ConstantArrayType>(AT)->getSize().getZExtValue();
+        StrLen = cast<ConstantArrayType>(AT)->getZExtSize();
       StructuredList->resizeInits(Context, StrLen);
 
       // Build a literal for each character in the string, and put them into
@@ -3124,7 +3123,7 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
       // Get the length of the string.
       uint64_t StrLen = Str.size();
       if (cast<ConstantArrayType>(AT)->getSize().ult(StrLen))
-        StrLen = cast<ConstantArrayType>(AT)->getSize().getZExtValue();
+        StrLen = cast<ConstantArrayType>(AT)->getZExtSize();
       StructuredList->resizeInits(Context, StrLen);
 
       // Build a literal for each character in the string, and put them into
@@ -3283,7 +3282,7 @@ InitListChecker::createInitListExpr(QualType CurrentObjectType,
   if (const ArrayType *AType
       = SemaRef.Context.getAsArrayType(CurrentObjectType)) {
     if (const ConstantArrayType *CAType = dyn_cast<ConstantArrayType>(AType)) {
-      NumElements = CAType->getSize().getZExtValue();
+      NumElements = CAType->getZExtSize();
       // Simple heuristic so that we don't allocate a very large
       // initializer with many empty entries at the end.
       if (NumElements > ExpectedNumInits)
@@ -5492,7 +5491,7 @@ static void TryOrBuildParenListInitialization(
     //   having k elements.
     if (const ConstantArrayType *CAT =
             S.getASTContext().getAsConstantArrayType(Entity.getType())) {
-      ArrayLength = CAT->getSize().getZExtValue();
+      ArrayLength = CAT->getZExtSize();
       ResultType = Entity.getType();
     } else if (const VariableArrayType *VAT =
                    S.getASTContext().getAsVariableArrayType(Entity.getType())) {
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index e9ad7bbde0f9..0ba54a3a9cae 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -21284,7 +21284,7 @@ static bool checkArrayExpressionDoesNotReferToWholeSize(Sema &SemaRef,
   if (isa<ArraySubscriptExpr>(E) ||
       (OASE && OASE->getColonLocFirst().isInvalid())) {
     if (const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.getTypePtr()))
-      return ATy->getSize().getSExtValue() != 1;
+      return ATy->getSExtSize() != 1;
     // Size can't be evaluated statically.
     return false;
   }
@@ -21325,7 +21325,7 @@ static bool checkArrayExpressionDoesNotReferToWholeSize(Sema &SemaRef,
     return false; // Can't get the integer value as a constant.
 
   llvm::APSInt ConstLength = Result.Val.getInt();
-  return CATy->getSize().getSExtValue() != ConstLength.getSExtValue();
+  return CATy->getSExtSize() != ConstLength.getSExtValue();
 }
 
 // Return true if it can be proven that the provided array expression (array
@@ -21350,7 +21350,7 @@ static bool checkArrayExpressionDoesNotReferToUnitySize(Sema &SemaRef,
   // is pointer.
   if (!Length) {
     if (const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.getTypePtr()))
-      return ATy->getSize().getSExtValue() != 1;
+      return ATy->getSExtSize() != 1;
     // We cannot assume anything.
     return false;
   }
diff --git a/clang/lib/Sema/SemaSYCL.cpp b/clang/lib/Sema/SemaSYCL.cpp
index ca0254d29e7f..18ebaa13346a 100644
--- a/clang/lib/Sema/SemaSYCL.cpp
+++ b/clang/lib/Sema/SemaSYCL.cpp
@@ -35,7 +35,7 @@ Sema::SemaDiagnosticBuilder Sema::SYCLDiagIfDeviceCode(SourceLocation Loc,
 
 static bool isZeroSizedArray(Sema &SemaRef, QualType Ty) {
   if (const auto *CAT = SemaRef.getASTContext().getAsConstantArrayType(Ty))
-    return CAT->getSize() == 0;
+    return CAT->isZeroSize();
   return false;
 }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp
index a50772f881f7..2cff97a591b8 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp
@@ -68,7 +68,7 @@ static bool evenFlexibleArraySize(ASTContext &Ctx, CharUnits RegionSize,
     FlexSize = Ctx.getTypeSizeInChars(ElemType);
     if (ArrayTy->getSize() == 1 && TypeSize > FlexSize)
       TypeSize -= FlexSize;
-    else if (ArrayTy->getSize() != 0)
+    else if (!ArrayTy->isZeroSize())
       return false;
   } else if (RD->hasFlexibleArrayMember()) {
     FlexSize = Ctx.getTypeSizeInChars(ElemType);
diff --git a/clang/lib/StaticAnalyzer/Checkers/PaddingChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/PaddingChecker.cpp
index eee9449f3180..4f35d9442ad9 100644
--- a/clang/lib/StaticAnalyzer/Checkers/PaddingChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/PaddingChecker.cpp
@@ -117,7 +117,7 @@ public:
       return;
     uint64_t Elts = 0;
     if (const ConstantArrayType *CArrTy = dyn_cast<ConstantArrayType>(ArrTy))
-      Elts = CArrTy->getSize().getZExtValue();
+      Elts = CArrTy->getZExtSize();
     if (Elts == 0)
       return;
     const RecordType *RT = ArrTy->getElementType()->getAs<RecordType>();
diff --git a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
index a8e573f7982b..d6e4f23cc353 100644
--- a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
+++ b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
@@ -825,7 +825,7 @@ DefinedOrUnknownSVal MemRegionManager::getStaticSize(const MemRegion *MR,
       };
       auto IsArrayOfZero = [](const ArrayType *AT) {
         const auto *CAT = dyn_cast<ConstantArrayType>(AT);
-        return CAT && CAT->getSize() == 0;
+        return CAT && CAT->isZeroSize();
       };
       auto IsArrayOfOne = [](const ArrayType *AT) {
         const auto *CAT = dyn_cast<ConstantArrayType>(AT);
diff --git a/clang/lib/StaticAnalyzer/Core/RegionStore.cpp b/clang/lib/StaticAnalyzer/Core/RegionStore.cpp
index da9a1a1a4d1f..755a8c4b22fd 100644
--- a/clang/lib/StaticAnalyzer/Core/RegionStore.cpp
+++ b/clang/lib/StaticAnalyzer/Core/RegionStore.cpp
@@ -1166,7 +1166,7 @@ void InvalidateRegionsWorker::VisitCluster(const MemRegion *baseR,
 
       // Compute lower and upper offsets for region within array.
       if (const ConstantArrayType *CAT = dyn_cast<ConstantArrayType>(AT))
-        NumElements = CAT->getSize().getZExtValue();
+        NumElements = CAT->getZExtSize();
       if (!NumElements) // We are not dealing with a constant size array
         goto conjure_default;
       QualType ElementTy = AT->getElementType();
@@ -1613,7 +1613,7 @@ getConstantArrayExtents(const ConstantArrayType *CAT) {
   CAT = cast<ConstantArrayType>(CAT->getCanonicalTypeInternal());
   SmallVector<uint64_t, 2> Extents;
   do {
-    Extents.push_back(CAT->getSize().getZExtValue());
+    Extents.push_back(CAT->getZExtSize());
   } while ((CAT = dyn_cast<ConstantArrayType>(CAT->getElementType())));
   return Extents;
 }
@@ -2436,7 +2436,7 @@ std::optional<RegionBindingsRef> RegionStoreManager::tryBindSmallArray(
     return std::nullopt;
 
   // If the array is too big, create a LCV instead.
-  uint64_t ArrSize = CAT->getSize().getLimitedValue();
+  uint64_t ArrSize = CAT->getLimitedSize();
   if (ArrSize > SmallArrayLimit)
     return std::nullopt;
 
@@ -2465,7 +2465,7 @@ RegionStoreManager::bindArray(RegionBindingsConstRef B,
   std::optional<uint64_t> Size;
 
   if (const ConstantArrayType* CAT = dyn_cast<ConstantArrayType>(AT))
-    Size = CAT->getSize().getZExtValue();
+    Size = CAT->getZExtSize();
 
   // Check if the init expr is a literal. If so, bind the rvalue instead.
   // FIXME: It's not responsibility of the Store to transform this lvalue
-- 
GitLab


From 5e6e40fee31d5db2f44d604b0362e1e819f41ba5 Mon Sep 17 00:00:00 2001
From: Piotr Zegar <me@piotrzegar.pl>
Date: Tue, 26 Mar 2024 20:19:23 +0100
Subject: [PATCH 333/404] [clang-tidy] Improve performance of
 google-runtime-int (#86596)

Main problem with performance of this check is caused by hasAncestor
matcher, and to be more precise by an llvm::DenseSet and std::deque in
matchesAnyAncestorOf.

To reduce impact of this matcher, multiple conditions that were checked
in check method were copied into AST matcher that is now checked before
hasAncestor.

Using custom getCheckTraversalKind to exclude template instances that
shouldn't be checked anyway is an additional improvement, but gain from
that one is low.

Tested on ffl_tests.cc, visible reduction from ~442 seconds to ~15
seconds (~96% reduction).

Closes #86553
---
 .../clang-tidy/google/IntegerTypesCheck.cpp   | 41 +++++++++++++++----
 .../clang-tidy/google/IntegerTypesCheck.h     |  3 ++
 clang-tools-extra/docs/ReleaseNotes.rst       |  3 ++
 3 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/google/IntegerTypesCheck.cpp b/clang-tools-extra/clang-tidy/google/IntegerTypesCheck.cpp
index ef511e9108f2..359d8efd100b 100644
--- a/clang-tools-extra/clang-tidy/google/IntegerTypesCheck.cpp
+++ b/clang-tools-extra/clang-tidy/google/IntegerTypesCheck.cpp
@@ -40,6 +40,34 @@ namespace {
 AST_MATCHER(FunctionDecl, isUserDefineLiteral) {
   return Node.getLiteralIdentifier() != nullptr;
 }
+
+AST_MATCHER(TypeLoc, isValidAndNotInMacro) {
+  const SourceLocation Loc = Node.getBeginLoc();
+  return Loc.isValid() && !Loc.isMacroID();
+}
+
+AST_MATCHER(TypeLoc, isBuiltinType) {
+  TypeLoc TL = Node;
+  if (auto QualLoc = Node.getAs<QualifiedTypeLoc>())
+    TL = QualLoc.getUnqualifiedLoc();
+
+  const auto BuiltinLoc = TL.getAs<BuiltinTypeLoc>();
+  if (!BuiltinLoc)
+    return false;
+
+  switch (BuiltinLoc.getTypePtr()->getKind()) {
+  case BuiltinType::Short:
+  case BuiltinType::Long:
+  case BuiltinType::LongLong:
+  case BuiltinType::UShort:
+  case BuiltinType::ULong:
+  case BuiltinType::ULongLong:
+    return true;
+  default:
+    return false;
+  }
+}
+
 } // namespace
 
 namespace tidy::google::runtime {
@@ -63,11 +91,11 @@ void IntegerTypesCheck::registerMatchers(MatchFinder *Finder) {
   // "Where possible, avoid passing arguments of types specified by
   // bitwidth typedefs to printf-based APIs."
   Finder->addMatcher(
-      typeLoc(loc(isInteger()),
-              unless(anyOf(hasAncestor(callExpr(
-                               callee(functionDecl(hasAttr(attr::Format))))),
-                           hasParent(parmVarDecl(hasAncestor(
-                               functionDecl(isUserDefineLiteral())))))))
+      typeLoc(loc(isInteger()), isValidAndNotInMacro(), isBuiltinType(),
+              unless(hasAncestor(
+                  callExpr(callee(functionDecl(hasAttr(attr::Format)))))),
+              unless(hasParent(parmVarDecl(
+                  hasAncestor(functionDecl(isUserDefineLiteral()))))))
           .bind("tl"),
       this);
   IdentTable = std::make_unique<IdentifierTable>(getLangOpts());
@@ -77,9 +105,6 @@ void IntegerTypesCheck::check(const MatchFinder::MatchResult &Result) {
   auto TL = *Result.Nodes.getNodeAs<TypeLoc>("tl");
   SourceLocation Loc = TL.getBeginLoc();
 
-  if (Loc.isInvalid() || Loc.isMacroID())
-    return;
-
   // Look through qualification.
   if (auto QualLoc = TL.getAs<QualifiedTypeLoc>())
     TL = QualLoc.getUnqualifiedLoc();
diff --git a/clang-tools-extra/clang-tidy/google/IntegerTypesCheck.h b/clang-tools-extra/clang-tidy/google/IntegerTypesCheck.h
index 3be7d24021b9..c62bda67ae2d 100644
--- a/clang-tools-extra/clang-tidy/google/IntegerTypesCheck.h
+++ b/clang-tools-extra/clang-tidy/google/IntegerTypesCheck.h
@@ -35,6 +35,9 @@ public:
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
+  std::optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_IgnoreUnlessSpelledInSource;
+  }
 
 private:
   const StringRef UnsignedTypePrefix;
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index f6ae712510ca..f119e14ae478 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -201,6 +201,9 @@ Changes in existing checks
   <clang-tidy/checks/google/global-names-in-headers>` check by replacing the local
   option `HeaderFileExtensions` by the global option of the same name.
 
+- Improved :doc:`google-runtime-int <clang-tidy/checks/google/runtime-int>`
+  check performance through optimizations.
+
 - Improved :doc:`llvm-header-guard
   <clang-tidy/checks/llvm/header-guard>` check by replacing the local
   option `HeaderFileExtensions` by the global option of the same name.
-- 
GitLab


From 156c2907462bc5e97d13d3e7d334a32a291bc787 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Mon, 25 Mar 2024 14:50:58 -0700
Subject: [PATCH 334/404] [lldb] Implement coalescing of disjoint progress
 events (#84854)

This implements coalescing of progress events using a timeout, as
discussed in the RFC on Discourse [1]. This PR consists of two commits
which, depending on the feedback, I may split up into two PRs. For now,
I think it's easier to review this as a whole.

1. The first commit introduces a new generic `Alarm` class. The class
lets you to schedule a function (callback) to be executed after a given
timeout expires. You can cancel and reset a callback before its
corresponding timeout expires. It achieves this with the help of a
worker thread that sleeps until the next timeout expires. The only
guarantee it provides is that your function is called no sooner than the
requested timeout. Because the callback is called directly from the
worker thread, a long running callback could potentially block the
worker thread. I intentionally kept the implementation as simple as
possible while addressing the needs for the `ProgressManager` use case.
If we want to rely on this somewhere else, we can reassess whether we
need to address those limitations.

2. The second commit uses the Alarm class to coalesce progress events.
To recap the Discourse discussion, when multiple progress events with
the same title execute in close succession, they get broadcast as one to
`eBroadcastBitProgressCategory`. The `ProgressManager` keeps track of
the in-flight progress events and when the refcount hits zero, the Alarm
class is used to schedule broadcasting the event. If a new progress
event comes in before the alarm fires, the alarm is reset (and the
process repeats when the new progress event ends). If no new event comes
in before the timeout expires, the progress event is broadcast.

[1]
https://discourse.llvm.org/t/rfc-improve-lldb-progress-reporting/75717/
---
 lldb/include/lldb/Core/Progress.h          |  35 ++++++-
 lldb/source/API/SystemInitializerFull.cpp  |  14 ++-
 lldb/source/Core/Progress.cpp              | 114 ++++++++++++++++-----
 lldb/unittests/Core/ProgressReportTest.cpp |  39 ++++++-
 4 files changed, 167 insertions(+), 35 deletions(-)

diff --git a/lldb/include/lldb/Core/Progress.h b/lldb/include/lldb/Core/Progress.h
index eb3af9816dc6..cd87be79c4f0 100644
--- a/lldb/include/lldb/Core/Progress.h
+++ b/lldb/include/lldb/Core/Progress.h
@@ -9,6 +9,7 @@
 #ifndef LLDB_CORE_PROGRESS_H
 #define LLDB_CORE_PROGRESS_H
 
+#include "lldb/Host/Alarm.h"
 #include "lldb/lldb-forward.h"
 #include "lldb/lldb-types.h"
 #include "llvm/ADT/StringMap.h"
@@ -150,9 +151,12 @@ public:
   void Increment(const Progress::ProgressData &);
   void Decrement(const Progress::ProgressData &);
 
+  static void Initialize();
+  static void Terminate();
+  static bool Enabled();
   static ProgressManager &Instance();
 
-private:
+protected:
   enum class EventType {
     Begin,
     End,
@@ -160,9 +164,32 @@ private:
   static void ReportProgress(const Progress::ProgressData &progress_data,
                              EventType type);
 
-  llvm::StringMap<std::pair<uint64_t, Progress::ProgressData>>
-      m_progress_category_map;
-  std::mutex m_progress_map_mutex;
+  static std::optional<ProgressManager> &InstanceImpl();
+
+  /// Helper function for reporting progress when the alarm in the corresponding
+  /// entry in the map expires.
+  void Expire(llvm::StringRef key);
+
+  /// Entry used for bookkeeping.
+  struct Entry {
+    /// Reference count used for overlapping events.
+    uint64_t refcount = 0;
+
+    /// Data used to emit progress events.
+    Progress::ProgressData data;
+
+    /// Alarm handle used when the refcount reaches zero.
+    Alarm::Handle handle = Alarm::INVALID_HANDLE;
+  };
+
+  /// Map used for bookkeeping.
+  llvm::StringMap<Entry> m_entries;
+
+  /// Mutex to provide the map.
+  std::mutex m_entries_mutex;
+
+  /// Alarm instance to coalesce progress events.
+  Alarm m_alarm;
 };
 
 } // namespace lldb_private
diff --git a/lldb/source/API/SystemInitializerFull.cpp b/lldb/source/API/SystemInitializerFull.cpp
index c48466f25ede..995d14f7c1fa 100644
--- a/lldb/source/API/SystemInitializerFull.cpp
+++ b/lldb/source/API/SystemInitializerFull.cpp
@@ -10,6 +10,7 @@
 #include "lldb/API/SBCommandInterpreter.h"
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/PluginManager.h"
+#include "lldb/Core/Progress.h"
 #include "lldb/Host/Config.h"
 #include "lldb/Host/Host.h"
 #include "lldb/Initialization/SystemInitializerCommon.h"
@@ -57,6 +58,7 @@ llvm::Error SystemInitializerFull::Initialize() {
   llvm::InitializeAllAsmPrinters();
   llvm::InitializeAllTargetMCs();
   llvm::InitializeAllDisassemblers();
+
   // Initialize the command line parser in LLVM. This usually isn't necessary
   // as we aren't dealing with command line options here, but otherwise some
   // other code in Clang/LLVM might be tempted to call this function from a
@@ -65,10 +67,13 @@ llvm::Error SystemInitializerFull::Initialize() {
   const char *arg0 = "lldb";
   llvm::cl::ParseCommandLineOptions(1, &arg0);
 
+  // Initialize the progress manager.
+  ProgressManager::Initialize();
+
 #define LLDB_PLUGIN(p) LLDB_PLUGIN_INITIALIZE(p);
 #include "Plugins/Plugins.def"
 
-  // Scan for any system or user LLDB plug-ins
+  // Scan for any system or user LLDB plug-ins.
   PluginManager::Initialize();
 
   // The process settings need to know about installed plug-ins, so the
@@ -84,15 +89,18 @@ llvm::Error SystemInitializerFull::Initialize() {
 void SystemInitializerFull::Terminate() {
   Debugger::SettingsTerminate();
 
-  // Terminate plug-ins in core LLDB
+  // Terminate plug-ins in core LLDB.
   ProcessTrace::Terminate();
 
-  // Terminate and unload and loaded system or user LLDB plug-ins
+  // Terminate and unload and loaded system or user LLDB plug-ins.
   PluginManager::Terminate();
 
 #define LLDB_PLUGIN(p) LLDB_PLUGIN_TERMINATE(p);
 #include "Plugins/Plugins.def"
 
+  // Terminate the progress manager.
+  ProgressManager::Terminate();
+
   // Now shutdown the common parts, in reverse order.
   SystemInitializerCommon::Terminate();
 }
diff --git a/lldb/source/Core/Progress.cpp b/lldb/source/Core/Progress.cpp
index b4b5e98b7ba4..161038284e21 100644
--- a/lldb/source/Core/Progress.cpp
+++ b/lldb/source/Core/Progress.cpp
@@ -35,7 +35,10 @@ Progress::Progress(std::string title, std::string details,
 
   std::lock_guard<std::mutex> guard(m_mutex);
   ReportProgress();
-  ProgressManager::Instance().Increment(m_progress_data);
+
+  // Report to the ProgressManager if that subsystem is enabled.
+  if (ProgressManager::Enabled())
+    ProgressManager::Instance().Increment(m_progress_data);
 }
 
 Progress::~Progress() {
@@ -45,7 +48,10 @@ Progress::~Progress() {
   if (!m_completed)
     m_completed = m_total;
   ReportProgress();
-  ProgressManager::Instance().Decrement(m_progress_data);
+
+  // Report to the ProgressManager if that subsystem is enabled.
+  if (ProgressManager::Enabled())
+    ProgressManager::Instance().Decrement(m_progress_data);
 }
 
 void Progress::Increment(uint64_t amount,
@@ -75,45 +81,84 @@ void Progress::ReportProgress() {
   }
 }
 
-ProgressManager::ProgressManager() : m_progress_category_map() {}
+ProgressManager::ProgressManager()
+    : m_entries(), m_alarm(std::chrono::milliseconds(100)) {}
 
 ProgressManager::~ProgressManager() {}
 
+void ProgressManager::Initialize() {
+  assert(!InstanceImpl() && "Already initialized.");
+  InstanceImpl().emplace();
+}
+
+void ProgressManager::Terminate() {
+  assert(InstanceImpl() && "Already terminated.");
+  InstanceImpl().reset();
+}
+
+bool ProgressManager::Enabled() { return InstanceImpl().operator bool(); }
+
 ProgressManager &ProgressManager::Instance() {
-  static std::once_flag g_once_flag;
-  static ProgressManager *g_progress_manager = nullptr;
-  std::call_once(g_once_flag, []() {
-    // NOTE: known leak to avoid global destructor chain issues.
-    g_progress_manager = new ProgressManager();
-  });
-  return *g_progress_manager;
+  assert(InstanceImpl() && "ProgressManager must be initialized");
+  return *InstanceImpl();
+}
+
+std::optional<ProgressManager> &ProgressManager::InstanceImpl() {
+  static std::optional<ProgressManager> g_progress_manager;
+  return g_progress_manager;
 }
 
 void ProgressManager::Increment(const Progress::ProgressData &progress_data) {
-  std::lock_guard<std::mutex> lock(m_progress_map_mutex);
-  // If the current category exists in the map then it is not an initial report,
-  // therefore don't broadcast to the category bit. Also, store the current
-  // progress data in the map so that we have a note of the ID used for the
-  // initial progress report.
-  if (!m_progress_category_map.contains(progress_data.title)) {
-    m_progress_category_map[progress_data.title].second = progress_data;
+  std::lock_guard<std::mutex> lock(m_entries_mutex);
+
+  llvm::StringRef key = progress_data.title;
+  bool new_entry = !m_entries.contains(key);
+  Entry &entry = m_entries[progress_data.title];
+
+  if (new_entry) {
+    // This is a new progress event. Report progress and store the progress
+    // data.
     ReportProgress(progress_data, EventType::Begin);
+    entry.data = progress_data;
+  } else if (entry.refcount == 0) {
+    // This is an existing entry that was scheduled to be deleted but a new one
+    // came in before the timer expired.
+    assert(entry.handle != Alarm::INVALID_HANDLE);
+
+    if (!m_alarm.Cancel(entry.handle)) {
+      // The timer expired before we had a chance to cancel it. We have to treat
+      // this as an entirely new progress event.
+      ReportProgress(progress_data, EventType::Begin);
+    }
+    // Clear the alarm handle.
+    entry.handle = Alarm::INVALID_HANDLE;
   }
-  m_progress_category_map[progress_data.title].first++;
+
+  // Regardless of how we got here, we need to bump the reference count.
+  entry.refcount++;
 }
 
 void ProgressManager::Decrement(const Progress::ProgressData &progress_data) {
-  std::lock_guard<std::mutex> lock(m_progress_map_mutex);
-  auto pos = m_progress_category_map.find(progress_data.title);
+  std::lock_guard<std::mutex> lock(m_entries_mutex);
+  llvm::StringRef key = progress_data.title;
 
-  if (pos == m_progress_category_map.end())
+  if (!m_entries.contains(key))
     return;
 
-  if (pos->second.first <= 1) {
-    ReportProgress(pos->second.second, EventType::End);
-    m_progress_category_map.erase(progress_data.title);
-  } else {
-    --pos->second.first;
+  Entry &entry = m_entries[key];
+  entry.refcount--;
+
+  if (entry.refcount == 0) {
+    assert(entry.handle == Alarm::INVALID_HANDLE);
+
+    // Copy the key to a std::string so we can pass it by value to the lambda.
+    // The underlying StringRef will not exist by the time the callback is
+    // called.
+    std::string key_str = std::string(key);
+
+    // Start a timer. If it expires before we see another progress event, it
+    // will be reported.
+    entry.handle = m_alarm.Create([=]() { Expire(key_str); });
   }
 }
 
@@ -129,3 +174,20 @@ void ProgressManager::ReportProgress(
                            progress_data.debugger_id,
                            Debugger::eBroadcastBitProgressCategory);
 }
+
+void ProgressManager::Expire(llvm::StringRef key) {
+  std::lock_guard<std::mutex> lock(m_entries_mutex);
+
+  // This shouldn't happen but be resilient anyway.
+  if (!m_entries.contains(key))
+    return;
+
+  // A new event came in and the alarm fired before we had a chance to restart
+  // it.
+  if (m_entries[key].refcount != 0)
+    return;
+
+  // We're done with this entry.
+  ReportProgress(m_entries[key].data, EventType::End);
+  m_entries.erase(key);
+}
diff --git a/lldb/unittests/Core/ProgressReportTest.cpp b/lldb/unittests/Core/ProgressReportTest.cpp
index 1f993180fd83..f0d253be9bf6 100644
--- a/lldb/unittests/Core/ProgressReportTest.cpp
+++ b/lldb/unittests/Core/ProgressReportTest.cpp
@@ -22,7 +22,7 @@
 using namespace lldb;
 using namespace lldb_private;
 
-static std::chrono::milliseconds TIMEOUT(100);
+static std::chrono::milliseconds TIMEOUT(500);
 
 class ProgressReportTest : public ::testing::Test {
 public:
@@ -56,7 +56,8 @@ protected:
 
   DebuggerSP m_debugger_sp;
   ListenerSP m_listener_sp;
-  SubsystemRAII<FileSystem, HostInfo, PlatformMacOSX> subsystems;
+  SubsystemRAII<FileSystem, HostInfo, PlatformMacOSX, ProgressManager>
+      subsystems;
 };
 
 TEST_F(ProgressReportTest, TestReportCreation) {
@@ -210,3 +211,37 @@ TEST_F(ProgressReportTest, TestOverlappingEvents) {
   // initial report.
   EXPECT_EQ(data->GetID(), expected_progress_id);
 }
+
+TEST_F(ProgressReportTest, TestProgressManagerDisjointReports) {
+  ListenerSP listener_sp =
+      CreateListenerFor(Debugger::eBroadcastBitProgressCategory);
+  EventSP event_sp;
+  const ProgressEventData *data;
+  uint64_t expected_progress_id;
+
+  { Progress progress("Coalesced report 1", "Starting report 1"); }
+  { Progress progress("Coalesced report 1", "Starting report 2"); }
+  { Progress progress("Coalesced report 1", "Starting report 3"); }
+
+  ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT));
+  data = ProgressEventData::GetEventDataFromEvent(event_sp.get());
+  expected_progress_id = data->GetID();
+
+  EXPECT_EQ(data->GetDetails(), "");
+  EXPECT_FALSE(data->IsFinite());
+  EXPECT_FALSE(data->GetCompleted());
+  EXPECT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal);
+  EXPECT_EQ(data->GetMessage(), "Coalesced report 1");
+
+  ASSERT_TRUE(listener_sp->GetEvent(event_sp, TIMEOUT));
+  data = ProgressEventData::GetEventDataFromEvent(event_sp.get());
+
+  EXPECT_EQ(data->GetID(), expected_progress_id);
+  EXPECT_EQ(data->GetDetails(), "");
+  EXPECT_FALSE(data->IsFinite());
+  EXPECT_TRUE(data->GetCompleted());
+  EXPECT_EQ(data->GetTotal(), Progress::kNonDeterministicTotal);
+  EXPECT_EQ(data->GetMessage(), "Coalesced report 1");
+
+  ASSERT_FALSE(listener_sp->GetEvent(event_sp, TIMEOUT));
+}
-- 
GitLab


From 8ea94b614bd8ac74802867588129bcfd0a326620 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?F=C3=A9lix-Antoine=20Constantin?=
 <60141446+felix642@users.noreply.github.com>
Date: Tue, 26 Mar 2024 15:39:05 -0400
Subject: [PATCH 335/404] [clang-tidy] Improved modernize-use-using by fixing a
 false-negative (#82947)

The check needs a parent decl to match but if the typedef is in a
function, the parent is a declStmt which is not a decl by itself.
Improved the matcher to match on either a decl or a declstmt and extract
the decl from the stmt in the latter case.

Fixes #72179
---
 .../clang-tidy/modernize/UseUsingCheck.cpp    | 24 +++++++++--
 clang-tools-extra/docs/ReleaseNotes.rst       |  3 ++
 .../checkers/modernize/use-using.cpp          | 43 ++++++++++++++++++-
 3 files changed, 66 insertions(+), 4 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/modernize/UseUsingCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseUsingCheck.cpp
index bb05f206c717..24eefdb082eb 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseUsingCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseUsingCheck.cpp
@@ -8,6 +8,7 @@
 
 #include "UseUsingCheck.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/DeclGroup.h"
 #include "clang/Lex/Lexer.h"
 
 using namespace clang::ast_matchers;
@@ -24,6 +25,7 @@ static constexpr llvm::StringLiteral ExternCDeclName = "extern-c-decl";
 static constexpr llvm::StringLiteral ParentDeclName = "parent-decl";
 static constexpr llvm::StringLiteral TagDeclName = "tag-decl";
 static constexpr llvm::StringLiteral TypedefName = "typedef";
+static constexpr llvm::StringLiteral DeclStmtName = "decl-stmt";
 
 UseUsingCheck::UseUsingCheck(StringRef Name, ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
@@ -41,7 +43,8 @@ void UseUsingCheck::registerMatchers(MatchFinder *Finder) {
           unless(isInstantiated()),
           optionally(hasAncestor(
               linkageSpecDecl(isExternCLinkage()).bind(ExternCDeclName))),
-          hasParent(decl().bind(ParentDeclName)))
+          anyOf(hasParent(decl().bind(ParentDeclName)),
+                hasParent(declStmt().bind(DeclStmtName))))
           .bind(TypedefName),
       this);
 
@@ -51,17 +54,32 @@ void UseUsingCheck::registerMatchers(MatchFinder *Finder) {
       tagDecl(
           anyOf(allOf(unless(anyOf(isImplicit(),
                                    classTemplateSpecializationDecl())),
-                      hasParent(decl().bind(ParentDeclName))),
+                      anyOf(hasParent(decl().bind(ParentDeclName)),
+                            hasParent(declStmt().bind(DeclStmtName)))),
                 // We want the parent of the ClassTemplateDecl, not the parent
                 // of the specialization.
                 classTemplateSpecializationDecl(hasAncestor(classTemplateDecl(
-                    hasParent(decl().bind(ParentDeclName)))))))
+                    anyOf(hasParent(decl().bind(ParentDeclName)),
+                          hasParent(declStmt().bind(DeclStmtName))))))))
           .bind(TagDeclName),
       this);
 }
 
 void UseUsingCheck::check(const MatchFinder::MatchResult &Result) {
   const auto *ParentDecl = Result.Nodes.getNodeAs<Decl>(ParentDeclName);
+
+  if (!ParentDecl) {
+    const auto *ParentDeclStmt = Result.Nodes.getNodeAs<DeclStmt>(DeclStmtName);
+    if (ParentDeclStmt) {
+      if (ParentDeclStmt->isSingleDecl())
+        ParentDecl = ParentDeclStmt->getSingleDecl();
+      else
+        ParentDecl =
+            ParentDeclStmt->getDeclGroup().getDeclGroup()
+                [ParentDeclStmt->getDeclGroup().getDeclGroup().size() - 1];
+    }
+  }
+
   if (!ParentDecl)
     return;
 
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index f119e14ae478..14c48547bcbe 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -242,6 +242,9 @@ Changes in existing checks
   analyzed, se the check now handles the common patterns
   `const auto e = (*vector_ptr)[i]` and `const auto e = vector_ptr->at(i);`.
 
+- Improved :doc:`modernize-use-using <clang-tidy/checks/modernize/use-using>`
+  check by adding support for detection of typedefs declared on function level.
+
 - Improved :doc:`readability-implicit-bool-conversion
   <clang-tidy/checks/readability/implicit-bool-conversion>` check to provide
   valid fix suggestions for ``static_cast`` without a preceding space and
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-using.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-using.cpp
index 462bc984fd3a..925e5f9c1ca5 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-using.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-using.cpp
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy %s modernize-use-using %t -- -- -I %S/Inputs/use-using/
+// RUN: %check_clang_tidy %s modernize-use-using %t -- -- -fno-delayed-template-parsing -I %S/Inputs/use-using/
 
 typedef int Type;
 // CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use 'using' instead of 'typedef' [modernize-use-using]
@@ -342,3 +342,44 @@ typedef int InExternCPP;
 // CHECK-FIXES: using InExternCPP = int;
 
 }
+
+namespace ISSUE_72179
+{  
+  void foo()
+  {
+    typedef int a;
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'using' instead of 'typedef' [modernize-use-using]
+    // CHECK-FIXES: using a = int;
+
+  }
+
+  void foo2()
+  {
+    typedef struct { int a; union { int b; }; } c;
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'using' instead of 'typedef' [modernize-use-using]
+    // CHECK-FIXES: using c = struct { int a; union { int b; }; };
+  }
+
+  template <typename T>
+  void foo3()
+  {
+    typedef T b;
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'using' instead of 'typedef' [modernize-use-using]
+    // CHECK-FIXES: using b = T;
+  }
+
+  template <typename T>
+  class MyClass
+  {
+    void foo()
+    {
+      typedef MyClass c;
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: use 'using' instead of 'typedef' [modernize-use-using]
+      // CHECK-FIXES: using c = MyClass;
+    }
+  };
+
+  const auto foo4 = [](int a){typedef int d;};
+  // CHECK-MESSAGES: :[[@LINE-1]]:31: warning: use 'using' instead of 'typedef' [modernize-use-using]
+  // CHECK-FIXES: const auto foo4 = [](int a){using d = int;};
+}
-- 
GitLab


From 8a84596310f5b141817c784f0b4b46a636767e6e Mon Sep 17 00:00:00 2001
From: Peter Klausler <35819229+klausler@users.noreply.github.com>
Date: Tue, 26 Mar 2024 12:41:54 -0700
Subject: [PATCH 336/404] [flang] Dodge bogus GCC 13.2.0 error message in new
 code (#86708)

Rearrange some new code a little bit to avoid a bogus error message
coming out from GCC 13.2.0 about an uninitialized data member in a
parser.
---
 flang/include/flang/Parser/parse-tree.h | 2 +-
 flang/lib/Parser/Fortran-parsers.cpp    | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 85e8121dd125..26b2e5f4e34b 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3317,7 +3317,7 @@ struct CompilerDirective {
     TUPLE_CLASS_BOILERPLATE(NameValue);
     std::tuple<Name, std::optional<std::uint64_t>> t;
   };
-  struct Unrecognized {};
+  EMPTY_CLASS(Unrecognized);
   CharBlock source;
   std::variant<std::list<IgnoreTKR>, LoopCount, std::list<AssumeAligned>,
       std::list<NameValue>, Unrecognized>
diff --git a/flang/lib/Parser/Fortran-parsers.cpp b/flang/lib/Parser/Fortran-parsers.cpp
index fd28eea0f947..21185694227d 100644
--- a/flang/lib/Parser/Fortran-parsers.cpp
+++ b/flang/lib/Parser/Fortran-parsers.cpp
@@ -1280,9 +1280,8 @@ TYPE_PARSER(beginDirective >> "DIR$ "_tok >>
                     many(construct<CompilerDirective::NameValue>(
                         name, maybe(("="_tok || ":"_tok) >> digitString64))))) /
             endOfStmt ||
-        construct<CompilerDirective>(
-            SkipTo<'\n'>{} >> pure<CompilerDirective::Unrecognized>()) /
-            endOfStmt))
+        construct<CompilerDirective>(pure<CompilerDirective::Unrecognized>()) /
+            SkipTo<'\n'>{}))
 
 TYPE_PARSER(extension<LanguageFeature::CrayPointer>(
     "nonstandard usage: based POINTER"_port_en_US,
-- 
GitLab


From 982ebeb212ddb354e2babc1ce1ddfccc06b6ea27 Mon Sep 17 00:00:00 2001
From: Bjorn Pettersson <bjorn.a.pettersson@ericsson.com>
Date: Mon, 25 Mar 2024 23:41:28 +0100
Subject: [PATCH 337/404] [X86] Pre-commit test case for bug in
 combineShiftRightArithmetic

It has been noticed that combineShiftRightArithmetic isn't dealing
properly with large shift amounts, as demonstrated by the test
case added in this commit.

I think the problem partly is related to X86 using i8 as shift amount
type during ISel. So shift amount larger then 127 may be treated
as negative shift amounts if not being careful.
---
 llvm/test/CodeGen/X86/sar_fold.ll | 44 +++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/llvm/test/CodeGen/X86/sar_fold.ll b/llvm/test/CodeGen/X86/sar_fold.ll
index 21655e19440a..7607ca386d57 100644
--- a/llvm/test/CodeGen/X86/sar_fold.ll
+++ b/llvm/test/CodeGen/X86/sar_fold.ll
@@ -44,3 +44,47 @@ define i32 @shl24sar25(i32 %a) #0 {
   %2 = ashr exact i32 %1, 25
   ret i32 %2
 }
+
+define void @shl144sar48(ptr %p) #0 {
+; CHECK-LABEL: shl144sar48:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movswl (%eax), %ecx
+; CHECK-NEXT:    movl %ecx, %edx
+; CHECK-NEXT:    sarl $31, %edx
+; CHECK-NEXT:    shldl $2, %ecx, %edx
+; CHECK-NEXT:    shll $2, %ecx
+; CHECK-NEXT:    movl %ecx, 12(%eax)
+; CHECK-NEXT:    movl %edx, 16(%eax)
+; CHECK-NEXT:    movl $0, 8(%eax)
+; CHECK-NEXT:    movl $0, 4(%eax)
+; CHECK-NEXT:    movl $0, (%eax)
+; CHECK-NEXT:    retl
+  %a = load i160, ptr %p
+  %1 = shl i160 %a, 144
+  %2 = ashr exact i160 %1, 46
+  store i160 %2, ptr %p
+  ret void
+}
+
+; This is incorrect. The 142 least significant bits in the stored value should
+; be zero, and but 142-157 should be taken from %a with a sign-extend into the
+; two most significant bits.
+define void @shl144sar2(ptr %p) #0 {
+; CHECK-LABEL: shl144sar2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movswl (%eax), %ecx
+; CHECK-NEXT:    sarl $31, %ecx
+; CHECK-NEXT:    movl %ecx, 16(%eax)
+; CHECK-NEXT:    movl %ecx, 8(%eax)
+; CHECK-NEXT:    movl %ecx, 12(%eax)
+; CHECK-NEXT:    movl %ecx, 4(%eax)
+; CHECK-NEXT:    movl %ecx, (%eax)
+; CHECK-NEXT:    retl
+  %a = load i160, ptr %p
+  %1 = shl i160 %a, 144
+  %2 = ashr exact i160 %1, 2
+  store i160 %2, ptr %p
+  ret void
+}
-- 
GitLab


From 4c72cfa31798a67c6daa25d474bcfe536e4fa0c0 Mon Sep 17 00:00:00 2001
From: Michael Jones <michaelrj@google.com>
Date: Tue, 26 Mar 2024 12:51:40 -0700
Subject: [PATCH 338/404] [libc] Remove definition of LIBC_NAMESPACE in test
 (#86704)

The canonicalize test added in #85940 defined the LIBC_NAMESPACE macro.
this macro is intended to be set only by the build system and never in
the code.
---
 libc/test/src/math/smoke/CanonicalizeTest.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/libc/test/src/math/smoke/CanonicalizeTest.h b/libc/test/src/math/smoke/CanonicalizeTest.h
index ce855d516856..c8af83f19838 100644
--- a/libc/test/src/math/smoke/CanonicalizeTest.h
+++ b/libc/test/src/math/smoke/CanonicalizeTest.h
@@ -22,8 +22,6 @@
 
 #define TEST_REGULAR(x, y, expected) TEST_SPECIAL(x, y, expected, 0)
 
-#define LIBC_NAMESPACE __llvm_libc_19_0_0_git
-
 template <typename T>
 class CanonicalizeTest : public LIBC_NAMESPACE::testing::Test {
 
-- 
GitLab


From 14e17ea1f62d9861f90a6a31e7c8a1d12bb081fc Mon Sep 17 00:00:00 2001
From: Razvan Lupusoru <razvan.lupusoru@gmail.com>
Date: Tue, 26 Mar 2024 12:52:13 -0700
Subject: [PATCH 339/404] [flang][acc] Add support for lowering combined
 constructs (#86696)

PR#80319 added support to record combined construct semantics via an
attribute. Add lowering support for this.
---
 flang/lib/Lower/OpenACC.cpp                   |  72 ++++++----
 flang/test/Lower/OpenACC/acc-kernels-loop.f90 | 131 +++++++++--------
 flang/test/Lower/OpenACC/acc-loop.f90         |   2 +-
 .../test/Lower/OpenACC/acc-parallel-loop.f90  | 135 ++++++++++--------
 flang/test/Lower/OpenACC/acc-private.f90      |  26 ++--
 flang/test/Lower/OpenACC/acc-serial-loop.f90  | 123 +++++++++-------
 6 files changed, 273 insertions(+), 216 deletions(-)

diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 7b7e4a875cd8..6e6714454f05 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -1667,15 +1667,17 @@ static void privatizeIv(Fortran::lower::AbstractConverter &converter,
   ivPrivate.push_back(privateValue);
 }
 
-static mlir::acc::LoopOp
-createLoopOp(Fortran::lower::AbstractConverter &converter,
-             mlir::Location currentLocation,
-             Fortran::semantics::SemanticsContext &semanticsContext,
-             Fortran::lower::StatementContext &stmtCtx,
-             const Fortran::parser::DoConstruct &outerDoConstruct,
-             Fortran::lower::pft::Evaluation &eval,
-             const Fortran::parser::AccClauseList &accClauseList,
-             bool needEarlyReturnHandling = false) {
+static mlir::acc::LoopOp createLoopOp(
+    Fortran::lower::AbstractConverter &converter,
+    mlir::Location currentLocation,
+    Fortran::semantics::SemanticsContext &semanticsContext,
+    Fortran::lower::StatementContext &stmtCtx,
+    const Fortran::parser::DoConstruct &outerDoConstruct,
+    Fortran::lower::pft::Evaluation &eval,
+    const Fortran::parser::AccClauseList &accClauseList,
+    std::optional<mlir::acc::CombinedConstructsType> combinedConstructs =
+        std::nullopt,
+    bool needEarlyReturnHandling = false) {
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   llvm::SmallVector<mlir::Value> tileOperands, privateOperands, ivPrivate,
       reductionOperands, cacheOperands, vectorOperands, workerNumOperands,
@@ -2015,6 +2017,10 @@ createLoopOp(Fortran::lower::AbstractConverter &converter,
   if (!collapseDeviceTypes.empty())
     loopOp.setCollapseDeviceTypeAttr(builder.getArrayAttr(collapseDeviceTypes));
 
+  if (combinedConstructs)
+    loopOp.setCombinedAttr(mlir::acc::CombinedConstructsTypeAttr::get(
+        builder.getContext(), *combinedConstructs));
+
   return loopOp;
 }
 
@@ -2060,7 +2066,7 @@ genACC(Fortran::lower::AbstractConverter &converter,
       std::get<std::optional<Fortran::parser::DoConstruct>>(loopConstruct.t);
   auto loopOp = createLoopOp(converter, currentLocation, semanticsContext,
                              stmtCtx, *outerDoConstruct, eval, accClauseList,
-                             needEarlyExitHandling);
+                             /*combinedConstructs=*/{}, needEarlyExitHandling);
   if (needEarlyExitHandling)
     return loopOp.getResult(0);
 
@@ -2092,14 +2098,14 @@ static void genDataOperandOperationsWithModifier(
 }
 
 template <typename Op>
-static Op
-createComputeOp(Fortran::lower::AbstractConverter &converter,
-                mlir::Location currentLocation,
-                Fortran::lower::pft::Evaluation &eval,
-                Fortran::semantics::SemanticsContext &semanticsContext,
-                Fortran::lower::StatementContext &stmtCtx,
-                const Fortran::parser::AccClauseList &accClauseList,
-                bool outerCombined = false) {
+static Op createComputeOp(
+    Fortran::lower::AbstractConverter &converter,
+    mlir::Location currentLocation, Fortran::lower::pft::Evaluation &eval,
+    Fortran::semantics::SemanticsContext &semanticsContext,
+    Fortran::lower::StatementContext &stmtCtx,
+    const Fortran::parser::AccClauseList &accClauseList,
+    std::optional<mlir::acc::CombinedConstructsType> combinedConstructs =
+        std::nullopt) {
 
   // Parallel operation operands
   mlir::Value ifCond;
@@ -2292,7 +2298,7 @@ createComputeOp(Fortran::lower::AbstractConverter &converter,
     } else if (const auto *privateClause =
                    std::get_if<Fortran::parser::AccClause::Private>(
                        &clause.u)) {
-      if (!outerCombined)
+      if (!combinedConstructs)
         genPrivatizations<mlir::acc::PrivateRecipeOp>(
             privateClause->v, converter, semanticsContext, stmtCtx,
             privateOperands, privatizations);
@@ -2310,7 +2316,7 @@ createComputeOp(Fortran::lower::AbstractConverter &converter,
       // combined - delay it to the loop. However, a reduction clause on a
       // combined construct implies a copy clause so issue an implicit copy
       // instead.
-      if (!outerCombined) {
+      if (!combinedConstructs) {
         genReductions(reductionClause->v, converter, semanticsContext, stmtCtx,
                       reductionOperands, reductionRecipes);
       } else {
@@ -2362,11 +2368,11 @@ createComputeOp(Fortran::lower::AbstractConverter &converter,
   if constexpr (std::is_same_v<Op, mlir::acc::KernelsOp>)
     computeOp = createRegionOp<Op, mlir::acc::TerminatorOp>(
         builder, currentLocation, currentLocation, eval, operands,
-        operandSegments, outerCombined);
+        operandSegments, /*outerCombined=*/combinedConstructs.has_value());
   else
     computeOp = createRegionOp<Op, mlir::acc::YieldOp>(
         builder, currentLocation, currentLocation, eval, operands,
-        operandSegments, outerCombined);
+        operandSegments, /*outerCombined=*/combinedConstructs.has_value());
 
   if (addSelfAttr)
     computeOp.setSelfAttrAttr(builder.getUnitAttr());
@@ -2419,6 +2425,9 @@ createComputeOp(Fortran::lower::AbstractConverter &converter,
           mlir::ArrayAttr::get(builder.getContext(), firstPrivatizations));
   }
 
+  if (combinedConstructs)
+    computeOp.setCombinedAttr(builder.getUnitAttr());
+
   auto insPt = builder.saveInsertionPoint();
   builder.setInsertionPointAfter(computeOp);
 
@@ -2734,21 +2743,24 @@ genACC(Fortran::lower::AbstractConverter &converter,
   if (combinedDirective.v == llvm::acc::ACCD_kernels_loop) {
     createComputeOp<mlir::acc::KernelsOp>(
         converter, currentLocation, eval, semanticsContext, stmtCtx,
-        accClauseList, /*outerCombined=*/true);
+        accClauseList, mlir::acc::CombinedConstructsType::KernelsLoop);
     createLoopOp(converter, currentLocation, semanticsContext, stmtCtx,
-                 *outerDoConstruct, eval, accClauseList);
+                 *outerDoConstruct, eval, accClauseList,
+                 mlir::acc::CombinedConstructsType::KernelsLoop);
   } else if (combinedDirective.v == llvm::acc::ACCD_parallel_loop) {
     createComputeOp<mlir::acc::ParallelOp>(
         converter, currentLocation, eval, semanticsContext, stmtCtx,
-        accClauseList, /*outerCombined=*/true);
+        accClauseList, mlir::acc::CombinedConstructsType::ParallelLoop);
     createLoopOp(converter, currentLocation, semanticsContext, stmtCtx,
-                 *outerDoConstruct, eval, accClauseList);
+                 *outerDoConstruct, eval, accClauseList,
+                 mlir::acc::CombinedConstructsType::ParallelLoop);
   } else if (combinedDirective.v == llvm::acc::ACCD_serial_loop) {
-    createComputeOp<mlir::acc::SerialOp>(converter, currentLocation, eval,
-                                         semanticsContext, stmtCtx,
-                                         accClauseList, /*outerCombined=*/true);
+    createComputeOp<mlir::acc::SerialOp>(
+        converter, currentLocation, eval, semanticsContext, stmtCtx,
+        accClauseList, mlir::acc::CombinedConstructsType::SerialLoop);
     createLoopOp(converter, currentLocation, semanticsContext, stmtCtx,
-                 *outerDoConstruct, eval, accClauseList);
+                 *outerDoConstruct, eval, accClauseList,
+                 mlir::acc::CombinedConstructsType::SerialLoop);
   } else {
     llvm::report_fatal_error("Unknown combined construct encountered");
   }
diff --git a/flang/test/Lower/OpenACC/acc-kernels-loop.f90 b/flang/test/Lower/OpenACC/acc-kernels-loop.f90
index e85065e54bc3..e5791f0e5b39 100644
--- a/flang/test/Lower/OpenACC/acc-kernels-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-kernels-loop.f90
@@ -37,13 +37,27 @@ subroutine acc_kernels_loop
 ! CHECK: %[[IFCONDITION:.*]] = fir.address_of(@{{.*}}ifcondition) : !fir.ref<!fir.logical<4>>
 ! CHECK: %[[DECLIFCONDITION:.*]]:2 = hlfir.declare %[[IFCONDITION]]
 
-  !$acc kernels loop
+  !$acc kernels
+  !$acc loop
   DO i = 1, n
     a(i) = b(i)
   END DO
+  !$acc end kernels
 
 ! CHECK:      acc.kernels {
-! CHECK:        acc.loop {{.*}} {
+! CHECK:        acc.loop private{{.*}} {
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.terminator
+! CHECK-NEXT: }{{$}}
+
+  !$acc kernels loop
+  DO i = 1, n
+    a(i) = b(i)
+  END DO
+
+! CHECK:      acc.kernels combined(loop) {
+! CHECK:        acc.loop combined(kernels) private{{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.terminator
@@ -55,7 +69,7 @@ subroutine acc_kernels_loop
   END DO
   !$acc end kernels loop
 
-! CHECK:      acc.kernels {
+! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -68,7 +82,7 @@ subroutine acc_kernels_loop
   END DO
 
 ! CHECK:      [[ASYNC1:%.*]] = arith.constant 1 : i32
-! CHECK:      acc.kernels async([[ASYNC1]] : i32) {
+! CHECK:      acc.kernels {{.*}} async([[ASYNC1]] : i32) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -81,7 +95,7 @@ subroutine acc_kernels_loop
   END DO
 
 ! CHECK:      [[ASYNC2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK:      acc.kernels async([[ASYNC2]] : i32) {
+! CHECK:      acc.kernels {{.*}} async([[ASYNC2]] : i32) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -93,7 +107,7 @@ subroutine acc_kernels_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.kernels wait {
+! CHECK:      acc.kernels {{.*}} wait {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -106,7 +120,7 @@ subroutine acc_kernels_loop
   END DO
 
 ! CHECK:      [[WAIT1:%.*]] = arith.constant 1 : i32
-! CHECK:      acc.kernels wait({[[WAIT1]] : i32}) {
+! CHECK:      acc.kernels {{.*}} wait({[[WAIT1]] : i32}) {
 ! CHECK:        acc.loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -120,7 +134,7 @@ subroutine acc_kernels_loop
 
 ! CHECK:      [[WAIT2:%.*]] = arith.constant 1 : i32
 ! CHECK:      [[WAIT3:%.*]] = arith.constant 2 : i32
-! CHECK:      acc.kernels wait({[[WAIT2]] : i32, [[WAIT3]] : i32}) {
+! CHECK:      acc.kernels {{.*}} wait({[[WAIT2]] : i32, [[WAIT3]] : i32}) {
 ! CHECK:        acc.loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -134,7 +148,7 @@ subroutine acc_kernels_loop
 
 ! CHECK:      [[WAIT4:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
 ! CHECK:      [[WAIT5:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK:      acc.kernels wait({[[WAIT4]] : i32, [[WAIT5]] : i32}) {
+! CHECK:      acc.kernels {{.*}} wait({[[WAIT4]] : i32, [[WAIT5]] : i32}) {
 ! CHECK:        acc.loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -147,7 +161,7 @@ subroutine acc_kernels_loop
   END DO
 
 ! CHECK:      [[NUMGANGS1:%.*]] = arith.constant 1 : i32
-! CHECK:      acc.kernels num_gangs({[[NUMGANGS1]] : i32}) {
+! CHECK:      acc.kernels {{.*}} num_gangs({[[NUMGANGS1]] : i32}) {
 ! CHECK:        acc.loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -160,7 +174,7 @@ subroutine acc_kernels_loop
   END DO
 
 ! CHECK:      [[NUMGANGS2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK:      acc.kernels num_gangs({[[NUMGANGS2]] : i32}) {
+! CHECK:      acc.kernels {{.*}} num_gangs({[[NUMGANGS2]] : i32}) {
 ! CHECK:        acc.loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -173,7 +187,7 @@ subroutine acc_kernels_loop
   END DO
 
 ! CHECK:      [[NUMWORKERS1:%.*]] = arith.constant 10 : i32
-! CHECK:      acc.kernels num_workers([[NUMWORKERS1]] : i32) {
+! CHECK:      acc.kernels {{.*}} num_workers([[NUMWORKERS1]] : i32) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -186,7 +200,7 @@ subroutine acc_kernels_loop
   END DO
 
 ! CHECK:      [[NUMWORKERS2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK:      acc.kernels num_workers([[NUMWORKERS2]] : i32) {
+! CHECK:      acc.kernels {{.*}} num_workers([[NUMWORKERS2]] : i32) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -199,7 +213,7 @@ subroutine acc_kernels_loop
   END DO
 
 ! CHECK:      [[VECTORLENGTH1:%.*]] = arith.constant 128 : i32
-! CHECK:      acc.kernels vector_length([[VECTORLENGTH1]] : i32) {
+! CHECK:      acc.kernels {{.*}} vector_length([[VECTORLENGTH1]] : i32) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -212,7 +226,7 @@ subroutine acc_kernels_loop
   END DO
 
 ! CHECK:      [[VECTORLENGTH2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK:      acc.kernels vector_length([[VECTORLENGTH2]] : i32) {
+! CHECK:      acc.kernels {{.*}} vector_length([[VECTORLENGTH2]] : i32) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -225,7 +239,7 @@ subroutine acc_kernels_loop
   END DO
 
 ! CHECK:      [[IF1:%.*]] = arith.constant true
-! CHECK:      acc.kernels if([[IF1]]) {
+! CHECK:      acc.kernels {{.*}} if([[IF1]]) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -239,7 +253,7 @@ subroutine acc_kernels_loop
 
 ! CHECK:      [[IFCOND:%.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
 ! CHECK:      [[IF2:%.*]] = fir.convert [[IFCOND]] : (!fir.logical<4>) -> i1
-! CHECK:      acc.kernels if([[IF2]]) {
+! CHECK:      acc.kernels {{.*}} if([[IF2]]) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -252,7 +266,7 @@ subroutine acc_kernels_loop
   END DO
 
 ! CHECK:      [[SELF1:%.*]] = arith.constant true
-! CHECK:      acc.kernels self([[SELF1]]) {
+! CHECK:      acc.kernels {{.*}} self([[SELF1]]) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -264,7 +278,7 @@ subroutine acc_kernels_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.kernels {
+! CHECK:      acc.kernels {{.*}}{
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -278,7 +292,7 @@ subroutine acc_kernels_loop
 
 
 ! CHECK:      %[[SELF2:.*]] = fir.convert %[[DECLIFCONDITION]]#1 : (!fir.ref<!fir.logical<4>>) -> i1
-! CHECK:      acc.kernels self(%[[SELF2]]) {
+! CHECK:      acc.kernels {{.*}} self(%[[SELF2]]) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -293,7 +307,7 @@ subroutine acc_kernels_loop
 
 ! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
 ! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:      acc.kernels dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.kernels {{.*}} dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -309,7 +323,7 @@ subroutine acc_kernels_loop
 
 ! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
 ! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:      acc.kernels dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.kernels {{.*}} dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -325,7 +339,7 @@ subroutine acc_kernels_loop
 
 ! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
-! CHECK:      acc.kernels dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.kernels {{.*}} dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -339,7 +353,7 @@ subroutine acc_kernels_loop
 
 ! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
 ! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "b"}
-! CHECK:      acc.kernels dataOperands(%[[CREATE_A]], %[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.kernels {{.*}} dataOperands(%[[CREATE_A]], %[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -355,7 +369,7 @@ subroutine acc_kernels_loop
 
 ! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
 ! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "a"}
-! CHECK:      acc.kernels dataOperands(%[[CREATE_B]], %[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.kernels {{.*}} dataOperands(%[[CREATE_B]], %[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -371,7 +385,7 @@ subroutine acc_kernels_loop
 
 ! CHECK:      %[[NOCREATE_A:.*]] = acc.nocreate varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK:      %[[NOCREATE_B:.*]] = acc.nocreate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      acc.kernels dataOperands(%[[NOCREATE_A]], %[[NOCREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.kernels {{.*}} dataOperands(%[[NOCREATE_A]], %[[NOCREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -385,7 +399,7 @@ subroutine acc_kernels_loop
 
 ! CHECK:      %[[PRESENT_A:.*]] = acc.present varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK:      %[[PRESENT_B:.*]] = acc.present varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      acc.kernels dataOperands(%[[PRESENT_A]], %[[PRESENT_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.kernels {{.*}} dataOperands(%[[PRESENT_A]], %[[PRESENT_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -399,7 +413,7 @@ subroutine acc_kernels_loop
 
 ! CHECK:      %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK:      %[[DEVICEPTR_B:.*]] = acc.deviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      acc.kernels dataOperands(%[[DEVICEPTR_A]], %[[DEVICEPTR_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.kernels {{.*}} dataOperands(%[[DEVICEPTR_A]], %[[DEVICEPTR_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -417,7 +431,7 @@ subroutine acc_kernels_loop
 ! CHECK:      %[[BOX_G:.*]] = fir.load %[[DECLG]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
 ! CHECK:      %[[BOX_ADDR_G:.*]] = fir.box_addr %[[BOX_G]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
 ! CHECK:      %[[ATTACH_G:.*]] = acc.attach varPtr(%[[BOX_ADDR_G]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "g"}
-! CHECK:      acc.kernels dataOperands(%[[ATTACH_F]], %[[ATTACH_G]] : !fir.ptr<f32>, !fir.ptr<f32>) {
+! CHECK:      acc.kernels {{.*}} dataOperands(%[[ATTACH_F]], %[[ATTACH_G]] : !fir.ptr<f32>, !fir.ptr<f32>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -429,7 +443,7 @@ subroutine acc_kernels_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.kernels {
+! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
@@ -441,7 +455,7 @@ subroutine acc_kernels_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.kernels {
+! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
@@ -453,7 +467,7 @@ subroutine acc_kernels_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.kernels {
+! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
@@ -465,8 +479,8 @@ subroutine acc_kernels_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.kernels {
-! CHECK:        acc.loop gang {{.*}} {
+! CHECK:      acc.kernels {{.*}} {
+! CHECK:        acc.loop {{.*}} gang {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>}{{$}}
 ! CHECK:        acc.terminator
@@ -477,9 +491,9 @@ subroutine acc_kernels_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.kernels {
+! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        [[GANGNUM1:%.*]] = arith.constant 8 : i32
-! CHECK-NEXT:   acc.loop gang({num=[[GANGNUM1]] : i32}) {{.*}} {
+! CHECK-NEXT:   acc.loop {{.*}} gang({num=[[GANGNUM1]] : i32}) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.terminator
@@ -490,9 +504,9 @@ subroutine acc_kernels_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.kernels {
+! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK-NEXT:   acc.loop gang({num=[[GANGNUM2]] : i32}) {{.*}} {
+! CHECK-NEXT:   acc.loop {{.*}} gang({num=[[GANGNUM2]] : i32}) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.terminator
@@ -503,8 +517,8 @@ subroutine acc_kernels_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.kernels {
-! CHECK:        acc.loop gang({num=%{{.*}} : i32, static=%{{.*}} : i32})
+! CHECK:      acc.kernels {{.*}} {
+! CHECK:        acc.loop {{.*}} gang({num=%{{.*}} : i32, static=%{{.*}} : i32})
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.terminator
@@ -514,8 +528,9 @@ subroutine acc_kernels_loop
   DO i = 1, n
     a(i) = b(i)
   END DO
-! CHECK:      acc.kernels {
-! CHECK:        acc.loop vector {{.*}} {
+
+! CHECK:      acc.kernels {{.*}} {
+! CHECK:        acc.loop {{.*}} vector {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>}{{$}}
 ! CHECK:        acc.terminator
@@ -526,9 +541,9 @@ subroutine acc_kernels_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.kernels {
+! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        [[CONSTANT128:%.*]] = arith.constant 128 : i32
-! CHECK:        acc.loop vector([[CONSTANT128]] : i32) {{.*}} {
+! CHECK:        acc.loop {{.*}} vector([[CONSTANT128]] : i32) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.terminator
@@ -539,9 +554,9 @@ subroutine acc_kernels_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.kernels {
+! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        [[VECTORLENGTH:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK:        acc.loop vector([[VECTORLENGTH]] : i32) {{.*}} {
+! CHECK:        acc.loop {{.*}} vector([[VECTORLENGTH]] : i32) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.terminator
@@ -552,8 +567,8 @@ subroutine acc_kernels_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.kernels {
-! CHECK:        acc.loop worker {{.*}} {
+! CHECK:      acc.kernels {{.*}} {
+! CHECK:        acc.loop {{.*}} worker {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>}{{$}}
 ! CHECK:        acc.terminator
@@ -564,9 +579,9 @@ subroutine acc_kernels_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.kernels {
+! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        [[WORKER128:%.*]] = arith.constant 128 : i32
-! CHECK:        acc.loop worker([[WORKER128]] : i32) {{.*}} {
+! CHECK:        acc.loop {{.*}} worker([[WORKER128]] : i32) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.terminator
@@ -579,7 +594,7 @@ subroutine acc_kernels_loop
     END DO
   END DO
 
-! CHECK:      acc.kernels {
+! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {collapse = [2], collapseDeviceType = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true, true>}
@@ -594,7 +609,7 @@ subroutine acc_kernels_loop
     END DO
   END DO
 
-! CHECK:      acc.kernels {
+! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:            acc.loop {{.*}} {
 ! CHECK:              acc.yield
@@ -609,7 +624,7 @@ subroutine acc_kernels_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.kernels {
+! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        [[TILESIZE:%.*]] = arith.constant 2 : i32
 ! CHECK:        acc.loop {{.*}} tile({[[TILESIZE]] : i32}) {{.*}} {
 ! CHECK:          acc.yield
@@ -622,7 +637,7 @@ subroutine acc_kernels_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.kernels {
+! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        [[TILESIZEM1:%.*]] = arith.constant -1 : i32
 ! CHECK:        acc.loop {{.*}} tile({[[TILESIZEM1]] : i32}) {{.*}} {
 ! CHECK:          acc.yield
@@ -637,7 +652,7 @@ subroutine acc_kernels_loop
     END DO
   END DO
 
-! CHECK:      acc.kernels {
+! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        [[TILESIZE1:%.*]] = arith.constant 2 : i32
 ! CHECK:        [[TILESIZE2:%.*]] = arith.constant 2 : i32
 ! CHECK:        acc.loop {{.*}} tile({[[TILESIZE1]] : i32, [[TILESIZE2]] : i32}) {{.*}} {
@@ -651,7 +666,7 @@ subroutine acc_kernels_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.kernels {
+! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        acc.loop {{.*}} tile({%{{.*}} : i32}) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -665,7 +680,7 @@ subroutine acc_kernels_loop
     END DO
   END DO
 
-! CHECK:      acc.kernels {
+! CHECK:      acc.kernels {{.*}} {
 ! CHECK:        acc.loop {{.*}} tile({%{{.*}} : i32, %{{.*}} : i32}) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -680,7 +695,7 @@ subroutine acc_kernels_loop
 
 ! CHECK:      %[[COPYINREDR:.*]] = acc.copyin varPtr(%{{.*}} : !fir.ref<f32>) -> !fir.ref<f32> {dataClause = #acc<data_clause acc_reduction>, implicit = true, name = "reduction_r"}
 ! CHECK:      %[[COPYINREDI:.*]] = acc.copyin varPtr(%{{.*}} : !fir.ref<i32>) -> !fir.ref<i32> {dataClause = #acc<data_clause acc_reduction>, implicit = true, name = "reduction_i"}
-! CHECK:      acc.kernels dataOperands(%[[COPYINREDR]], %[[COPYINREDI]] : !fir.ref<f32>, !fir.ref<i32>) {
+! CHECK:      acc.kernels {{.*}} dataOperands(%[[COPYINREDR]], %[[COPYINREDI]] : !fir.ref<f32>, !fir.ref<i32>) {
 ! CHECK:        acc.loop {{.*}} reduction(@reduction_add_ref_f32 -> %{{.*}} : !fir.ref<f32>, @reduction_mul_ref_i32 -> %{{.*}} : !fir.ref<i32>) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
diff --git a/flang/test/Lower/OpenACC/acc-loop.f90 b/flang/test/Lower/OpenACC/acc-loop.f90
index 1c64fab3536b..fa910e79bd76 100644
--- a/flang/test/Lower/OpenACC/acc-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-loop.f90
@@ -324,5 +324,5 @@ end subroutine
 ! CHECK: %[[P_I:.*]] = acc.private varPtr(%[[DC_I]] : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = ""}
 ! CHECK: %[[P_J:.*]] = acc.private varPtr(%[[DC_J]] : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = ""}
 ! CHECK: %[[P_K:.*]] = acc.private varPtr(%[[DC_K]] : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = ""}
-! CHECK: acc.loop private(@privatization_ref_i32 -> %[[P_I]] : !fir.ref<i32>, @privatization_ref_i32 -> %[[P_J]] : !fir.ref<i32>, @privatization_ref_i32 -> %[[P_K]] : !fir.ref<i32>) control(%{{.*}} : i32, %{{.*}} : i32, %{{.*}} : i32) = (%c1{{.*}}, %c1{{.*}}, %c1{{.*}} : i32, i32, i32) to (%c10{{.*}}, %c100{{.*}}, %c200{{.*}} : i32, i32, i32)  step (%c1{{.*}}, %c1{{.*}}, %c1{{.*}} : i32, i32, i32)
+! CHECK: acc.loop combined(parallel) private(@privatization_ref_i32 -> %[[P_I]] : !fir.ref<i32>, @privatization_ref_i32 -> %[[P_J]] : !fir.ref<i32>, @privatization_ref_i32 -> %[[P_K]] : !fir.ref<i32>) control(%{{.*}} : i32, %{{.*}} : i32, %{{.*}} : i32) = (%c1{{.*}}, %c1{{.*}}, %c1{{.*}} : i32, i32, i32) to (%c10{{.*}}, %c100{{.*}}, %c200{{.*}} : i32, i32, i32)  step (%c1{{.*}}, %c1{{.*}}, %c1{{.*}} : i32, i32, i32)
 ! CHECK: } attributes {inclusiveUpperbound = array<i1: true, true, true>}
diff --git a/flang/test/Lower/OpenACC/acc-parallel-loop.f90 b/flang/test/Lower/OpenACC/acc-parallel-loop.f90
index 39f18307486a..48ceda0710e8 100644
--- a/flang/test/Lower/OpenACC/acc-parallel-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-parallel-loop.f90
@@ -39,13 +39,27 @@ subroutine acc_parallel_loop
 ! CHECK: %[[IFCONDITION:.*]] = fir.address_of(@{{.*}}ifcondition) : !fir.ref<!fir.logical<4>>
 ! CHECK: %[[DECLIFCONDITION:.*]]:2 = hlfir.declare %[[IFCONDITION]]
 
-  !$acc parallel loop
+  !$acc parallel
+  !$acc loop
   DO i = 1, n
     a(i) = b(i)
   END DO
+  !$acc end parallel
 
 ! CHECK:      acc.parallel {
-! CHECK:        acc.loop {{.*}} {
+! CHECK:        acc.loop private{{.*}} {
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
+
+  !$acc parallel loop
+  DO i = 1, n
+    a(i) = b(i)
+  END DO
+
+! CHECK:      acc.parallel combined(loop) {
+! CHECK:        acc.loop combined(parallel) private{{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
@@ -57,7 +71,7 @@ subroutine acc_parallel_loop
   END DO
   !$acc end parallel loop
 
-! CHECK:      acc.parallel {
+! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -70,7 +84,7 @@ subroutine acc_parallel_loop
   END DO
 
 ! CHECK:      [[ASYNC1:%.*]] = arith.constant 1 : i32
-! CHECK:      acc.parallel async([[ASYNC1]] : i32) {
+! CHECK:      acc.parallel {{.*}} async([[ASYNC1]] : i32) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -83,7 +97,7 @@ subroutine acc_parallel_loop
   END DO
 
 ! CHECK:      [[ASYNC2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK:      acc.parallel async([[ASYNC2]] : i32) {
+! CHECK:      acc.parallel {{.*}} async([[ASYNC2]] : i32) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -95,7 +109,7 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.parallel wait {
+! CHECK:      acc.parallel {{.*}} wait {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -108,7 +122,7 @@ subroutine acc_parallel_loop
   END DO
 
 ! CHECK:      [[WAIT1:%.*]] = arith.constant 1 : i32
-! CHECK:      acc.parallel wait({[[WAIT1]] : i32}) {
+! CHECK:      acc.parallel {{.*}} wait({[[WAIT1]] : i32}) {
 ! CHECK:        acc.loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -122,7 +136,7 @@ subroutine acc_parallel_loop
 
 ! CHECK:      [[WAIT2:%.*]] = arith.constant 1 : i32
 ! CHECK:      [[WAIT3:%.*]] = arith.constant 2 : i32
-! CHECK:      acc.parallel wait({[[WAIT2]] : i32, [[WAIT3]] : i32}) {
+! CHECK:      acc.parallel {{.*}} wait({[[WAIT2]] : i32, [[WAIT3]] : i32}) {
 ! CHECK:        acc.loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -136,7 +150,7 @@ subroutine acc_parallel_loop
 
 ! CHECK:      [[WAIT4:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
 ! CHECK:      [[WAIT5:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK:      acc.parallel wait({[[WAIT4]] : i32, [[WAIT5]] : i32}) {
+! CHECK:      acc.parallel {{.*}} wait({[[WAIT4]] : i32, [[WAIT5]] : i32}) {
 ! CHECK:        acc.loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -149,7 +163,7 @@ subroutine acc_parallel_loop
   END DO
 
 ! CHECK:      [[NUMGANGS1:%.*]] = arith.constant 1 : i32
-! CHECK:      acc.parallel num_gangs({[[NUMGANGS1]] : i32}) {
+! CHECK:      acc.parallel {{.*}} num_gangs({[[NUMGANGS1]] : i32}) {
 ! CHECK:        acc.loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -162,7 +176,7 @@ subroutine acc_parallel_loop
   END DO
 
 ! CHECK:      [[NUMGANGS2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK:      acc.parallel num_gangs({[[NUMGANGS2]] : i32}) {
+! CHECK:      acc.parallel {{.*}} num_gangs({[[NUMGANGS2]] : i32}) {
 ! CHECK:        acc.loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -175,7 +189,7 @@ subroutine acc_parallel_loop
   END DO
 
 ! CHECK:      [[NUMWORKERS1:%.*]] = arith.constant 10 : i32
-! CHECK:      acc.parallel num_workers([[NUMWORKERS1]] : i32) {
+! CHECK:      acc.parallel {{.*}} num_workers([[NUMWORKERS1]] : i32) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -188,7 +202,7 @@ subroutine acc_parallel_loop
   END DO
 
 ! CHECK:      [[NUMWORKERS2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK:      acc.parallel num_workers([[NUMWORKERS2]] : i32) {
+! CHECK:      acc.parallel {{.*}} num_workers([[NUMWORKERS2]] : i32) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -201,7 +215,7 @@ subroutine acc_parallel_loop
   END DO
 
 ! CHECK:      [[VECTORLENGTH1:%.*]] = arith.constant 128 : i32
-! CHECK:      acc.parallel vector_length([[VECTORLENGTH1]] : i32) {
+! CHECK:      acc.parallel {{.*}} vector_length([[VECTORLENGTH1]] : i32) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -214,7 +228,7 @@ subroutine acc_parallel_loop
   END DO
 
 ! CHECK:      [[VECTORLENGTH2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK:      acc.parallel vector_length([[VECTORLENGTH2]] : i32) {
+! CHECK:      acc.parallel {{.*}} vector_length([[VECTORLENGTH2]] : i32) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -227,7 +241,7 @@ subroutine acc_parallel_loop
   END DO
 
 ! CHECK:      [[IF1:%.*]] = arith.constant true
-! CHECK:      acc.parallel if([[IF1]]) {
+! CHECK:      acc.parallel {{.*}} if([[IF1]]) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -241,7 +255,7 @@ subroutine acc_parallel_loop
 
 ! CHECK:      [[IFCOND:%.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
 ! CHECK:      [[IF2:%.*]] = fir.convert [[IFCOND]] : (!fir.logical<4>) -> i1
-! CHECK:      acc.parallel if([[IF2]]) {
+! CHECK:      acc.parallel {{.*}} if([[IF2]]) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -254,7 +268,7 @@ subroutine acc_parallel_loop
   END DO
 
 ! CHECK:      [[SELF1:%.*]] = arith.constant true
-! CHECK:      acc.parallel self([[SELF1]]) {
+! CHECK:      acc.parallel {{.*}} self([[SELF1]]) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -266,7 +280,7 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.parallel {
+! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -279,7 +293,7 @@ subroutine acc_parallel_loop
   END DO
 
 ! CHECK:      %[[SELF2:.*]] = fir.convert %[[DECLIFCONDITION]]#1 : (!fir.ref<!fir.logical<4>>) -> i1
-! CHECK:      acc.parallel self(%[[SELF2]]) {
+! CHECK:      acc.parallel {{.*}} self(%[[SELF2]]) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -293,7 +307,7 @@ subroutine acc_parallel_loop
 
 ! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
 ! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:      acc.parallel dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.parallel {{.*}} dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -309,7 +323,7 @@ subroutine acc_parallel_loop
 
 ! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
 ! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:      acc.parallel dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.parallel {{.*}} dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -325,7 +339,7 @@ subroutine acc_parallel_loop
 
 ! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
-! CHECK:      acc.parallel dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.parallel {{.*}} dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -339,7 +353,7 @@ subroutine acc_parallel_loop
 
 ! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
 ! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "b"}
-! CHECK:      acc.parallel dataOperands(%[[CREATE_A]], %[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.parallel {{.*}} dataOperands(%[[CREATE_A]], %[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -355,7 +369,7 @@ subroutine acc_parallel_loop
 
 ! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
 ! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "a"}
-! CHECK:      acc.parallel dataOperands(%[[CREATE_B]], %[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.parallel {{.*}} dataOperands(%[[CREATE_B]], %[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -371,7 +385,7 @@ subroutine acc_parallel_loop
 
 ! CHECK:      %[[NOCREATE_A:.*]] = acc.nocreate varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK:      %[[NOCREATE_B:.*]] = acc.nocreate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      acc.parallel dataOperands(%[[NOCREATE_A]], %[[NOCREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.parallel {{.*}} dataOperands(%[[NOCREATE_A]], %[[NOCREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -385,7 +399,7 @@ subroutine acc_parallel_loop
 
 ! CHECK:      %[[PRESENT_A:.*]] = acc.present varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK:      %[[PRESENT_B:.*]] = acc.present varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      acc.parallel dataOperands(%[[PRESENT_A]], %[[PRESENT_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.parallel {{.*}} dataOperands(%[[PRESENT_A]], %[[PRESENT_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -399,7 +413,7 @@ subroutine acc_parallel_loop
 
 ! CHECK:      %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK:      %[[DEVICEPTR_B:.*]] = acc.deviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      acc.parallel dataOperands(%[[DEVICEPTR_A]], %[[DEVICEPTR_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.parallel {{.*}} dataOperands(%[[DEVICEPTR_A]], %[[DEVICEPTR_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -417,7 +431,7 @@ subroutine acc_parallel_loop
 ! CHECK:      %[[BOX_G:.*]] = fir.load %[[DECLG]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
 ! CHECK:      %[[BOX_ADDR_G:.*]] = fir.box_addr %[[BOX_G]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
 ! CHECK:      %[[ATTACH_G:.*]] = acc.attach varPtr(%[[BOX_ADDR_G]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "g"}
-! CHECK:      acc.parallel dataOperands(%[[ATTACH_F]], %[[ATTACH_G]] : !fir.ptr<f32>, !fir.ptr<f32>) {
+! CHECK:      acc.parallel {{.*}} dataOperands(%[[ATTACH_F]], %[[ATTACH_G]] : !fir.ptr<f32>, !fir.ptr<f32>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -430,9 +444,9 @@ subroutine acc_parallel_loop
   END DO
 
 ! CHECK:      %[[ACC_PRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      acc.parallel firstprivate(@firstprivatization_section_ext10_ref_10xf32 -> %[[ACC_PRIVATE_B]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.parallel {{.*}} firstprivate(@firstprivatization_section_ext10_ref_10xf32 -> %[[ACC_PRIVATE_B]] : !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK:        acc.loop private({{.*}}@privatization_ref_10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK:        acc.loop {{.*}} private({{.*}}@privatization_ref_10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10xf32>>)
 ! CHECK-NOT:      fir.do_loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -444,7 +458,7 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.parallel {
+! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
@@ -456,7 +470,7 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.parallel {
+! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
@@ -468,7 +482,7 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.parallel {
+! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
@@ -480,8 +494,8 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.parallel {
-! CHECK:        acc.loop gang
+! CHECK:      acc.parallel {{.*}} {
+! CHECK:        acc.loop {{.*}} gang
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>}{{$}}
 ! CHECK:        acc.yield
@@ -492,9 +506,9 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.parallel {
+! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        [[GANGNUM1:%.*]] = arith.constant 8 : i32
-! CHECK-NEXT:   acc.loop gang({num=[[GANGNUM1]] : i32})
+! CHECK-NEXT:   acc.loop {{.*}} gang({num=[[GANGNUM1]] : i32})
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
@@ -505,9 +519,9 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.parallel {
+! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK-NEXT:   acc.loop gang({num=[[GANGNUM2]] : i32})
+! CHECK-NEXT:   acc.loop {{.*}} gang({num=[[GANGNUM2]] : i32})
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
@@ -518,8 +532,8 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.parallel {
-! CHECK:        acc.loop gang({num=%{{.*}} : i32, static=%{{.*}} : i32})
+! CHECK:      acc.parallel {{.*}} {
+! CHECK:        acc.loop {{.*}} gang({num=%{{.*}} : i32, static=%{{.*}} : i32})
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
@@ -529,8 +543,9 @@ subroutine acc_parallel_loop
   DO i = 1, n
     a(i) = b(i)
   END DO
-! CHECK:      acc.parallel {
-! CHECK:        acc.loop vector
+
+! CHECK:      acc.parallel {{.*}} {
+! CHECK:        acc.loop {{.*}} vector
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>}{{$}}
 ! CHECK:        acc.yield
@@ -541,9 +556,9 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.parallel {
+! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        [[CONSTANT128:%.*]] = arith.constant 128 : i32
-! CHECK:        acc.loop vector([[CONSTANT128]] : i32) {{.*}} {
+! CHECK:        acc.loop {{.*}} vector([[CONSTANT128]] : i32) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
@@ -554,9 +569,9 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.parallel {
+! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        [[VECTORLENGTH:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK:        acc.loop vector([[VECTORLENGTH]] : i32) {{.*}} {
+! CHECK:        acc.loop {{.*}} vector([[VECTORLENGTH]] : i32) {{.*}} {
 ! CHECK-NOT:      fir.do_loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -568,8 +583,8 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.parallel {
-! CHECK:        acc.loop worker {{.*}} {
+! CHECK:      acc.parallel {{.*}} {
+! CHECK:        acc.loop {{.*}} worker {{.*}} {
 ! CHECK-NOT:      fir.do_loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>}{{$}}
@@ -581,9 +596,9 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.parallel {
+! CHECK:      acc.parallel {{.*}}{
 ! CHECK:        [[WORKER128:%.*]] = arith.constant 128 : i32
-! CHECK:        acc.loop worker([[WORKER128]] : i32) {{.*}} {
+! CHECK:        acc.loop {{.*}} worker([[WORKER128]] : i32) {{.*}} {
 ! CHECK-NOT:      fir.do_loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -597,7 +612,7 @@ subroutine acc_parallel_loop
     END DO
   END DO
 
-! CHECK:      acc.parallel {
+! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {collapse = [2], collapseDeviceType = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true, true>}
@@ -612,7 +627,7 @@ subroutine acc_parallel_loop
     END DO
   END DO
 
-! CHECK:      acc.parallel {
+! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:            acc.loop {{.*}} {
 ! CHECK:              acc.yield
@@ -627,7 +642,7 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.parallel {
+! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        [[TILESIZE:%.*]] = arith.constant 2 : i32
 ! CHECK:        acc.loop {{.*}} tile({[[TILESIZE]] : i32}) {{.*}} {
 ! CHECK:          acc.yield
@@ -640,7 +655,7 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.parallel {
+! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        [[TILESIZEM1:%.*]] = arith.constant -1 : i32
 ! CHECK:        acc.loop {{.*}} tile({[[TILESIZEM1]] : i32}) {{.*}} {
 ! CHECK:          acc.yield
@@ -655,7 +670,7 @@ subroutine acc_parallel_loop
     END DO
   END DO
 
-! CHECK:      acc.parallel {
+! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        [[TILESIZE1:%.*]] = arith.constant 2 : i32
 ! CHECK:        [[TILESIZE2:%.*]] = arith.constant 2 : i32
 ! CHECK:        acc.loop {{.*}} tile({[[TILESIZE1]] : i32, [[TILESIZE2]] : i32}) {{.*}} {
@@ -669,7 +684,7 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.parallel {
+! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        acc.loop {{.*}} tile({%{{.*}} : i32}) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -683,7 +698,7 @@ subroutine acc_parallel_loop
     END DO
   END DO
 
-! CHECK:      acc.parallel {
+! CHECK:      acc.parallel {{.*}} {
 ! CHECK:        acc.loop {{.*}} tile({%{{.*}} : i32, %{{.*}} : i32}) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -698,7 +713,7 @@ subroutine acc_parallel_loop
 
 ! CHECK:      %[[COPYINREDR:.*]] = acc.copyin varPtr(%{{.*}} : !fir.ref<f32>) -> !fir.ref<f32> {dataClause = #acc<data_clause acc_reduction>, implicit = true, name = "reduction_r"}
 ! CHECK:      %[[COPYINREDI:.*]] = acc.copyin varPtr(%{{.*}} : !fir.ref<i32>) -> !fir.ref<i32> {dataClause = #acc<data_clause acc_reduction>, implicit = true, name = "reduction_i"}
-! CHECK:      acc.parallel dataOperands(%[[COPYINREDR]], %[[COPYINREDI]] : !fir.ref<f32>, !fir.ref<i32>) {
+! CHECK:      acc.parallel {{.*}} dataOperands(%[[COPYINREDR]], %[[COPYINREDI]] : !fir.ref<f32>, !fir.ref<i32>) {
 ! CHECK:        acc.loop {{.*}} reduction(@reduction_add_ref_f32 -> %{{.*}} : !fir.ref<f32>, @reduction_mul_ref_i32 -> %{{.*}} : !fir.ref<i32>) {{.*}}
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
diff --git a/flang/test/Lower/OpenACC/acc-private.f90 b/flang/test/Lower/OpenACC/acc-private.f90
index ba582d40f0f5..4d9f84b1fa74 100644
--- a/flang/test/Lower/OpenACC/acc-private.f90
+++ b/flang/test/Lower/OpenACC/acc-private.f90
@@ -228,7 +228,7 @@ program acc_private
   END DO
 
 ! CHECK: %[[FP_C:.*]] = acc.firstprivate varPtr(%[[DECLC]]#0 : !fir.ref<i32>)   -> !fir.ref<i32> {name = "c"}
-! CHECK: acc.parallel firstprivate(@firstprivatization_ref_i32 -> %[[FP_C]] : !fir.ref<i32>)
+! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_ref_i32 -> %[[FP_C]] : !fir.ref<i32>)
 ! CHECK: acc.yield
 
   !$acc parallel loop firstprivate(b)
@@ -242,7 +242,7 @@ program acc_private
 ! CHECK: %[[UB:.*]] = arith.subi %{{.*}}, %[[C1]] : index
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
 ! CHECK: %[[FP_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xf32>> {name = "b"}
-! CHECK: acc.parallel firstprivate(@firstprivatization_section_ext100_ref_100xf32 -> %[[FP_B]] : !fir.ref<!fir.array<100xf32>>)
+! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_section_ext100_ref_100xf32 -> %[[FP_B]] : !fir.ref<!fir.array<100xf32>>)
 ! CHECK: acc.yield
 
   !$acc parallel loop firstprivate(b(51:100))
@@ -256,7 +256,7 @@ program acc_private
 ! CHECK: %[[UB:.*]] = arith.constant 99 : index
 ! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%{{.*}} : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
 ! CHECK: %[[FP_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<100xf32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<50xf32>> {name = "b(51:100)"}
-! CHECK: acc.parallel firstprivate(@firstprivatization_section_lb50.ub99_ref_50xf32 -> %[[FP_B]] : !fir.ref<!fir.array<50xf32>>)
+! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_section_lb50.ub99_ref_50xf32 -> %[[FP_B]] : !fir.ref<!fir.array<50xf32>>)
 
 end program
 
@@ -272,10 +272,10 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPacc_private_assumed_shape(
 ! CHECK-SAME:    %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]] {uniq_name = "_QFacc_private_assumed_shapeEa"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
-! CHECK: acc.parallel {
+! CHECK: acc.parallel {{.*}} {
 ! CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECL_A]]#0 : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
 ! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[ADDR]] : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<?xi32>> {name = "a"}
-! CHECK: acc.loop private({{.*}}@privatization_box_Uxi32 -> %[[PRIVATE]] : !fir.ref<!fir.array<?xi32>>)
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_Uxi32 -> %[[PRIVATE]] : !fir.ref<!fir.array<?xi32>>)
 
 subroutine acc_private_allocatable_array(a, n)
   integer, allocatable :: a(:)
@@ -294,11 +294,11 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPacc_private_allocatable_array(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {fir.bindc_name = "a"}
 ! CHECK: %[[DECLA_A:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFacc_private_allocatable_arrayEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
-! CHECK: acc.parallel {
+! CHECK: acc.parallel {{.*}} {
 ! CHECK: %[[BOX:.*]] = fir.load %[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
 ! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.heap<!fir.array<?xi32>> {name = "a"}
-! CHECK: acc.loop private({{.*}}@privatization_box_heap_Uxi32 -> %[[PRIVATE]] : !fir.heap<!fir.array<?xi32>>)
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_heap_Uxi32 -> %[[PRIVATE]] : !fir.heap<!fir.array<?xi32>>)
 ! CHECK: acc.serial private(@privatization_box_heap_Uxi32 -> %{{.*}} : !fir.heap<!fir.array<?xi32>>)
 
 subroutine acc_private_pointer_array(a, n)
@@ -314,11 +314,11 @@ end subroutine
 ! CHECK-LABEL: func.func @_QPacc_private_pointer_array(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> {fir.bindc_name = "a"}, %arg1: !fir.ref<i32> {fir.bindc_name = "n"}) {
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %arg0 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFacc_private_pointer_arrayEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
-! CHECK: acc.parallel {
+! CHECK: acc.parallel {{.*}} {
 ! CHECK: %[[BOX:.*]] = fir.load %[[DECLA_A]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>) -> !fir.ptr<!fir.array<?xi32>>
 ! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.ptr<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ptr<!fir.array<?xi32>> {name = "a"}
-! CHECK: acc.loop private({{.*}}@privatization_box_ptr_Uxi32 -> %[[PRIVATE]] : !fir.ptr<!fir.array<?xi32>>)
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_ptr_Uxi32 -> %[[PRIVATE]] : !fir.ptr<!fir.array<?xi32>>)
 
 subroutine acc_private_dynamic_extent(a, n)
   integer :: n, i
@@ -334,10 +334,10 @@ end subroutine
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<?x?x2xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) {
 ! CHECK: %[[DECL_N:.*]]:2 = hlfir.declare %[[ARG1]] {uniq_name = "_QFacc_private_dynamic_extentEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) {uniq_name = "_QFacc_private_dynamic_extentEa"} : (!fir.ref<!fir.array<?x?x2xi32>>, !fir.shape<3>) -> (!fir.box<!fir.array<?x?x2xi32>>, !fir.ref<!fir.array<?x?x2xi32>>)
-! CHECK: acc.parallel {
+! CHECK: acc.parallel {{.*}} {
 ! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[DECL_A]]#0 : (!fir.box<!fir.array<?x?x2xi32>>) -> !fir.ref<!fir.array<?x?x2xi32>>
 ! CHECK: %[[PRIV:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.ref<!fir.array<?x?x2xi32>>) bounds(%{{.*}}, %{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<?x?x2xi32>> {name = "a"}
-! CHECK: acc.loop private({{.*}}@privatization_box_UxUx2xi32 -> %[[PRIV]] : !fir.ref<!fir.array<?x?x2xi32>>)
+! CHECK: acc.loop {{.*}} private({{.*}}@privatization_box_UxUx2xi32 -> %[[PRIV]] : !fir.ref<!fir.array<?x?x2xi32>>)
 
 subroutine acc_firstprivate_assumed_shape(a, n)
   integer :: a(:), i, n
@@ -367,7 +367,7 @@ subroutine acc_firstprivate_dynamic_extent(a, n)
   end do
 end subroutine
 
-! CHECK: acc.parallel firstprivate(@firstprivatization_box_UxUx2xi32 -> %{{.*}} : !fir.ref<!fir.array<?x?x2xi32>>)
+! CHECK: acc.parallel {{.*}} firstprivate(@firstprivatization_box_UxUx2xi32 -> %{{.*}} : !fir.ref<!fir.array<?x?x2xi32>>)
 
 module acc_declare_equivalent
   integer, parameter :: n = 10
@@ -398,6 +398,6 @@ end
 ! CHECK: acc.parallel
 ! CHECK: %[[PRIV_I:.*]] = acc.private varPtr(%[[DECL_I]]#1 : !fir.ref<i32>) -> !fir.ref<i32> {implicit = true, name = ""}
 ! CHECK: %[[DECL_PRIV_I:.*]]:2 = hlfir.declare %[[PRIV_I]] {uniq_name = "_QFacc_private_useEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-! CHECK: acc.loop private(@privatization_ref_i32 -> %[[PRIV_I]] : !fir.ref<i32>) control(%[[IV0:.*]] : i32) = (%c1{{.*}} : i32) to (%c10{{.*}} : i32) step (%c1{{.*}} : i32)
+! CHECK: acc.loop {{.*}} private(@privatization_ref_i32 -> %[[PRIV_I]] : !fir.ref<i32>) control(%[[IV0:.*]] : i32) = (%c1{{.*}} : i32) to (%c10{{.*}} : i32) step (%c1{{.*}} : i32)
 ! CHECK:   fir.store %[[IV0]] to %[[DECL_PRIV_I]]#0 : !fir.ref<i32>
 ! CHECK:   %{{.*}} = fir.load %[[DECL_PRIV_I]]#0 : !fir.ref<i32>
diff --git a/flang/test/Lower/OpenACC/acc-serial-loop.f90 b/flang/test/Lower/OpenACC/acc-serial-loop.f90
index 77f5e1eaa7c8..fa3b3f758908 100644
--- a/flang/test/Lower/OpenACC/acc-serial-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-serial-loop.f90
@@ -58,13 +58,27 @@ subroutine acc_serial_loop
 ! CHECK: %[[IFCONDITION:.*]] = fir.address_of(@{{.*}}ifcondition) : !fir.ref<!fir.logical<4>>
 ! CHECK: %[[DECLIFCONDITION:.*]]:2 = hlfir.declare %[[IFCONDITION]]
 
-  !$acc serial loop
+  !$acc serial
+  !$acc loop
   DO i = 1, n
     a(i) = b(i)
   END DO
+  !$acc end serial
 
 ! CHECK:      acc.serial {
-! CHECK:        acc.loop {{.*}} {
+! CHECK:        acc.loop private{{.*}} {
+! CHECK:          acc.yield
+! CHECK-NEXT:   }{{$}}
+! CHECK:        acc.yield
+! CHECK-NEXT: }{{$}}
+
+  !$acc serial loop
+  DO i = 1, n
+    a(i) = b(i)
+  END DO
+
+! CHECK:      acc.serial combined(loop) {
+! CHECK:        acc.loop combined(serial) private{{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
@@ -76,7 +90,7 @@ subroutine acc_serial_loop
   END DO
   !$acc end serial loop
 
-! CHECK:      acc.serial {
+! CHECK:      acc.serial {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -89,7 +103,7 @@ subroutine acc_serial_loop
   END DO
 
 ! CHECK:      [[ASYNC1:%.*]] = arith.constant 1 : i32
-! CHECK:      acc.serial async([[ASYNC1]] : i32) {
+! CHECK:      acc.serial {{.*}} async([[ASYNC1]] : i32) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -102,7 +116,7 @@ subroutine acc_serial_loop
   END DO
 
 ! CHECK:      [[ASYNC2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK:      acc.serial async([[ASYNC2]] : i32) {
+! CHECK:      acc.serial {{.*}} async([[ASYNC2]] : i32) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -114,7 +128,7 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.serial wait {
+! CHECK:      acc.serial {{.*}} wait {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -127,7 +141,7 @@ subroutine acc_serial_loop
   END DO
 
 ! CHECK:      [[WAIT1:%.*]] = arith.constant 1 : i32
-! CHECK:      acc.serial wait({[[WAIT1]] : i32}) {
+! CHECK:      acc.serial {{.*}} wait({[[WAIT1]] : i32}) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -141,7 +155,7 @@ subroutine acc_serial_loop
 
 ! CHECK:      [[WAIT2:%.*]] = arith.constant 1 : i32
 ! CHECK:      [[WAIT3:%.*]] = arith.constant 2 : i32
-! CHECK:      acc.serial wait({[[WAIT2]] : i32, [[WAIT3]] : i32}) {
+! CHECK:      acc.serial {{.*}} wait({[[WAIT2]] : i32, [[WAIT3]] : i32}) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -155,7 +169,7 @@ subroutine acc_serial_loop
 
 ! CHECK:      [[WAIT4:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
 ! CHECK:      [[WAIT5:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK:      acc.serial wait({[[WAIT4]] : i32, [[WAIT5]] : i32}) {
+! CHECK:      acc.serial {{.*}} wait({[[WAIT4]] : i32, [[WAIT5]] : i32}) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -168,7 +182,7 @@ subroutine acc_serial_loop
   END DO
 
 ! CHECK:      [[IF1:%.*]] = arith.constant true
-! CHECK:      acc.serial if([[IF1]]) {
+! CHECK:      acc.serial {{.*}} if([[IF1]]) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -182,7 +196,7 @@ subroutine acc_serial_loop
 
 ! CHECK:      [[IFCOND:%.*]] = fir.load %{{.*}} : !fir.ref<!fir.logical<4>>
 ! CHECK:      [[IF2:%.*]] = fir.convert [[IFCOND]] : (!fir.logical<4>) -> i1
-! CHECK:      acc.serial if([[IF2]]) {
+! CHECK:      acc.serial {{.*}} if([[IF2]]) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -195,7 +209,7 @@ subroutine acc_serial_loop
   END DO
 
 ! CHECK:      [[SELF1:%.*]] = arith.constant true
-! CHECK:      acc.serial self([[SELF1]]) {
+! CHECK:      acc.serial {{.*}} self([[SELF1]]) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -207,7 +221,7 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.serial {
+! CHECK:      acc.serial {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -220,7 +234,7 @@ subroutine acc_serial_loop
   END DO
 
 ! CHECK:      %[[SELF2:.*]] = fir.convert %[[DECLIFCONDITION]]#1 : (!fir.ref<!fir.logical<4>>) -> i1
-! CHECK:      acc.serial self(%[[SELF2]]) {
+! CHECK:      acc.serial {{.*}} self(%[[SELF2]]) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -234,7 +248,7 @@ subroutine acc_serial_loop
 
 ! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
 ! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:      acc.serial dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.serial {{.*}} dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -250,7 +264,7 @@ subroutine acc_serial_loop
 
 ! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "a"}
 ! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copy>, name = "b"}
-! CHECK:      acc.serial dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.serial {{.*}} dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -266,7 +280,7 @@ subroutine acc_serial_loop
 
 ! CHECK:      %[[COPYIN_A:.*]] = acc.copyin varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK:      %[[COPYIN_B:.*]] = acc.copyin varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyin_readonly>, name = "b"}
-! CHECK:      acc.serial dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.serial {{.*}} dataOperands(%[[COPYIN_A]], %[[COPYIN_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -280,7 +294,7 @@ subroutine acc_serial_loop
 
 ! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "a"}
 ! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_copyout>, name = "b"}
-! CHECK:      acc.serial dataOperands(%[[CREATE_A]], %[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.serial {{.*}} dataOperands(%[[CREATE_A]], %[[CREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -296,7 +310,7 @@ subroutine acc_serial_loop
 
 ! CHECK:      %[[CREATE_B:.*]] = acc.create varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
 ! CHECK:      %[[CREATE_A:.*]] = acc.create varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {dataClause = #acc<data_clause acc_create_zero>, name = "a"}
-! CHECK:      acc.serial dataOperands(%[[CREATE_B]], %[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.serial {{.*}} dataOperands(%[[CREATE_B]], %[[CREATE_A]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -312,7 +326,7 @@ subroutine acc_serial_loop
 
 ! CHECK:      %[[NOCREATE_A:.*]] = acc.nocreate varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK:      %[[NOCREATE_B:.*]] = acc.nocreate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      acc.serial dataOperands(%[[NOCREATE_A]], %[[NOCREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.serial {{.*}} dataOperands(%[[NOCREATE_A]], %[[NOCREATE_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -326,7 +340,7 @@ subroutine acc_serial_loop
 
 ! CHECK:      %[[PRESENT_A:.*]] = acc.present varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK:      %[[PRESENT_B:.*]] = acc.present varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      acc.serial dataOperands(%[[PRESENT_A]], %[[PRESENT_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.serial {{.*}} dataOperands(%[[PRESENT_A]], %[[PRESENT_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -340,7 +354,7 @@ subroutine acc_serial_loop
 
 ! CHECK:      %[[DEVICEPTR_A:.*]] = acc.deviceptr varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK:      %[[DEVICEPTR_B:.*]] = acc.deviceptr varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      acc.serial dataOperands(%[[DEVICEPTR_A]], %[[DEVICEPTR_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.serial {{.*}} dataOperands(%[[DEVICEPTR_A]], %[[DEVICEPTR_B]] : !fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -358,7 +372,7 @@ subroutine acc_serial_loop
 ! CHECK:      %[[BOX_G:.*]] = fir.load %[[DECLG]]#0 : !fir.ref<!fir.box<!fir.ptr<f32>>>
 ! CHECK:      %[[BOX_ADDR_G:.*]] = fir.box_addr %[[BOX_G]] : (!fir.box<!fir.ptr<f32>>) -> !fir.ptr<f32>
 ! CHECK:      %[[ATTACH_G:.*]] = acc.attach varPtr(%[[BOX_ADDR_G]] : !fir.ptr<f32>) -> !fir.ptr<f32> {name = "g"}
-! CHECK:      acc.serial dataOperands(%[[ATTACH_F]], %[[ATTACH_G]] : !fir.ptr<f32>, !fir.ptr<f32>) {
+! CHECK:      acc.serial {{.*}} dataOperands(%[[ATTACH_F]], %[[ATTACH_G]] : !fir.ptr<f32>, !fir.ptr<f32>) {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -371,9 +385,9 @@ subroutine acc_serial_loop
   END DO
 
 ! CHECK:      %[[ACC_FPRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      acc.serial firstprivate(@firstprivatization_section_ext10_ref_10xf32 -> %[[ACC_FPRIVATE_B]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.serial {{.*}} firstprivate(@firstprivatization_section_ext10_ref_10xf32 -> %[[ACC_FPRIVATE_B]] : !fir.ref<!fir.array<10xf32>>) {
 ! CHECK:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#0 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! CHECK:        acc.loop private({{.*}}@privatization_ref_10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10xf32>>)
+! CHECK:        acc.loop {{.*}} private({{.*}}@privatization_ref_10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10xf32>>)
 ! CHECK-NOT:          fir.do_loop
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -385,7 +399,7 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.serial {
+! CHECK:      acc.serial {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>, seq = [#acc.device_type<none>]}
@@ -397,7 +411,7 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.serial {
+! CHECK:      acc.serial {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {auto_ = [#acc.device_type<none>], inclusiveUpperbound = array<i1: true>}
@@ -409,7 +423,7 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.serial {
+! CHECK:      acc.serial {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>, independent = [#acc.device_type<none>]}
@@ -421,8 +435,8 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.serial {
-! CHECK:        acc.loop gang {{.*}} {
+! CHECK:      acc.serial {{.*}} {
+! CHECK:        acc.loop {{.*}} gang {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>}{{$}}
 ! CHECK:        acc.yield
@@ -433,9 +447,9 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.serial {
+! CHECK:      acc.serial {{.*}} {
 ! CHECK:        [[GANGNUM1:%.*]] = arith.constant 8 : i32
-! CHECK-NEXT:   acc.loop gang({num=[[GANGNUM1]] : i32}) {{.*}} {
+! CHECK-NEXT:   acc.loop {{.*}} gang({num=[[GANGNUM1]] : i32}) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
@@ -446,9 +460,9 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.serial {
+! CHECK:      acc.serial {{.*}} {
 ! CHECK:        [[GANGNUM2:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK-NEXT:   acc.loop gang({num=[[GANGNUM2]] : i32}) {{.*}} {
+! CHECK-NEXT:   acc.loop {{.*}} gang({num=[[GANGNUM2]] : i32}) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
@@ -459,8 +473,8 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.serial {
-! CHECK:        acc.loop gang({num=%{{.*}} : i32, static=%{{.*}} : i32}) {{.*}} {
+! CHECK:      acc.serial {{.*}} {
+! CHECK:        acc.loop {{.*}} gang({num=%{{.*}} : i32, static=%{{.*}} : i32}) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
@@ -470,8 +484,9 @@ subroutine acc_serial_loop
   DO i = 1, n
     a(i) = b(i)
   END DO
-! CHECK:      acc.serial {
-! CHECK:        acc.loop vector {{.*}} {
+
+! CHECK:      acc.serial {{.*}} {
+! CHECK:        acc.loop {{.*}} vector {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>}{{$}}
 ! CHECK:        acc.yield
@@ -482,9 +497,9 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.serial {
+! CHECK:      acc.serial {{.*}} {
 ! CHECK:        [[CONSTANT128:%.*]] = arith.constant 128 : i32
-! CHECK:        acc.loop vector([[CONSTANT128]] : i32) {{.*}} {
+! CHECK:        acc.loop {{.*}} vector([[CONSTANT128]] : i32) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
@@ -495,9 +510,9 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.serial {
+! CHECK:      acc.serial {{.*}} {
 ! CHECK:        [[VECTORLENGTH:%.*]] = fir.load %{{.*}} : !fir.ref<i32>
-! CHECK:        acc.loop vector([[VECTORLENGTH]] : i32) {{.*}} {
+! CHECK:        acc.loop {{.*}} vector([[VECTORLENGTH]] : i32) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
@@ -508,8 +523,8 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.serial {
-! CHECK:        acc.loop worker {{.*}} {
+! CHECK:      acc.serial {{.*}} {
+! CHECK:        acc.loop {{.*}} worker {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   } attributes {inclusiveUpperbound = array<i1: true>}{{$}}
 ! CHECK:        acc.yield
@@ -520,9 +535,9 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.serial {
+! CHECK:      acc.serial {{.*}} {
 ! CHECK:        [[WORKER128:%.*]] = arith.constant 128 : i32
-! CHECK:        acc.loop worker([[WORKER128]] : i32) {{.*}} {
+! CHECK:        acc.loop {{.*}} worker([[WORKER128]] : i32) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
 ! CHECK:        acc.yield
@@ -535,7 +550,7 @@ subroutine acc_serial_loop
     END DO
   END DO
 
-! CHECK:      acc.serial {
+! CHECK:      acc.serial {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK-NOT:            fir.do_loop
 ! CHECK:          acc.yield
@@ -551,7 +566,7 @@ subroutine acc_serial_loop
     END DO
   END DO
 
-! CHECK:      acc.serial {
+! CHECK:      acc.serial {{.*}} {
 ! CHECK:        acc.loop {{.*}} {
 ! CHECK:            acc.loop {{.*}} {
 ! CHECK:              acc.yield
@@ -566,7 +581,7 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.serial {
+! CHECK:      acc.serial {{.*}} {
 ! CHECK:        [[TILESIZE:%.*]] = arith.constant 2 : i32
 ! CHECK:        acc.loop {{.*}} tile({[[TILESIZE]] : i32}) {{.*}} {
 ! CHECK:          acc.yield
@@ -579,7 +594,7 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.serial {
+! CHECK:      acc.serial {{.*}} {
 ! CHECK:        [[TILESIZEM1:%.*]] = arith.constant -1 : i32
 ! CHECK:        acc.loop {{.*}} tile({[[TILESIZEM1]] : i32}) {{.*}} {
 ! CHECK:          acc.yield
@@ -594,7 +609,7 @@ subroutine acc_serial_loop
     END DO
   END DO
 
-! CHECK:      acc.serial {
+! CHECK:      acc.serial {{.*}} {
 ! CHECK:        [[TILESIZE1:%.*]] = arith.constant 2 : i32
 ! CHECK:        [[TILESIZE2:%.*]] = arith.constant 2 : i32
 ! CHECK:        acc.loop {{.*}} tile({[[TILESIZE1]] : i32, [[TILESIZE2]] : i32}) {{.*}} {
@@ -608,7 +623,7 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! CHECK:      acc.serial {
+! CHECK:      acc.serial {{.*}} {
 ! CHECK:        acc.loop {{.*}} tile({%{{.*}} : i32}) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -622,7 +637,7 @@ subroutine acc_serial_loop
     END DO
   END DO
 
-! CHECK:      acc.serial {
+! CHECK:      acc.serial {{.*}} {
 ! CHECK:        acc.loop {{.*}} tile({%{{.*}} : i32, %{{.*}} : i32}) {{.*}} {
 ! CHECK:          acc.yield
 ! CHECK-NEXT:   }{{$}}
@@ -637,7 +652,7 @@ subroutine acc_serial_loop
 
 ! CHECK:      %[[COPYINREDR:.*]] = acc.copyin varPtr(%{{.*}} : !fir.ref<f32>) -> !fir.ref<f32> {dataClause = #acc<data_clause acc_reduction>, implicit = true, name = "reduction_r"}
 ! CHECK:      %[[COPYINREDI:.*]] = acc.copyin varPtr(%{{.*}} : !fir.ref<i32>) -> !fir.ref<i32> {dataClause = #acc<data_clause acc_reduction>, implicit = true, name = "reduction_i"}
-! CHECK:      acc.serial dataOperands(%[[COPYINREDR]], %[[COPYINREDI]] : !fir.ref<f32>, !fir.ref<i32>) {
+! CHECK:      acc.serial {{.*}} dataOperands(%[[COPYINREDR]], %[[COPYINREDI]] : !fir.ref<f32>, !fir.ref<i32>) {
 ! CHECK:        acc.loop {{.*}} reduction(@reduction_add_ref_f32 -> %{{.*}} : !fir.ref<f32>, @reduction_mul_ref_i32 -> %{{.*}} : !fir.ref<i32>)
 ! CHECK-NOT:      fir.do_loop
 ! CHECK:          acc.yield
-- 
GitLab


From 3e6e54eb795ce7a1ccd47df8c22fc08125a88886 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Pettersson?= <bjorn.a.pettersson@ericsson.com>
Date: Tue, 26 Mar 2024 20:53:34 +0100
Subject: [PATCH 340/404] [X86] Fix miscompile in combineShiftRightArithmetic
 (#86597)

When folding (ashr (shl, x, c1), c2) we need to treat c1 and c2
as unsigned to find out if the combined shift should be a left
or right shift.
Also do an early out during pre-legalization in case c1 and c2
has differet types, as that otherwise complicated the comparison
of c1 and c2 a bit.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 29 ++++++++++++++-----------
 llvm/test/CodeGen/X86/sar_fold.ll       | 13 +++++------
 2 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 1ce742a7c19a..9bad38f97c6a 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -47406,10 +47406,13 @@ static SDValue combineShiftRightArithmetic(SDNode *N, SelectionDAG &DAG,
     return DAG.getNode(X86ISD::VSRAV, DL, N->getVTList(), N0, ShrAmtVal);
   }
 
-  // fold (ashr (shl, a, [56,48,32,24,16]), SarConst)
-  // into (shl, (sext (a), [56,48,32,24,16] - SarConst)) or
-  // into (lshr, (sext (a), SarConst - [56,48,32,24,16]))
-  // depending on sign of (SarConst - [56,48,32,24,16])
+  // fold (SRA (SHL X, ShlConst), SraConst)
+  // into (SHL (sext_in_reg X), ShlConst - SraConst)
+  //   or (sext_in_reg X)
+  //   or (SRA (sext_in_reg X), SraConst - ShlConst)
+  // depending on relation between SraConst and ShlConst.
+  // We only do this if (Size - ShlConst) is equal to 8, 16 or 32. That allows
+  // us to do the sext_in_reg from corresponding bit.
 
   // sexts in X86 are MOVs. The MOVs have the same code size
   // as above SHIFTs (only SHIFT on 1 has lower code size).
@@ -47425,29 +47428,29 @@ static SDValue combineShiftRightArithmetic(SDNode *N, SelectionDAG &DAG,
   SDValue N00 = N0.getOperand(0);
   SDValue N01 = N0.getOperand(1);
   APInt ShlConst = N01->getAsAPIntVal();
-  APInt SarConst = N1->getAsAPIntVal();
+  APInt SraConst = N1->getAsAPIntVal();
   EVT CVT = N1.getValueType();
 
-  if (SarConst.isNegative())
+  if (CVT != N01.getValueType())
+    return SDValue();
+  if (SraConst.isNegative())
     return SDValue();
 
   for (MVT SVT : { MVT::i8, MVT::i16, MVT::i32 }) {
     unsigned ShiftSize = SVT.getSizeInBits();
-    // skipping types without corresponding sext/zext and
-    // ShlConst that is not one of [56,48,32,24,16]
+    // Only deal with (Size - ShlConst) being equal to 8, 16 or 32.
     if (ShiftSize >= Size || ShlConst != Size - ShiftSize)
       continue;
     SDLoc DL(N);
     SDValue NN =
         DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, N00, DAG.getValueType(SVT));
-    SarConst = SarConst - (Size - ShiftSize);
-    if (SarConst == 0)
+    if (SraConst.eq(ShlConst))
       return NN;
-    if (SarConst.isNegative())
+    if (SraConst.ult(ShlConst))
       return DAG.getNode(ISD::SHL, DL, VT, NN,
-                         DAG.getConstant(-SarConst, DL, CVT));
+                         DAG.getConstant(ShlConst - SraConst, DL, CVT));
     return DAG.getNode(ISD::SRA, DL, VT, NN,
-                       DAG.getConstant(SarConst, DL, CVT));
+                       DAG.getConstant(SraConst - ShlConst, DL, CVT));
   }
   return SDValue();
 }
diff --git a/llvm/test/CodeGen/X86/sar_fold.ll b/llvm/test/CodeGen/X86/sar_fold.ll
index 7607ca386d57..0f1396954b03 100644
--- a/llvm/test/CodeGen/X86/sar_fold.ll
+++ b/llvm/test/CodeGen/X86/sar_fold.ll
@@ -67,20 +67,17 @@ define void @shl144sar48(ptr %p) #0 {
   ret void
 }
 
-; This is incorrect. The 142 least significant bits in the stored value should
-; be zero, and but 142-157 should be taken from %a with a sign-extend into the
-; two most significant bits.
 define void @shl144sar2(ptr %p) #0 {
 ; CHECK-LABEL: shl144sar2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movswl (%eax), %ecx
-; CHECK-NEXT:    sarl $31, %ecx
+; CHECK-NEXT:    shll $14, %ecx
 ; CHECK-NEXT:    movl %ecx, 16(%eax)
-; CHECK-NEXT:    movl %ecx, 8(%eax)
-; CHECK-NEXT:    movl %ecx, 12(%eax)
-; CHECK-NEXT:    movl %ecx, 4(%eax)
-; CHECK-NEXT:    movl %ecx, (%eax)
+; CHECK-NEXT:    movl $0, 8(%eax)
+; CHECK-NEXT:    movl $0, 12(%eax)
+; CHECK-NEXT:    movl $0, 4(%eax)
+; CHECK-NEXT:    movl $0, (%eax)
 ; CHECK-NEXT:    retl
   %a = load i160, ptr %p
   %1 = shl i160 %a, 144
-- 
GitLab


From 2d97ba22f8ca0fbb8896fd0f54c0cd7c69f4d0cd Mon Sep 17 00:00:00 2001
From: Vinayak Dev <104419489+vinayakdsci@users.noreply.github.com>
Date: Wed, 27 Mar 2024 01:46:35 +0530
Subject: [PATCH 341/404] [libc]: Remove unused includes from strfrom*.cpp
 (#86657)

Removes unused header includes from `strfrom*()` implementation files.
---
 libc/src/stdlib/strfromd.cpp | 3 ---
 libc/src/stdlib/strfromf.cpp | 3 ---
 libc/src/stdlib/strfroml.cpp | 3 ---
 3 files changed, 9 deletions(-)

diff --git a/libc/src/stdlib/strfromd.cpp b/libc/src/stdlib/strfromd.cpp
index 1d02a7ad1124..329f6fdcaff7 100644
--- a/libc/src/stdlib/strfromd.cpp
+++ b/libc/src/stdlib/strfromd.cpp
@@ -9,9 +9,6 @@
 #include "src/stdlib/strfromd.h"
 #include "src/stdlib/str_from_util.h"
 
-#include <stdarg.h>
-#include <stddef.h>
-
 namespace LIBC_NAMESPACE {
 
 LLVM_LIBC_FUNCTION(int, strfromd,
diff --git a/libc/src/stdlib/strfromf.cpp b/libc/src/stdlib/strfromf.cpp
index 40eff87eb454..80e1d74797c8 100644
--- a/libc/src/stdlib/strfromf.cpp
+++ b/libc/src/stdlib/strfromf.cpp
@@ -9,9 +9,6 @@
 #include "src/stdlib/strfromf.h"
 #include "src/stdlib/str_from_util.h"
 
-#include <stdarg.h>
-#include <stddef.h>
-
 namespace LIBC_NAMESPACE {
 
 LLVM_LIBC_FUNCTION(int, strfromf,
diff --git a/libc/src/stdlib/strfroml.cpp b/libc/src/stdlib/strfroml.cpp
index 6b8781e7c04e..f0bc9354c7ad 100644
--- a/libc/src/stdlib/strfroml.cpp
+++ b/libc/src/stdlib/strfroml.cpp
@@ -9,9 +9,6 @@
 #include "src/stdlib/strfroml.h"
 #include "src/stdlib/str_from_util.h"
 
-#include <stdarg.h>
-#include <stddef.h>
-
 namespace LIBC_NAMESPACE {
 
 LLVM_LIBC_FUNCTION(int, strfroml,
-- 
GitLab


From a3f21a32495abeaddfa64a9789cd8c1b3fdad87c Mon Sep 17 00:00:00 2001
From: Vinayak Dev <104419489+vinayakdsci@users.noreply.github.com>
Date: Wed, 27 Mar 2024 01:52:14 +0530
Subject: [PATCH 342/404] [libc] Remove LIBC_HAS_ATTRIBUTE macro (#86656)

Resolves #86547

Replace occurrences of the macro `LIBC_HAS_ATTRIBUTE` with `__has_attribute`.
---
 libc/src/__support/macros/config.h        | 8 --------
 libc/test/src/__support/arg_list_test.cpp | 2 +-
 2 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/libc/src/__support/macros/config.h b/libc/src/__support/macros/config.h
index fcc8f551a783..6666c1366696 100644
--- a/libc/src/__support/macros/config.h
+++ b/libc/src/__support/macros/config.h
@@ -39,12 +39,4 @@
 #define LIBC_HAS_FEATURE(f) 0
 #endif
 
-// Compiler attribute-detection.
-// https://clang.llvm.org/docs/LanguageExtensions.html#has-attribute
-#ifdef __has_attribute
-#define LIBC_HAS_ATTRIBUTE(f) __has_attribute(f)
-#else
-#define LIBC_HAS_ATTRIBUTE(f) 0
-#endif
-
 #endif // LLVM_LIBC_SRC___SUPPORT_MACROS_CONFIG_H
diff --git a/libc/test/src/__support/arg_list_test.cpp b/libc/test/src/__support/arg_list_test.cpp
index 1876cf7f70b4..4f229e2bfe69 100644
--- a/libc/test/src/__support/arg_list_test.cpp
+++ b/libc/test/src/__support/arg_list_test.cpp
@@ -120,7 +120,7 @@ TEST(LlvmLibcArgListTest, TestStructTypes) {
 }
 
 // Test vector extensions from clang.
-#if LIBC_HAS_ATTRIBUTE(ext_vector_type)
+#if __has_attribute(ext_vector_type)
 
 using int1 = int __attribute__((ext_vector_type(1)));
 using int2 = int __attribute__((ext_vector_type(2)));
-- 
GitLab


From ac1af75051195e63cc518cf12312dd4eaf2e9bf8 Mon Sep 17 00:00:00 2001
From: OverMighty <its.overmighty@gmail.com>
Date: Tue, 26 Mar 2024 20:22:47 +0000
Subject: [PATCH 343/404] [clang] Implement constexpr support for
 __builtin_{clzg,ctzg} (#86577)

Fixes #86549.
---
 clang/docs/LanguageExtensions.rst     |   2 +
 clang/docs/ReleaseNotes.rst           |   5 ++
 clang/include/clang/Basic/Builtins.td |   4 +-
 clang/lib/AST/ExprConstant.cpp        |  37 ++++++--
 clang/test/Sema/constant-builtins-2.c | 116 ++++++++++++++++++++++++++
 5 files changed, 153 insertions(+), 11 deletions(-)

diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index 5711972b55e6..7b23e4d1c2f3 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -5420,10 +5420,12 @@ The following builtin intrinsics can be used in constant expressions:
 * ``__builtin_clzl``
 * ``__builtin_clzll``
 * ``__builtin_clzs``
+* ``__builtin_clzg``
 * ``__builtin_ctz``
 * ``__builtin_ctzl``
 * ``__builtin_ctzll``
 * ``__builtin_ctzs``
+* ``__builtin_ctzg``
 * ``__builtin_ffs``
 * ``__builtin_ffsl``
 * ``__builtin_ffsll``
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index af2295c43d7e..01abe2b39e87 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -188,6 +188,11 @@ Non-comprehensive list of changes in this release
 
 - Lambda expressions are now accepted in C++03 mode as an extension.
 
+- Added ``__builtin_clzg`` and ``__builtin_ctzg`` as type-generic alternatives
+  to ``__builtin_clz{,s,l,ll}`` and ``__builtin_ctz{,s,l,ll}`` respectively,
+  with support for any unsigned integer type. Like the previous builtins, these
+  new builtins are constexpr and may be used in constant expressions.
+
 New Compiler Flags
 ------------------
 
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 21ab9bb86d1b..52c0dd52c28b 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -678,7 +678,7 @@ def Clz : Builtin, BitShort_Int_Long_LongLongTemplate {
 
 def Clzg : Builtin {
   let Spellings = ["__builtin_clzg"];
-  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Attributes = [NoThrow, Const, Constexpr, CustomTypeChecking];
   let Prototype = "int(...)";
 }
 
@@ -690,7 +690,7 @@ def Ctz : Builtin, BitShort_Int_Long_LongLongTemplate {
 
 def Ctzg : Builtin {
   let Spellings = ["__builtin_ctzg"];
-  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Attributes = [NoThrow, Const, Constexpr, CustomTypeChecking];
   let Prototype = "int(...)";
 }
 
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index c5bdff0df06b..5a36621dc5cc 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -12353,6 +12353,7 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
   case Builtin::BI__builtin_clzl:
   case Builtin::BI__builtin_clzll:
   case Builtin::BI__builtin_clzs:
+  case Builtin::BI__builtin_clzg:
   case Builtin::BI__lzcnt16: // Microsoft variants of count leading-zeroes
   case Builtin::BI__lzcnt:
   case Builtin::BI__lzcnt64: {
@@ -12360,14 +12361,23 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
     if (!EvaluateInteger(E->getArg(0), Val, Info))
       return false;
 
-    // When the argument is 0, the result of GCC builtins is undefined, whereas
-    // for Microsoft intrinsics, the result is the bit-width of the argument.
-    bool ZeroIsUndefined = BuiltinOp != Builtin::BI__lzcnt16 &&
-                           BuiltinOp != Builtin::BI__lzcnt &&
-                           BuiltinOp != Builtin::BI__lzcnt64;
+    if (!Val) {
+      if (BuiltinOp == Builtin::BI__builtin_clzg && E->getNumArgs() > 1) {
+        if (!EvaluateInteger(E->getArg(1), Val, Info))
+          return false;
+        return Success(Val, E);
+      }
 
-    if (ZeroIsUndefined && !Val)
-      return Error(E);
+      // When the argument is 0, the result of GCC builtins is undefined,
+      // whereas for Microsoft intrinsics, the result is the bit-width of the
+      // argument.
+      bool ZeroIsUndefined = BuiltinOp != Builtin::BI__lzcnt16 &&
+                             BuiltinOp != Builtin::BI__lzcnt &&
+                             BuiltinOp != Builtin::BI__lzcnt64;
+
+      if (ZeroIsUndefined)
+        return Error(E);
+    }
 
     return Success(Val.countl_zero(), E);
   }
@@ -12409,12 +12419,21 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
   case Builtin::BI__builtin_ctz:
   case Builtin::BI__builtin_ctzl:
   case Builtin::BI__builtin_ctzll:
-  case Builtin::BI__builtin_ctzs: {
+  case Builtin::BI__builtin_ctzs:
+  case Builtin::BI__builtin_ctzg: {
     APSInt Val;
     if (!EvaluateInteger(E->getArg(0), Val, Info))
       return false;
-    if (!Val)
+
+    if (!Val) {
+      if (BuiltinOp == Builtin::BI__builtin_ctzg && E->getNumArgs() > 1) {
+        if (!EvaluateInteger(E->getArg(1), Val, Info))
+          return false;
+        return Success(Val, E);
+      }
+
       return Error(E);
+    }
 
     return Success(Val.countr_zero(), E);
   }
diff --git a/clang/test/Sema/constant-builtins-2.c b/clang/test/Sema/constant-builtins-2.c
index 6dd1d88759c7..a60a1f16a458 100644
--- a/clang/test/Sema/constant-builtins-2.c
+++ b/clang/test/Sema/constant-builtins-2.c
@@ -218,6 +218,64 @@ char clz6[__builtin_clzll(0xFFLL) == BITSIZE(long long) - 8 ? 1 : -1];
 char clz7[__builtin_clzs(0x1) == BITSIZE(short) - 1 ? 1 : -1];
 char clz8[__builtin_clzs(0xf) == BITSIZE(short) - 4 ? 1 : -1];
 char clz9[__builtin_clzs(0xfff) == BITSIZE(short) - 12 ? 1 : -1];
+int clz10 = __builtin_clzg((unsigned char)0); // expected-error {{not a compile-time constant}}
+char clz11[__builtin_clzg((unsigned char)0, 42) == 42 ? 1 : -1];
+char clz12[__builtin_clzg((unsigned char)0x1) == BITSIZE(char) - 1 ? 1 : -1];
+char clz13[__builtin_clzg((unsigned char)0x1, 42) == BITSIZE(char) - 1 ? 1 : -1];
+char clz14[__builtin_clzg((unsigned char)0xf) == BITSIZE(char) - 4 ? 1 : -1];
+char clz15[__builtin_clzg((unsigned char)0xf, 42) == BITSIZE(char) - 4 ? 1 : -1];
+char clz16[__builtin_clzg((unsigned char)(1 << (BITSIZE(char) - 1))) == 0 ? 1 : -1];
+char clz17[__builtin_clzg((unsigned char)(1 << (BITSIZE(char) - 1)), 42) == 0 ? 1 : -1];
+int clz18 = __builtin_clzg((unsigned short)0); // expected-error {{not a compile-time constant}}
+char clz19[__builtin_clzg((unsigned short)0, 42) == 42 ? 1 : -1];
+char clz20[__builtin_clzg((unsigned short)0x1) == BITSIZE(short) - 1 ? 1 : -1];
+char clz21[__builtin_clzg((unsigned short)0x1, 42) == BITSIZE(short) - 1 ? 1 : -1];
+char clz22[__builtin_clzg((unsigned short)0xf) == BITSIZE(short) - 4 ? 1 : -1];
+char clz23[__builtin_clzg((unsigned short)0xf, 42) == BITSIZE(short) - 4 ? 1 : -1];
+char clz24[__builtin_clzg((unsigned short)(1 << (BITSIZE(short) - 1))) == 0 ? 1 : -1];
+char clz25[__builtin_clzg((unsigned short)(1 << (BITSIZE(short) - 1)), 42) == 0 ? 1 : -1];
+int clz26 = __builtin_clzg(0U); // expected-error {{not a compile-time constant}}
+char clz27[__builtin_clzg(0U, 42) == 42 ? 1 : -1];
+char clz28[__builtin_clzg(0x1U) == BITSIZE(int) - 1 ? 1 : -1];
+char clz29[__builtin_clzg(0x1U, 42) == BITSIZE(int) - 1 ? 1 : -1];
+char clz30[__builtin_clzg(0xfU) == BITSIZE(int) - 4 ? 1 : -1];
+char clz31[__builtin_clzg(0xfU, 42) == BITSIZE(int) - 4 ? 1 : -1];
+char clz32[__builtin_clzg(1U << (BITSIZE(int) - 1)) == 0 ? 1 : -1];
+char clz33[__builtin_clzg(1U << (BITSIZE(int) - 1), 42) == 0 ? 1 : -1];
+int clz34 = __builtin_clzg(0UL); // expected-error {{not a compile-time constant}}
+char clz35[__builtin_clzg(0UL, 42) == 42 ? 1 : -1];
+char clz36[__builtin_clzg(0x1UL) == BITSIZE(long) - 1 ? 1 : -1];
+char clz37[__builtin_clzg(0x1UL, 42) == BITSIZE(long) - 1 ? 1 : -1];
+char clz38[__builtin_clzg(0xfUL) == BITSIZE(long) - 4 ? 1 : -1];
+char clz39[__builtin_clzg(0xfUL, 42) == BITSIZE(long) - 4 ? 1 : -1];
+char clz40[__builtin_clzg(1UL << (BITSIZE(long) - 1)) == 0 ? 1 : -1];
+char clz41[__builtin_clzg(1UL << (BITSIZE(long) - 1), 42) == 0 ? 1 : -1];
+int clz42 = __builtin_clzg(0ULL); // expected-error {{not a compile-time constant}}
+char clz43[__builtin_clzg(0ULL, 42) == 42 ? 1 : -1];
+char clz44[__builtin_clzg(0x1ULL) == BITSIZE(long long) - 1 ? 1 : -1];
+char clz45[__builtin_clzg(0x1ULL, 42) == BITSIZE(long long) - 1 ? 1 : -1];
+char clz46[__builtin_clzg(0xfULL) == BITSIZE(long long) - 4 ? 1 : -1];
+char clz47[__builtin_clzg(0xfULL, 42) == BITSIZE(long long) - 4 ? 1 : -1];
+char clz48[__builtin_clzg(1ULL << (BITSIZE(long long) - 1)) == 0 ? 1 : -1];
+char clz49[__builtin_clzg(1ULL << (BITSIZE(long long) - 1), 42) == 0 ? 1 : -1];
+#ifdef __SIZEOF_INT128__
+int clz50 = __builtin_clzg((unsigned __int128)0); // expected-error {{not a compile-time constant}}
+char clz51[__builtin_clzg((unsigned __int128)0, 42) == 42 ? 1 : -1];
+char clz52[__builtin_clzg((unsigned __int128)0x1) == BITSIZE(__int128) - 1 ? 1 : -1];
+char clz53[__builtin_clzg((unsigned __int128)0x1, 42) == BITSIZE(__int128) - 1 ? 1 : -1];
+char clz54[__builtin_clzg((unsigned __int128)0xf) == BITSIZE(__int128) - 4 ? 1 : -1];
+char clz55[__builtin_clzg((unsigned __int128)0xf, 42) == BITSIZE(__int128) - 4 ? 1 : -1];
+char clz56[__builtin_clzg((unsigned __int128)(1 << (BITSIZE(__int128) - 1))) == 0 ? 1 : -1];
+char clz57[__builtin_clzg((unsigned __int128)(1 << (BITSIZE(__int128) - 1)), 42) == 0 ? 1 : -1];
+#endif
+int clz58 = __builtin_clzg((unsigned _BitInt(128))0); // expected-error {{not a compile-time constant}}
+char clz59[__builtin_clzg((unsigned _BitInt(128))0, 42) == 42 ? 1 : -1];
+char clz60[__builtin_clzg((unsigned _BitInt(128))0x1) == BITSIZE(_BitInt(128)) - 1 ? 1 : -1];
+char clz61[__builtin_clzg((unsigned _BitInt(128))0x1, 42) == BITSIZE(_BitInt(128)) - 1 ? 1 : -1];
+char clz62[__builtin_clzg((unsigned _BitInt(128))0xf) == BITSIZE(_BitInt(128)) - 4 ? 1 : -1];
+char clz63[__builtin_clzg((unsigned _BitInt(128))0xf, 42) == BITSIZE(_BitInt(128)) - 4 ? 1 : -1];
+char clz64[__builtin_clzg((unsigned _BitInt(128))(1 << (BITSIZE(_BitInt(128)) - 1))) == 0 ? 1 : -1];
+char clz65[__builtin_clzg((unsigned _BitInt(128))(1 << (BITSIZE(_BitInt(128)) - 1)), 42) == 0 ? 1 : -1];
 
 char ctz1[__builtin_ctz(1) == 0 ? 1 : -1];
 char ctz2[__builtin_ctz(8) == 3 ? 1 : -1];
@@ -226,6 +284,64 @@ int ctz4 = __builtin_ctz(0); // expected-error {{not a compile-time constant}}
 char ctz5[__builtin_ctzl(0x10L) == 4 ? 1 : -1];
 char ctz6[__builtin_ctzll(0x100LL) == 8 ? 1 : -1];
 char ctz7[__builtin_ctzs(1 << (BITSIZE(short) - 1)) == BITSIZE(short) - 1 ? 1 : -1];
+int ctz8 = __builtin_ctzg((unsigned char)0); // expected-error {{not a compile-time constant}}
+char ctz9[__builtin_ctzg((unsigned char)0, 42) == 42 ? 1 : -1];
+char ctz10[__builtin_ctzg((unsigned char)0x1) == 0 ? 1 : -1];
+char ctz11[__builtin_ctzg((unsigned char)0x1, 42) == 0 ? 1 : -1];
+char ctz12[__builtin_ctzg((unsigned char)0x10) == 4 ? 1 : -1];
+char ctz13[__builtin_ctzg((unsigned char)0x10, 42) == 4 ? 1 : -1];
+char ctz14[__builtin_ctzg((unsigned char)(1 << (BITSIZE(char) - 1))) == BITSIZE(char) - 1 ? 1 : -1];
+char ctz15[__builtin_ctzg((unsigned char)(1 << (BITSIZE(char) - 1)), 42) == BITSIZE(char) - 1 ? 1 : -1];
+int ctz16 = __builtin_ctzg((unsigned short)0); // expected-error {{not a compile-time constant}}
+char ctz17[__builtin_ctzg((unsigned short)0, 42) == 42 ? 1 : -1];
+char ctz18[__builtin_ctzg((unsigned short)0x1) == 0 ? 1 : -1];
+char ctz19[__builtin_ctzg((unsigned short)0x1, 42) == 0 ? 1 : -1];
+char ctz20[__builtin_ctzg((unsigned short)0x10) == 4 ? 1 : -1];
+char ctz21[__builtin_ctzg((unsigned short)0x10, 42) == 4 ? 1 : -1];
+char ctz22[__builtin_ctzg((unsigned short)(1 << (BITSIZE(short) - 1))) == BITSIZE(short) - 1 ? 1 : -1];
+char ctz23[__builtin_ctzg((unsigned short)(1 << (BITSIZE(short) - 1)), 42) == BITSIZE(short) - 1 ? 1 : -1];
+int ctz24 = __builtin_ctzg(0U); // expected-error {{not a compile-time constant}}
+char ctz25[__builtin_ctzg(0U, 42) == 42 ? 1 : -1];
+char ctz26[__builtin_ctzg(0x1U) == 0 ? 1 : -1];
+char ctz27[__builtin_ctzg(0x1U, 42) == 0 ? 1 : -1];
+char ctz28[__builtin_ctzg(0x10U) == 4 ? 1 : -1];
+char ctz29[__builtin_ctzg(0x10U, 42) == 4 ? 1 : -1];
+char ctz30[__builtin_ctzg(1U << (BITSIZE(int) - 1)) == BITSIZE(int) - 1 ? 1 : -1];
+char ctz31[__builtin_ctzg(1U << (BITSIZE(int) - 1), 42) == BITSIZE(int) - 1 ? 1 : -1];
+int ctz32 = __builtin_ctzg(0UL); // expected-error {{not a compile-time constant}}
+char ctz33[__builtin_ctzg(0UL, 42) == 42 ? 1 : -1];
+char ctz34[__builtin_ctzg(0x1UL) == 0 ? 1 : -1];
+char ctz35[__builtin_ctzg(0x1UL, 42) == 0 ? 1 : -1];
+char ctz36[__builtin_ctzg(0x10UL) == 4 ? 1 : -1];
+char ctz37[__builtin_ctzg(0x10UL, 42) == 4 ? 1 : -1];
+char ctz38[__builtin_ctzg(1UL << (BITSIZE(long) - 1)) == BITSIZE(long) - 1 ? 1 : -1];
+char ctz39[__builtin_ctzg(1UL << (BITSIZE(long) - 1), 42) == BITSIZE(long) - 1 ? 1 : -1];
+int ctz40 = __builtin_ctzg(0ULL); // expected-error {{not a compile-time constant}}
+char ctz41[__builtin_ctzg(0ULL, 42) == 42 ? 1 : -1];
+char ctz42[__builtin_ctzg(0x1ULL) == 0 ? 1 : -1];
+char ctz43[__builtin_ctzg(0x1ULL, 42) == 0 ? 1 : -1];
+char ctz44[__builtin_ctzg(0x10ULL) == 4 ? 1 : -1];
+char ctz45[__builtin_ctzg(0x10ULL, 42) == 4 ? 1 : -1];
+char ctz46[__builtin_ctzg(1ULL << (BITSIZE(long long) - 1)) == BITSIZE(long long) - 1 ? 1 : -1];
+char ctz47[__builtin_ctzg(1ULL << (BITSIZE(long long) - 1), 42) == BITSIZE(long long) - 1 ? 1 : -1];
+#ifdef __SIZEOF_INT128__
+int ctz48 = __builtin_ctzg((unsigned __int128)0); // expected-error {{not a compile-time constant}}
+char ctz49[__builtin_ctzg((unsigned __int128)0, 42) == 42 ? 1 : -1];
+char ctz50[__builtin_ctzg((unsigned __int128)0x1) == 0 ? 1 : -1];
+char ctz51[__builtin_ctzg((unsigned __int128)0x1, 42) == 0 ? 1 : -1];
+char ctz52[__builtin_ctzg((unsigned __int128)0x10) == 4 ? 1 : -1];
+char ctz53[__builtin_ctzg((unsigned __int128)0x10, 42) == 4 ? 1 : -1];
+char ctz54[__builtin_ctzg((unsigned __int128)1 << (BITSIZE(__int128) - 1)) == BITSIZE(__int128) - 1 ? 1 : -1];
+char ctz55[__builtin_ctzg((unsigned __int128)1 << (BITSIZE(__int128) - 1), 42) == BITSIZE(__int128) - 1 ? 1 : -1];
+#endif
+int ctz56 = __builtin_ctzg((unsigned _BitInt(128))0); // expected-error {{not a compile-time constant}}
+char ctz57[__builtin_ctzg((unsigned _BitInt(128))0, 42) == 42 ? 1 : -1];
+char ctz58[__builtin_ctzg((unsigned _BitInt(128))0x1) == 0 ? 1 : -1];
+char ctz59[__builtin_ctzg((unsigned _BitInt(128))0x1, 42) == 0 ? 1 : -1];
+char ctz60[__builtin_ctzg((unsigned _BitInt(128))0x10) == 4 ? 1 : -1];
+char ctz61[__builtin_ctzg((unsigned _BitInt(128))0x10, 42) == 4 ? 1 : -1];
+char ctz62[__builtin_ctzg((unsigned _BitInt(128))1 << (BITSIZE(_BitInt(128)) - 1)) == BITSIZE(_BitInt(128)) - 1 ? 1 : -1];
+char ctz63[__builtin_ctzg((unsigned _BitInt(128))1 << (BITSIZE(_BitInt(128)) - 1), 42) == BITSIZE(_BitInt(128)) - 1 ? 1 : -1];
 
 char popcount1[__builtin_popcount(0) == 0 ? 1 : -1];
 char popcount2[__builtin_popcount(0xF0F0) == 8 ? 1 : -1];
-- 
GitLab


From 158401493a42014947ef2fb64ca761b57741815a Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 26 Mar 2024 13:31:21 -0700
Subject: [PATCH 344/404] [RISCV] Check that the stack adjust immediate for
 cm.push/pop* has the correct sign and is divisible by 16. (#85295)

To do this I've added a new AsmOperand for cm.push to expect a negative
value. We also use that to customize the print function so that we don't
need to detect cm.push opcode to add the negative sign.

I've renamed some places that used Spimm to be StackAdj since that's
what is being parsed. I'm still not about where we should use Spimm or
StackAdj.

I've removed the printSpimm helper function which in one usage printed
the sp[5:4]<<4 value and the other usage printed the full stack
adjustment. There wasn't anything interesting about how it was printed
it just passed the value to the raw_stream. If there was something
special needed, it's unclear whether it would be the same for the two
different usages so I inlined it.

One open question is whether we need to support stack adjustments
expressed as an expression rather than a literal integer.
---
 .../Target/RISCV/AsmParser/RISCVAsmParser.cpp | 18 ++++---
 .../RISCV/MCTargetDesc/RISCVBaseInfo.cpp      |  2 -
 .../Target/RISCV/MCTargetDesc/RISCVBaseInfo.h |  8 +--
 .../RISCV/MCTargetDesc/RISCVInstPrinter.cpp   | 19 +++----
 .../RISCV/MCTargetDesc/RISCVInstPrinter.h     |  9 +++-
 llvm/lib/Target/RISCV/RISCVInstrInfoZc.td     | 51 ++++++++++++++-----
 llvm/test/MC/RISCV/rv32zcmp-invalid.s         | 12 +++++
 llvm/test/MC/RISCV/rv64zcmp-invalid.s         | 12 +++++
 8 files changed, 95 insertions(+), 36 deletions(-)

diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index 5e594d6cad6b..123b37442329 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -213,7 +213,11 @@ class RISCVAsmParser : public MCTargetAsmParser {
   ParseStatus parseReglist(OperandVector &Operands);
   ParseStatus parseRegReg(OperandVector &Operands);
   ParseStatus parseRetval(OperandVector &Operands);
-  ParseStatus parseZcmpSpimm(OperandVector &Operands);
+  ParseStatus parseZcmpStackAdj(OperandVector &Operands,
+                                bool ExpectNegative = false);
+  ParseStatus parseZcmpNegStackAdj(OperandVector &Operands) {
+    return parseZcmpStackAdj(Operands, /*ExpectNegative*/ true);
+  }
 
   bool parseOperand(OperandVector &Operands, StringRef Mnemonic);
 
@@ -1062,7 +1066,7 @@ public:
       break;
     case KindTy::Spimm:
       OS << "<Spimm: ";
-      RISCVZC::printSpimm(Spimm.Val, OS);
+      OS << Spimm.Val;
       OS << '>';
       break;
     case KindTy::RegReg:
@@ -1608,7 +1612,7 @@ bool RISCVAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
         ErrorLoc,
         "operand must be {ra [, s0[-sN]]} or {x1 [, x8[-x9][, x18[-xN]]]}");
   }
-  case Match_InvalidSpimm: {
+  case Match_InvalidStackAdj: {
     SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
     return Error(
         ErrorLoc,
@@ -2583,8 +2587,9 @@ ParseStatus RISCVAsmParser::parseReglist(OperandVector &Operands) {
   return ParseStatus::Success;
 }
 
-ParseStatus RISCVAsmParser::parseZcmpSpimm(OperandVector &Operands) {
-  (void)parseOptionalToken(AsmToken::Minus);
+ParseStatus RISCVAsmParser::parseZcmpStackAdj(OperandVector &Operands,
+                                              bool ExpectNegative) {
+  bool Negative = parseOptionalToken(AsmToken::Minus);
 
   SMLoc S = getLoc();
   int64_t StackAdjustment = getLexer().getTok().getIntVal();
@@ -2592,7 +2597,8 @@ ParseStatus RISCVAsmParser::parseZcmpSpimm(OperandVector &Operands) {
   unsigned RlistVal = static_cast<RISCVOperand *>(Operands[1].get())->Rlist.Val;
 
   bool IsEABI = isRVE();
-  if (!RISCVZC::getSpimm(RlistVal, Spimm, StackAdjustment, isRV64(), IsEABI))
+  if (Negative != ExpectNegative ||
+      !RISCVZC::getSpimm(RlistVal, Spimm, StackAdjustment, isRV64(), IsEABI))
     return ParseStatus::NoMatch;
   Operands.push_back(RISCVOperand::createSpimm(Spimm << 4, S));
   getLexer().Lex();
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp
index 61f8e7171037..5d9a58babe60 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp
@@ -236,6 +236,4 @@ void RISCVZC::printRlist(unsigned SlistEncode, raw_ostream &OS) {
   OS << "}";
 }
 
-void RISCVZC::printSpimm(int64_t Spimm, raw_ostream &OS) { OS << Spimm; }
-
 } // namespace llvm
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
index 6d0381c30d3e..c65b51218772 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
@@ -580,15 +580,17 @@ inline static bool getSpimm(unsigned RlistVal, unsigned &SpimmVal,
                             int64_t StackAdjustment, bool IsRV64, bool IsEABI) {
   if (RlistVal == RLISTENCODE::INVALID_RLIST)
     return false;
-  unsigned stackAdj = getStackAdjBase(RlistVal, IsRV64, IsEABI);
-  SpimmVal = (StackAdjustment - stackAdj) / 16;
+  unsigned StackAdjBase = getStackAdjBase(RlistVal, IsRV64, IsEABI);
+  StackAdjustment -= StackAdjBase;
+  if (StackAdjustment % 16 != 0)
+    return false;
+  SpimmVal = StackAdjustment / 16;
   if (SpimmVal > 3)
     return false;
   return true;
 }
 
 void printRlist(unsigned SlistEncode, raw_ostream &OS);
-void printSpimm(int64_t Spimm, raw_ostream &OS);
 } // namespace RISCVZC
 
 } // namespace llvm
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
index bd899495812f..04e02e9fa0ab 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
@@ -292,24 +292,25 @@ void RISCVInstPrinter::printRegReg(const MCInst *MI, unsigned OpNo,
   O << ")";
 }
 
-void RISCVInstPrinter::printSpimm(const MCInst *MI, unsigned OpNo,
-                                  const MCSubtargetInfo &STI, raw_ostream &O) {
+void RISCVInstPrinter::printStackAdj(const MCInst *MI, unsigned OpNo,
+                                     const MCSubtargetInfo &STI, raw_ostream &O,
+                                     bool Negate) {
   int64_t Imm = MI->getOperand(OpNo).getImm();
-  unsigned Opcode = MI->getOpcode();
   bool IsRV64 = STI.hasFeature(RISCV::Feature64Bit);
   bool IsEABI = STI.hasFeature(RISCV::FeatureRVE);
-  int64_t Spimm = 0;
+  int64_t StackAdj = 0;
   auto RlistVal = MI->getOperand(0).getImm();
   assert(RlistVal != 16 && "Incorrect rlist.");
   auto Base = RISCVZC::getStackAdjBase(RlistVal, IsRV64, IsEABI);
-  Spimm = Imm + Base;
-  assert((Spimm >= Base && Spimm <= Base + 48) && "Incorrect spimm");
-  if (Opcode == RISCV::CM_PUSH)
-    Spimm = -Spimm;
+  StackAdj = Imm + Base;
+  assert((StackAdj >= Base && StackAdj <= Base + 48) &&
+         "Incorrect stack adjust");
+  if (Negate)
+    StackAdj = -StackAdj;
 
   // RAII guard for ANSI color escape sequences
   WithMarkup ScopedMarkup = markup(O, Markup::Immediate);
-  RISCVZC::printSpimm(Spimm, O);
+  O << StackAdj;
 }
 
 void RISCVInstPrinter::printVMaskReg(const MCInst *MI, unsigned OpNo,
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h
index 4512bd5f4c4b..77cc7a67e889 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h
@@ -52,8 +52,13 @@ public:
                      const MCSubtargetInfo &STI, raw_ostream &O);
   void printRlist(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
                   raw_ostream &O);
-  void printSpimm(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
-                  raw_ostream &O);
+  void printStackAdj(const MCInst *MI, unsigned OpNo,
+                     const MCSubtargetInfo &STI, raw_ostream &O,
+                     bool Negate = false);
+  void printNegStackAdj(const MCInst *MI, unsigned OpNo,
+                        const MCSubtargetInfo &STI, raw_ostream &O) {
+    return printStackAdj(MI, OpNo, STI, O, /*Negate*/ true);
+  }
   void printRegReg(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
                    raw_ostream &O);
   // Autogenerated by tblgen.
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZc.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZc.td
index 2c8451c5c4ce..a327bd3d0c28 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZc.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZc.td
@@ -41,10 +41,20 @@ def RlistAsmOperand : AsmOperandClass {
   let DiagnosticType = "InvalidRlist";
 }
 
-def SpimmAsmOperand : AsmOperandClass {
-  let Name = "Spimm";
-  let ParserMethod = "parseZcmpSpimm";
-  let DiagnosticType = "InvalidSpimm";
+def StackAdjAsmOperand : AsmOperandClass {
+  let Name = "StackAdj";
+  let ParserMethod = "parseZcmpStackAdj";
+  let DiagnosticType = "InvalidStackAdj";
+  let PredicateMethod = "isSpimm";
+  let RenderMethod = "addSpimmOperands";
+}
+
+def NegStackAdjAsmOperand : AsmOperandClass {
+  let Name = "NegStackAdj";
+  let ParserMethod = "parseZcmpNegStackAdj";
+  let DiagnosticType = "InvalidStackAdj";
+  let PredicateMethod = "isSpimm";
+  let RenderMethod = "addSpimmOperands";
 }
 
 def rlist : Operand<OtherVT> {
@@ -59,11 +69,23 @@ def rlist : Operand<OtherVT> {
     // 0~3 Reserved for EABI
     return isUInt<4>(Imm) && Imm >= 4;
   }];
- }
+}
+
+def stackadj : Operand<OtherVT> {
+  let ParserMatchClass = StackAdjAsmOperand;
+  let PrintMethod = "printStackAdj";
+  let DecoderMethod = "decodeZcmpSpimm";
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (!MCOp.evaluateAsConstantImm(Imm))
+      return false;
+    return isShiftedUInt<2, 4>(Imm);
+  }];
+}
 
-def spimm : Operand<OtherVT> {
-  let ParserMatchClass = SpimmAsmOperand;
-  let PrintMethod = "printSpimm";
+def negstackadj : Operand<OtherVT> {
+  let ParserMatchClass = NegStackAdjAsmOperand;
+  let PrintMethod = "printNegStackAdj";
   let DecoderMethod = "decodeZcmpSpimm";
   let MCOperandPredicate = [{
     int64_t Imm;
@@ -124,14 +146,15 @@ class RVZcArith_r<bits<5> funct5, string OpcodeStr> :
   let Constraints = "$rd = $rd_wb";
 }
 
-class RVInstZcCPPP<bits<5> funct5, string opcodestr>
-    : RVInst16<(outs), (ins rlist:$rlist, spimm:$spimm),
-               opcodestr, "$rlist, $spimm", [], InstFormatOther> {
+class RVInstZcCPPP<bits<5> funct5, string opcodestr,
+                   DAGOperand immtype = stackadj>
+    : RVInst16<(outs), (ins rlist:$rlist, immtype:$stackadj),
+               opcodestr, "$rlist, $stackadj", [], InstFormatOther> {
   bits<4> rlist;
-  bits<16> spimm;
+  bits<16> stackadj;
 
   let Inst{1-0} = 0b10;
-  let Inst{3-2} = spimm{5-4};
+  let Inst{3-2} = stackadj{5-4};
   let Inst{7-4} = rlist;
   let Inst{12-8} = funct5;
   let Inst{15-13} = 0b101;
@@ -195,7 +218,7 @@ def CM_MVSA01 : RVInst16CA<0b101011, 0b01, 0b10, (outs SR07:$rs1, SR07:$rs2),
 
 let DecoderNamespace = "RVZcmp", Predicates = [HasStdExtZcmp] in {
 let hasSideEffects = 0, mayLoad = 0, mayStore = 1, Uses = [X2], Defs = [X2] in
-def CM_PUSH : RVInstZcCPPP<0b11000, "cm.push">,
+def CM_PUSH : RVInstZcCPPP<0b11000, "cm.push", negstackadj>,
               Sched<[WriteIALU, ReadIALU, ReadStoreData, ReadStoreData,
                      ReadStoreData, ReadStoreData, ReadStoreData, ReadStoreData,
                      ReadStoreData, ReadStoreData, ReadStoreData, ReadStoreData,
diff --git a/llvm/test/MC/RISCV/rv32zcmp-invalid.s b/llvm/test/MC/RISCV/rv32zcmp-invalid.s
index cb99bba0aaa1..1acea187585f 100644
--- a/llvm/test/MC/RISCV/rv32zcmp-invalid.s
+++ b/llvm/test/MC/RISCV/rv32zcmp-invalid.s
@@ -15,3 +15,15 @@ cm.popretz {ra, s0-s10}, 112
 
 # CHECK-ERROR: error: stack adjustment is invalid for this instruction and register list; refer to Zc spec for a detailed range of stack adjustment
 cm.popretz {ra, s0-s1}, 112
+
+# CHECK-ERROR: error: stack adjustment is invalid for this instruction and register list; refer to Zc spec for a detailed range of stack adjustment
+cm.push {ra}, 16
+
+# CHECK-ERROR: error: stack adjustment is invalid for this instruction and register list; refer to Zc spec for a detailed range of stack adjustment
+cm.pop {ra, s0-s1}, -32
+
+# CHECK-ERROR: error: stack adjustment is invalid for this instruction and register list; refer to Zc spec for a detailed range of stack adjustment
+cm.push {ra}, -8
+
+# CHECK-ERROR: error: stack adjustment is invalid for this instruction and register list; refer to Zc spec for a detailed range of stack adjustment
+cm.pop {ra, s0-s1}, -40
diff --git a/llvm/test/MC/RISCV/rv64zcmp-invalid.s b/llvm/test/MC/RISCV/rv64zcmp-invalid.s
index 103934583495..bf34554095ea 100644
--- a/llvm/test/MC/RISCV/rv64zcmp-invalid.s
+++ b/llvm/test/MC/RISCV/rv64zcmp-invalid.s
@@ -15,3 +15,15 @@ cm.popretz {ra, s0-s10}, 112
 
 # CHECK-ERROR: error: stack adjustment is invalid for this instruction and register list; refer to Zc spec for a detailed range of stack adjustment
 cm.popretz {ra, s0-s1}, 112
+
+# CHECK-ERROR: error: stack adjustment is invalid for this instruction and register list; refer to Zc spec for a detailed range of stack adjustment
+cm.push {ra}, 16
+
+# CHECK-ERROR: error: stack adjustment is invalid for this instruction and register list; refer to Zc spec for a detailed range of stack adjustment
+cm.pop {ra, s0-s1}, -32
+
+# CHECK-ERROR: error: stack adjustment is invalid for this instruction and register list; refer to Zc spec for a detailed range of stack adjustment
+cm.push {ra}, -15
+
+# CHECK-ERROR: error: stack adjustment is invalid for this instruction and register list; refer to Zc spec for a detailed range of stack adjustment
+cm.pop {ra, s0-s1}, -33
-- 
GitLab


From b0b8b16bb8dfb8c60fe9665c6a1c306306b716d2 Mon Sep 17 00:00:00 2001
From: Florian Mayer <fmayer@google.com>
Date: Tue, 26 Mar 2024 13:41:13 -0700
Subject: [PATCH 345/404] [NFC] [HWASan] Add link to proof for address
 calculation (#86367)

---
 llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index 3c95610fa3e8..c8f7ed448279 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -1104,6 +1104,8 @@ void HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag,
     // FIXME: the interceptor is not as fast as real memset. Consider lowering
     // llvm.memset right here into either a sequence of stores, or a call to
     // hwasan_tag_memory.
+    // Mechanical proof of this address calculation can be found at:
+    // https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/prove_hwasanwrap.smt2
     if (ShadowSize)
       IRB.CreateMemSet(ShadowPtr, Tag, ShadowSize, Align(1));
     if (Size != AlignedSize) {
-- 
GitLab


From 880eb339cae3acba9be0b2ea3271dba22ea9ec1f Mon Sep 17 00:00:00 2001
From: Florian Mayer <fmayer@google.com>
Date: Tue, 26 Mar 2024 13:46:08 -0700
Subject: [PATCH 346/404] [NFC] [HWASan] make getAndroidSlotPtr function
 generic (#86200)

This is so we can use a different slot for stack MTE.
---
 llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h  | 2 +-
 llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp | 5 ++++-
 llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp         | 4 ++--
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h b/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h
index 158c358a9e46..fb3ab33a0629 100644
--- a/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h
+++ b/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h
@@ -84,7 +84,7 @@ bool isLifetimeIntrinsic(Value *V);
 Value *readRegister(IRBuilder<> &IRB, StringRef Name);
 Value *getFP(IRBuilder<> &IRB);
 Value *getPC(const Triple &TargetTriple, IRBuilder<> &IRB);
-Value *getAndroidSanitizerSlotPtr(IRBuilder<> &IRB);
+Value *getAndroidSlotPtr(IRBuilder<> &IRB, int Slot);
 
 } // namespace memtag
 } // namespace llvm
diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index c8f7ed448279..5d366e3d6dee 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -1222,8 +1222,11 @@ Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
 }
 
 Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB) {
+  // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
+  // in Bionic's libc/platform/bionic/tls_defines.h.
+  constexpr int SanitizerSlot = 6;
   if (TargetTriple.isAArch64() && TargetTriple.isAndroid())
-    return memtag::getAndroidSanitizerSlotPtr(IRB);
+    return memtag::getAndroidSlotPtr(IRB, SanitizerSlot);
   return ThreadPtrGlobal;
 }
 
diff --git a/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp b/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp
index fd94a120bc66..7b1eb70168d8 100644
--- a/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp
+++ b/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp
@@ -273,14 +273,14 @@ Value *getFP(IRBuilder<> &IRB) {
       IRB.getIntPtrTy(M->getDataLayout()));
 }
 
-Value *getAndroidSanitizerSlotPtr(IRBuilder<> &IRB) {
+Value *getAndroidSlotPtr(IRBuilder<> &IRB, int Slot) {
   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
   // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
   // in Bionic's libc/private/bionic_tls.h.
   Function *ThreadPointerFunc =
       Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
   return IRB.CreateConstGEP1_32(IRB.getInt8Ty(),
-                                IRB.CreateCall(ThreadPointerFunc), 0x30);
+                                IRB.CreateCall(ThreadPointerFunc), 8 * Slot);
 }
 
 } // namespace memtag
-- 
GitLab


From d5224b73ccd09a6759759791f58426b6acd4a2e2 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Tue, 26 Mar 2024 14:09:39 -0700
Subject: [PATCH 347/404] [tsan] Refine fstat{,64} interceptors (#86625)

In glibc versions before 2.33. `libc_nonshared.a` defines
`__fxstat/__fxstat64` but there is no `fstat/fstat64`. glibc 2.33 added
`fstat/fstat64` and obsoleted `__fxstat/__fxstat64`. Ports added after
2.33 do not provide `__fxstat/__fxstat64`, so our `fstat/fstat64`
interceptors using `__fxstat/__fxstat64` interceptors would lead to
runtime failures on such ports (LoongArch and certain RISC-V ports).

Similar to https://reviews.llvm.org/D118423, refine the conditions that
we define fstat{,64} interceptors. `fstat` is supported by musl/*BSD
while `fstat64` is glibc only.
---
 .../lib/tsan/rtl/tsan_interceptors_posix.cpp  | 43 ++++++++-----------
 1 file changed, 18 insertions(+), 25 deletions(-)

diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
index 3103f4717c16..94adea777caf 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
@@ -14,6 +14,7 @@
 
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_glibc_version.h"
 #include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_linux.h"
 #include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
@@ -1613,47 +1614,40 @@ TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {
     FdAccess(thr, pc, fd);
   return REAL(__fxstat)(version, fd, buf);
 }
-#define TSAN_MAYBE_INTERCEPT___FXSTAT TSAN_INTERCEPT(__fxstat)
+
+TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {
+  SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf);
+  if (fd > 0)
+    FdAccess(thr, pc, fd);
+  return REAL(__fxstat64)(version, fd, buf);
+}
+#define TSAN_MAYBE_INTERCEPT___FXSTAT TSAN_INTERCEPT(__fxstat); TSAN_INTERCEPT(__fxstat64)
 #else
 #define TSAN_MAYBE_INTERCEPT___FXSTAT
 #endif
 
+#if !SANITIZER_GLIBC || __GLIBC_PREREQ(2, 33)
 TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) {
-#if SANITIZER_GLIBC
-  SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf);
-  if (fd > 0)
-    FdAccess(thr, pc, fd);
-  return REAL(__fxstat)(0, fd, buf);
-#else
   SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf);
   if (fd > 0)
     FdAccess(thr, pc, fd);
   return REAL(fstat)(fd, buf);
-#endif
-}
-
-#if SANITIZER_GLIBC
-TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {
-  SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf);
-  if (fd > 0)
-    FdAccess(thr, pc, fd);
-  return REAL(__fxstat64)(version, fd, buf);
 }
-#define TSAN_MAYBE_INTERCEPT___FXSTAT64 TSAN_INTERCEPT(__fxstat64)
+#  define TSAN_MAYBE_INTERCEPT_FSTAT TSAN_INTERCEPT(fstat)
 #else
-#define TSAN_MAYBE_INTERCEPT___FXSTAT64
+#  define TSAN_MAYBE_INTERCEPT_FSTAT
 #endif
 
-#if SANITIZER_GLIBC
+#if __GLIBC_PREREQ(2, 33)
 TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) {
-  SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf);
+  SCOPED_TSAN_INTERCEPTOR(fstat64, fd, buf);
   if (fd > 0)
     FdAccess(thr, pc, fd);
-  return REAL(__fxstat64)(0, fd, buf);
+  return REAL(fstat64)(fd, buf);
 }
-#define TSAN_MAYBE_INTERCEPT_FSTAT64 TSAN_INTERCEPT(fstat64)
+#  define TSAN_MAYBE_INTERCEPT_FSTAT64 TSAN_INTERCEPT(fstat64)
 #else
-#define TSAN_MAYBE_INTERCEPT_FSTAT64
+#  define TSAN_MAYBE_INTERCEPT_FSTAT64
 #endif
 
 TSAN_INTERCEPTOR(int, open, const char *name, int oflag, ...) {
@@ -2985,10 +2979,9 @@ void InitializeInterceptors() {
 
   TSAN_INTERCEPT(pthread_once);
 
-  TSAN_INTERCEPT(fstat);
   TSAN_MAYBE_INTERCEPT___FXSTAT;
+  TSAN_MAYBE_INTERCEPT_FSTAT;
   TSAN_MAYBE_INTERCEPT_FSTAT64;
-  TSAN_MAYBE_INTERCEPT___FXSTAT64;
   TSAN_INTERCEPT(open);
   TSAN_MAYBE_INTERCEPT_OPEN64;
   TSAN_INTERCEPT(creat);
-- 
GitLab


From 80487e1c622d51f4c0df12b64cca8a0b346e9b85 Mon Sep 17 00:00:00 2001
From: Piotr Zegar <me@piotrzegar.pl>
Date: Tue, 26 Mar 2024 21:09:54 +0000
Subject: [PATCH 348/404] [clang-tidy][NFC] Reorder entrys in release notes

Sort entrys in release notes.
---
 clang-tools-extra/docs/ReleaseNotes.rst | 26 ++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 14c48547bcbe..78b09d23d442 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -180,13 +180,13 @@ Changes in existing checks
   <clang-tidy/checks/cppcoreguidelines/owning-memory>` check to properly handle
   return type in lambdas and in nested functions.
 
-- Cleaned up :doc:`cppcoreguidelines-prefer-member-initializer
-  <clang-tidy/checks/cppcoreguidelines/prefer-member-initializer>`
+- Improved :doc:`cppcoreguidelines-prefer-member-initializer
+  <clang-tidy/checks/cppcoreguidelines/prefer-member-initializer>` check
   by removing enforcement of rule `C.48
   <https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#c48-prefer-in-class-initializers-to-member-initializers-in-constructors-for-constant-initializers>`_,
   which was deprecated since :program:`clang-tidy` 17. This rule is now covered
   by :doc:`cppcoreguidelines-use-default-member-init
-  <clang-tidy/checks/cppcoreguidelines/use-default-member-init>` and fixes
+  <clang-tidy/checks/cppcoreguidelines/use-default-member-init>`. Fixed
   incorrect hints when using list-initialization.
 
 - Improved :doc:`google-build-namespaces
@@ -236,14 +236,19 @@ Changes in existing checks
   <clang-tidy/checks/modernize/use-override>` check to also remove any trailing
   whitespace when deleting the ``virtual`` keyword.
 
+- Improved :doc:`modernize-use-using <clang-tidy/checks/modernize/use-using>`
+  check by adding support for detection of typedefs declared on function level.
+
 - Improved :doc:`performance-unnecessary-copy-initialization
   <clang-tidy/checks/performance/unnecessary-copy-initialization>` check by
   detecting more cases of constant access. In particular, pointers can be
   analyzed, se the check now handles the common patterns
   `const auto e = (*vector_ptr)[i]` and `const auto e = vector_ptr->at(i);`.
 
-- Improved :doc:`modernize-use-using <clang-tidy/checks/modernize/use-using>`
-  check by adding support for detection of typedefs declared on function level.
+- Improved :doc:`readability-identifier-naming
+  <clang-tidy/checks/readability/identifier-naming>` check in `GetConfigPerFile`
+  mode by resolving symbolic links to header files. Fixed handling of Hungarian
+  Prefix when configured to `LowerCase`.
 
 - Improved :doc:`readability-implicit-bool-conversion
   <clang-tidy/checks/readability/implicit-bool-conversion>` check to provide
@@ -254,11 +259,6 @@ Changes in existing checks
   <clang-tidy/checks/readability/redundant-inline-specifier>` check to properly
   emit warnings for static data member with an in-class initializer.
 
-- Improved :doc:`readability-identifier-naming
-  <clang-tidy/checks/readability/identifier-naming>` check in `GetConfigPerFile`
-  mode by resolving symbolic links to header files. Fixed handling of Hungarian
-  Prefix when configured to `LowerCase`.
-
 - Improved :doc:`readability-static-definition-in-anonymous-namespace
   <clang-tidy/checks/readability/static-definition-in-anonymous-namespace>`
   check by resolving fix-it overlaps in template code by disregarding implicit
@@ -273,9 +273,9 @@ Removed checks
 Miscellaneous
 ^^^^^^^^^^^^^
 
-- Fixed incorrect formatting in ``clang-apply-replacements`` when no ``--format``
-  option is specified. Now ``clang-apply-replacements`` applies formatting only with
-  the option.
+- Fixed incorrect formatting in :program:`clang-apply-replacements` when no
+  ``--format`` option is specified. Now :program:`clang-apply-replacements`
+  applies formatting only with the option.
 
 Improvements to include-fixer
 -----------------------------
-- 
GitLab


From a22bd00ce03a77a38be2911979a4c4f2ca01379d Mon Sep 17 00:00:00 2001
From: Alina Sbirlea <asbirlea@google.com>
Date: Tue, 26 Mar 2024 14:18:41 -0700
Subject: [PATCH 349/404] [libc] [bazel] [NFC] Format deps.

---
 utils/bazel/llvm-project-overlay/libc/BUILD.bazel | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 57dc75a3a48e..e0790e0ab59e 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -622,12 +622,12 @@ libc_support_library(
     name = "__support_fputil_basic_operations",
     hdrs = ["src/__support/FPUtil/BasicOperations.h"],
     deps = [
-        ":__support_fputil_fenv_impl",
-        ":__support_macros_optimization",
-        ":__support_uint128",
         ":__support_common",
         ":__support_cpp_type_traits",
+        ":__support_fputil_fenv_impl",
         ":__support_fputil_fp_bits",
+        ":__support_macros_optimization",
+        ":__support_uint128",
     ],
 )
 
-- 
GitLab


From c6d419c15bf836085392212b8ab7600f7402829b Mon Sep 17 00:00:00 2001
From: Matthias Gehre <matthias.gehre@amd.com>
Date: Tue, 26 Mar 2024 22:27:11 +0100
Subject: [PATCH 350/404] [TOSA] Allow all integer types in most ops (#86509)

As discussed in one of the previous TOSA community meetings, we would
like to allow for more integer types in the TOSA dialect to enable more
use cases.

For strict standards conformance, the TosaValidation pass can be used.

Follow up PRs will extend conversions from TOSA where needed.
---
 mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td  |  2 +-
 .../mlir/Dialect/Tosa/IR/TosaTypesBase.td     | 27 +++-------
 .../Tosa/Transforms/TosaValidation.cpp        | 51 +++++++++++++++++--
 mlir/test/Dialect/Tosa/level_check.mlir       | 16 ++++++
 4 files changed, 71 insertions(+), 25 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
index 0ecded75c5d8..306e4a439520 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
@@ -1942,7 +1942,7 @@ def Tosa_ConstOp : Tosa_Op<"const", [ConstantLike, Pure,
   );
 
   let results = (outs
-    TensorOf<[AnyTypeOf<[Tosa_AnyNumber_Plus_F64, Tosa_Int4]>]>:$output
+    TensorOf<[AnyTypeOf<[Tosa_AnyNumber_Plus_F64]>]>:$output
   );
 
   let hasFolder = 1;
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
index 5a4d6ff464f1..cff3de0a69af 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
@@ -38,29 +38,17 @@ class Tosa_QuantizedType<string n, list<int> params, bit signed>
 // Used to express accumulator results or compare results.
 //===----------------------------------------------------------------------===//
 
-def Tosa_UInt8 : UI<8>;
-def Tosa_UInt16 : UI<16>;
-
 def Tosa_Int4 : I<4>;
 def Tosa_Int8 : I<8>;
-def Tosa_Int16 : I<16>;
 def Tosa_Int32 : I<32>;
-def Tosa_Int48 : I<48>;
 def Tosa_Int64 : I<64>;
 
-def Tosa_SignedInt : AnyTypeOf<[Tosa_Int8,
-                                Tosa_Int16,
-                                Tosa_Int32,
-                                Tosa_Int48,
-                                Tosa_Int64]>;
-
-def Tosa_Bool : I<1>;
-
-// No unsigned unquantized int types.
-def Tosa_Int : AnyTypeOf<[Tosa_Bool,
-                          Tosa_UInt8,
-                          Tosa_UInt16,
-                          Tosa_SignedInt]>;
+// The TOSA dialect allows more types than the TOSA standard to allow for
+// experimentation. For historical reasons, signless is used in the place of
+// signed.
+// The TosaValidation pass can be used to check for standard conformance.
+def Tosa_Int : AnyTypeOf<[AnyUnsignedInteger,
+                          AnySignlessInteger]>;
 
 def Tosa_Int32Or64 : AnyTypeOf<[Tosa_Int32,
                    	        Tosa_Int64]>;
@@ -172,9 +160,6 @@ class Tosa_TypeLike<list<Type> types, string description = ""> : TypeConstraint<
 
 def Tosa_IntLike : Tosa_TypeLike<[Tosa_Int], "signless-integer-like">;
 def Tosa_Int8Like : Tosa_TypeLike<[Tosa_Int8], "signless-integer-8-bit-like">;
-def Tosa_Int16Like : Tosa_TypeLike<[Tosa_Int16], "signless-integer-16-bit-like">;
-def Tosa_Int32Like : Tosa_TypeLike<[Tosa_Int32], "signless-integer-32-bit-like">;
-def Tosa_Int64Like : Tosa_TypeLike<[Tosa_Int64], "signless-integer-64-bit-like">;
 
 //===----------------------------------------------------------------------===//
 // Attribute predicates and classes.
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
index 967775281ad9..74ef6381f3d7 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
@@ -410,6 +410,8 @@ private:
   bool CheckVariable(Operation *op);
   bool CheckVariableReadOrWrite(Operation *op);
 
+  bool isValidElementType(Type type);
+
   SmallVector<std::function<LogicalResult(Operation *)>> constCheckers;
   TosaLevel tosaLevel;
   DenseMap<StringAttr, mlir::Type> variablesMap;
@@ -503,15 +505,58 @@ LogicalResult TosaValidation::applyVariableCheck(Operation *op) {
   return success();
 }
 
+bool TosaValidation::isValidElementType(Type type) {
+  if ((profile == TosaProfileEnum::BaseInference) && isa<FloatType>(type)) {
+    return false;
+  }
+  if (type.isF64()) {
+    return false;
+  }
+  if (auto intTy = dyn_cast<IntegerType>(type)) {
+    if (intTy.isUnsigned()) {
+      switch (intTy.getWidth()) {
+      case 8:
+      case 16:
+        return true;
+      default:
+        return false;
+      }
+    } else {
+      // Signless - treated as signed.
+      switch (intTy.getWidth()) {
+      case 1:
+      case 4:
+      case 8:
+      case 16:
+      case 32:
+      case 48:
+      case 64:
+        return true;
+      default:
+        return false;
+      }
+    }
+    return false;
+  }
+  return true;
+}
+
 void TosaValidation::runOnOperation() {
   configLevelAndProfile();
   getOperation().walk([&](Operation *op) {
     for (Value operand : op->getOperands()) {
-      if ((profile == TosaProfileEnum::BaseInference) &&
-          isa<FloatType>(getElementTypeOrSelf(operand))) {
+      auto elementTy = getElementTypeOrSelf(operand);
+      if (!isValidElementType(elementTy)) {
+        op->emitOpError() << "is not profile-aligned: element type "
+                          << elementTy << " is not legal";
         return signalPassFailure();
       }
-      if (getElementTypeOrSelf(operand).isF64()) {
+    }
+    for (Type resultTy : op->getResultTypes()) {
+      auto elementTy = getElementTypeOrSelf(resultTy);
+      if (!isValidElementType(elementTy)) {
+        op->emitOpError() << "is not profile-aligned: element type "
+                          << elementTy << " is not legal";
         return signalPassFailure();
       }
     }
diff --git a/mlir/test/Dialect/Tosa/level_check.mlir b/mlir/test/Dialect/Tosa/level_check.mlir
index 35ecbcc799e3..d8dd878051f1 100644
--- a/mlir/test/Dialect/Tosa/level_check.mlir
+++ b/mlir/test/Dialect/Tosa/level_check.mlir
@@ -115,6 +115,22 @@ func.func @test_const(%arg0 : tensor<1x1xi32>) -> tensor<1x1x1x1x1x1x1xi32> {
 
 // -----
 
+func.func @test_const_i2(%arg0 : tensor<1xi2>) {
+  // expected-error@+1 {{'tosa.const' op is not profile-aligned: element type 'i2' is not legal}}
+  %0 = "tosa.const"() {value = dense<0> : tensor<1xi2>} : () -> tensor<1xi2>
+  return
+}
+
+// -----
+
+func.func @test_const_ui32(%arg0 : tensor<1xui32>) {
+  // expected-error@+1 {{'tosa.const' op is not profile-aligned: element type 'ui32' is not legal}}
+  %0 = "tosa.const"() {value = dense<0> : tensor<1xui32>} : () -> tensor<1xui32>
+  return
+}
+
+// -----
+
 func.func @test_avgpool2d_kernel_y(%arg0: tensor<1x32x32x8xf32>) -> tensor<1x32x32x8xf32> {
   // expected-error@+1 {{'tosa.avg_pool2d' op failed level check: kernel <= MAX_KERNEL}}
   %0 = "tosa.avg_pool2d"(%arg0) {kernel = array<i64: 8193, 1>, pad = array<i64: 4, 4, 4, 4>, stride = array<i64: 1, 1>, acc_type = f32} :
-- 
GitLab


From 630283c38bce9ec188c96ddb90b8251afc2b3d8e Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Tue, 26 Mar 2024 16:35:32 -0500
Subject: [PATCH 351/404] [libc] Fix misplaced `[[noreturn]]` attribute.

Summary:
This needs to go after `extern "C"`.
---
 libc/src/__support/OSUtil/baremetal/quick_exit.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libc/src/__support/OSUtil/baremetal/quick_exit.cpp b/libc/src/__support/OSUtil/baremetal/quick_exit.cpp
index f0971d4db352..5b6fcf42341e 100644
--- a/libc/src/__support/OSUtil/baremetal/quick_exit.cpp
+++ b/libc/src/__support/OSUtil/baremetal/quick_exit.cpp
@@ -9,7 +9,7 @@
 #include "src/__support/OSUtil/quick_exit.h"
 
 // This is intended to be provided by the vendor.
-[[noreturn]] extern "C" void __llvm_libc_quick_exit(int status);
+extern "C" [[noreturn]] void __llvm_libc_quick_exit(int status);
 
 namespace LIBC_NAMESPACE {
 
-- 
GitLab


From 1949f7d6c9bd59172c01c7933e1c558797c47eac Mon Sep 17 00:00:00 2001
From: Christopher Ferris <cferris1000@users.noreply.github.com>
Date: Tue, 26 Mar 2024 14:47:48 -0700
Subject: [PATCH 352/404] [scudo] Clean up string handling (#86364)

Do not abort if a vector cannot increase its own capacity. In that case,
push_back calls silently fail.

Modify the ScopedString implementation so that it no longer requires two
passes to do the format. Move the helper functions to be private member
functions so that they can use push_back directly. This allows the
capacity to be increased under the hood and/or silently discards data if
the capacity is exceeded and cannot be increased.

Add new tests for the Vector and ScopedString for capacity increase
failures.

Doing this so that if a map call fails, and we are attempting to write
an error string, we can still get some of the message dumped. This also
avoids crashing in Scudo code, and makes the caller handle any failures.
---
 compiler-rt/lib/scudo/standalone/fuchsia.cpp  |   7 +-
 .../lib/scudo/standalone/mem_map_fuchsia.cpp  |   7 +-
 .../lib/scudo/standalone/report_linux.cpp     |  25 ++-
 .../lib/scudo/standalone/string_utils.cpp     | 148 +++++++-----------
 .../lib/scudo/standalone/string_utils.h       |  13 +-
 .../scudo/standalone/tests/strings_test.cpp   |  32 ++++
 .../scudo/standalone/tests/vector_test.cpp    |  35 +++++
 compiler-rt/lib/scudo/standalone/vector.h     |  21 ++-
 8 files changed, 165 insertions(+), 123 deletions(-)

diff --git a/compiler-rt/lib/scudo/standalone/fuchsia.cpp b/compiler-rt/lib/scudo/standalone/fuchsia.cpp
index 0788c4198e53..2144f1b63f89 100644
--- a/compiler-rt/lib/scudo/standalone/fuchsia.cpp
+++ b/compiler-rt/lib/scudo/standalone/fuchsia.cpp
@@ -34,11 +34,10 @@ static_assert(ZX_HANDLE_INVALID == 0, "");
 
 static void NORETURN dieOnError(zx_status_t Status, const char *FnName,
                                 uptr Size) {
-  char Error[128];
-  formatString(Error, sizeof(Error),
-               "SCUDO ERROR: %s failed with size %zuKB (%s)", FnName,
+  ScopedString Error;
+  Error.append("SCUDO ERROR: %s failed with size %zuKB (%s)", FnName,
                Size >> 10, zx_status_get_string(Status));
-  outputRaw(Error);
+  outputRaw(Error.data());
   die();
 }
 
diff --git a/compiler-rt/lib/scudo/standalone/mem_map_fuchsia.cpp b/compiler-rt/lib/scudo/standalone/mem_map_fuchsia.cpp
index 0566ab065526..28e5a11a37f2 100644
--- a/compiler-rt/lib/scudo/standalone/mem_map_fuchsia.cpp
+++ b/compiler-rt/lib/scudo/standalone/mem_map_fuchsia.cpp
@@ -22,11 +22,10 @@ namespace scudo {
 
 static void NORETURN dieOnError(zx_status_t Status, const char *FnName,
                                 uptr Size) {
-  char Error[128];
-  formatString(Error, sizeof(Error),
-               "SCUDO ERROR: %s failed with size %zuKB (%s)", FnName,
+  ScopedString Error;
+  Error.append("SCUDO ERROR: %s failed with size %zuKB (%s)", FnName,
                Size >> 10, _zx_status_get_string(Status));
-  outputRaw(Error);
+  outputRaw(Error.data());
   die();
 }
 
diff --git a/compiler-rt/lib/scudo/standalone/report_linux.cpp b/compiler-rt/lib/scudo/standalone/report_linux.cpp
index 6a983036e6cd..dfddef3324bd 100644
--- a/compiler-rt/lib/scudo/standalone/report_linux.cpp
+++ b/compiler-rt/lib/scudo/standalone/report_linux.cpp
@@ -24,33 +24,30 @@ namespace scudo {
 
 // Fatal internal map() error (potentially OOM related).
 void NORETURN reportMapError(uptr SizeIfOOM) {
-  char Error[128] = "Scudo ERROR: internal map failure\n";
+  ScopedString Error;
+  Error.append("Scudo ERROR: internal map failure");
   if (SizeIfOOM) {
-    formatString(
-        Error, sizeof(Error),
-        "Scudo ERROR: internal map failure (NO MEMORY) requesting %zuKB\n",
-        SizeIfOOM >> 10);
+    Error.append(" (NO MEMORY) requesting %zuKB", SizeIfOOM >> 10);
   }
-  reportRawError(Error);
+  Error.append("\n");
+  reportRawError(Error.data());
 }
 
 void NORETURN reportUnmapError(uptr Addr, uptr Size) {
-  char Error[128];
-  formatString(Error, sizeof(Error),
-               "Scudo ERROR: internal unmap failure (error desc=%s) Addr 0x%zx "
+  ScopedString Error;
+  Error.append("Scudo ERROR: internal unmap failure (error desc=%s) Addr 0x%zx "
                "Size %zu\n",
                strerror(errno), Addr, Size);
-  reportRawError(Error);
+  reportRawError(Error.data());
 }
 
 void NORETURN reportProtectError(uptr Addr, uptr Size, int Prot) {
-  char Error[128];
-  formatString(
-      Error, sizeof(Error),
+  ScopedString Error;
+  Error.append(
       "Scudo ERROR: internal protect failure (error desc=%s) Addr 0x%zx "
       "Size %zu Prot %x\n",
       strerror(errno), Addr, Size, Prot);
-  reportRawError(Error);
+  reportRawError(Error.data());
 }
 
 } // namespace scudo
diff --git a/compiler-rt/lib/scudo/standalone/string_utils.cpp b/compiler-rt/lib/scudo/standalone/string_utils.cpp
index d4e4e3becd0e..e584bd806e57 100644
--- a/compiler-rt/lib/scudo/standalone/string_utils.cpp
+++ b/compiler-rt/lib/scudo/standalone/string_utils.cpp
@@ -14,30 +14,21 @@
 
 namespace scudo {
 
-static int appendChar(char **Buffer, const char *BufferEnd, char C) {
-  if (*Buffer < BufferEnd) {
-    **Buffer = C;
-    (*Buffer)++;
-  }
-  return 1;
-}
-
 // Appends number in a given Base to buffer. If its length is less than
 // |MinNumberLength|, it is padded with leading zeroes or spaces, depending
 // on the value of |PadWithZero|.
-static int appendNumber(char **Buffer, const char *BufferEnd, u64 AbsoluteValue,
-                        u8 Base, u8 MinNumberLength, bool PadWithZero,
-                        bool Negative, bool Upper) {
+void ScopedString::appendNumber(u64 AbsoluteValue, u8 Base, u8 MinNumberLength,
+                                bool PadWithZero, bool Negative, bool Upper) {
   constexpr uptr MaxLen = 30;
   RAW_CHECK(Base == 10 || Base == 16);
   RAW_CHECK(Base == 10 || !Negative);
   RAW_CHECK(AbsoluteValue || !Negative);
   RAW_CHECK(MinNumberLength < MaxLen);
-  int Res = 0;
   if (Negative && MinNumberLength)
     --MinNumberLength;
-  if (Negative && PadWithZero)
-    Res += appendChar(Buffer, BufferEnd, '-');
+  if (Negative && PadWithZero) {
+    String.push_back('-');
+  }
   uptr NumBuffer[MaxLen];
   int Pos = 0;
   do {
@@ -55,34 +46,32 @@ static int appendNumber(char **Buffer, const char *BufferEnd, u64 AbsoluteValue,
   Pos--;
   for (; Pos >= 0 && NumBuffer[Pos] == 0; Pos--) {
     char c = (PadWithZero || Pos == 0) ? '0' : ' ';
-    Res += appendChar(Buffer, BufferEnd, c);
+    String.push_back(c);
   }
   if (Negative && !PadWithZero)
-    Res += appendChar(Buffer, BufferEnd, '-');
+    String.push_back('-');
   for (; Pos >= 0; Pos--) {
     char Digit = static_cast<char>(NumBuffer[Pos]);
     Digit = static_cast<char>((Digit < 10) ? '0' + Digit
                                            : (Upper ? 'A' : 'a') + Digit - 10);
-    Res += appendChar(Buffer, BufferEnd, Digit);
+    String.push_back(Digit);
   }
-  return Res;
 }
 
-static int appendUnsigned(char **Buffer, const char *BufferEnd, u64 Num,
-                          u8 Base, u8 MinNumberLength, bool PadWithZero,
-                          bool Upper) {
-  return appendNumber(Buffer, BufferEnd, Num, Base, MinNumberLength,
-                      PadWithZero, /*Negative=*/false, Upper);
+void ScopedString::appendUnsigned(u64 Num, u8 Base, u8 MinNumberLength,
+                                  bool PadWithZero, bool Upper) {
+  appendNumber(Num, Base, MinNumberLength, PadWithZero, /*Negative=*/false,
+               Upper);
 }
 
-static int appendSignedDecimal(char **Buffer, const char *BufferEnd, s64 Num,
-                               u8 MinNumberLength, bool PadWithZero) {
+void ScopedString::appendSignedDecimal(s64 Num, u8 MinNumberLength,
+                                       bool PadWithZero) {
   const bool Negative = (Num < 0);
   const u64 UnsignedNum = (Num == INT64_MIN)
                               ? static_cast<u64>(INT64_MAX) + 1
                               : static_cast<u64>(Negative ? -Num : Num);
-  return appendNumber(Buffer, BufferEnd, UnsignedNum, 10, MinNumberLength,
-                      PadWithZero, Negative, /*Upper=*/false);
+  appendNumber(UnsignedNum, 10, MinNumberLength, PadWithZero, Negative,
+               /*Upper=*/false);
 }
 
 // Use the fact that explicitly requesting 0 Width (%0s) results in UB and
@@ -90,44 +79,45 @@ static int appendSignedDecimal(char **Buffer, const char *BufferEnd, s64 Num,
 // Width == 0 - no Width requested
 // Width  < 0 - left-justify S within and pad it to -Width chars, if necessary
 // Width  > 0 - right-justify S, not implemented yet
-static int appendString(char **Buffer, const char *BufferEnd, int Width,
-                        int MaxChars, const char *S) {
+void ScopedString::appendString(int Width, int MaxChars, const char *S) {
   if (!S)
     S = "<null>";
-  int Res = 0;
+  int NumChars = 0;
   for (; *S; S++) {
-    if (MaxChars >= 0 && Res >= MaxChars)
+    if (MaxChars >= 0 && NumChars >= MaxChars)
       break;
-    Res += appendChar(Buffer, BufferEnd, *S);
+    String.push_back(*S);
+    NumChars++;
+  }
+  if (Width < 0) {
+    // Only left justification supported.
+    Width = -Width - NumChars;
+    while (Width-- > 0)
+      String.push_back(' ');
   }
-  // Only the left justified strings are supported.
-  while (Width < -Res)
-    Res += appendChar(Buffer, BufferEnd, ' ');
-  return Res;
 }
 
-static int appendPointer(char **Buffer, const char *BufferEnd, u64 ptr_value) {
-  int Res = 0;
-  Res += appendString(Buffer, BufferEnd, 0, -1, "0x");
-  Res += appendUnsigned(Buffer, BufferEnd, ptr_value, 16,
-                        SCUDO_POINTER_FORMAT_LENGTH, /*PadWithZero=*/true,
-                        /*Upper=*/false);
-  return Res;
+void ScopedString::appendPointer(u64 ptr_value) {
+  appendString(0, -1, "0x");
+  appendUnsigned(ptr_value, 16, SCUDO_POINTER_FORMAT_LENGTH,
+                 /*PadWithZero=*/true,
+                 /*Upper=*/false);
 }
 
-static int formatString(char *Buffer, uptr BufferLength, const char *Format,
-                        va_list Args) {
+void ScopedString::vappend(const char *Format, va_list &Args) {
+  // Since the string contains the '\0' terminator, put our size before it
+  // so that push_back calls work correctly.
+  DCHECK(String.size() > 0);
+  String.resize(String.size() - 1);
+
   static const char *PrintfFormatsHelp =
-      "Supported formatString formats: %([0-9]*)?(z|ll)?{d,u,x,X}; %p; "
+      "Supported formats: %([0-9]*)?(z|ll)?{d,u,x,X}; %p; "
       "%[-]([0-9]*)?(\\.\\*)?s; %c\n";
   RAW_CHECK(Format);
-  RAW_CHECK(BufferLength > 0);
-  const char *BufferEnd = &Buffer[BufferLength - 1];
   const char *Cur = Format;
-  int Res = 0;
   for (; *Cur; Cur++) {
     if (*Cur != '%') {
-      Res += appendChar(&Buffer, BufferEnd, *Cur);
+      String.push_back(*Cur);
       continue;
     }
     Cur++;
@@ -162,7 +152,7 @@ static int formatString(char *Buffer, uptr BufferLength, const char *Format,
       DVal = HaveLL  ? va_arg(Args, s64)
              : HaveZ ? va_arg(Args, sptr)
                      : va_arg(Args, int);
-      Res += appendSignedDecimal(&Buffer, BufferEnd, DVal, Width, PadWithZero);
+      appendSignedDecimal(DVal, Width, PadWithZero);
       break;
     }
     case 'u':
@@ -172,27 +162,25 @@ static int formatString(char *Buffer, uptr BufferLength, const char *Format,
              : HaveZ ? va_arg(Args, uptr)
                      : va_arg(Args, unsigned);
       const bool Upper = (*Cur == 'X');
-      Res += appendUnsigned(&Buffer, BufferEnd, UVal, (*Cur == 'u') ? 10 : 16,
-                            Width, PadWithZero, Upper);
+      appendUnsigned(UVal, (*Cur == 'u') ? 10 : 16, Width, PadWithZero, Upper);
       break;
     }
     case 'p': {
       RAW_CHECK_MSG(!HaveFlags, PrintfFormatsHelp);
-      Res += appendPointer(&Buffer, BufferEnd, va_arg(Args, uptr));
+      appendPointer(va_arg(Args, uptr));
       break;
     }
     case 's': {
       RAW_CHECK_MSG(!HaveLength, PrintfFormatsHelp);
       // Only left-justified Width is supported.
       CHECK(!HaveWidth || LeftJustified);
-      Res += appendString(&Buffer, BufferEnd, LeftJustified ? -Width : Width,
-                          Precision, va_arg(Args, char *));
+      appendString(LeftJustified ? -Width : Width, Precision,
+                   va_arg(Args, char *));
       break;
     }
     case 'c': {
       RAW_CHECK_MSG(!HaveFlags, PrintfFormatsHelp);
-      Res +=
-          appendChar(&Buffer, BufferEnd, static_cast<char>(va_arg(Args, int)));
+      String.push_back(static_cast<char>(va_arg(Args, int)));
       break;
     }
     // In Scudo, `s64`/`u64` are supposed to use `lld` and `llu` respectively.
@@ -207,19 +195,17 @@ static int formatString(char *Buffer, uptr BufferLength, const char *Format,
 
       if (*Cur == 'd') {
         DVal = va_arg(Args, s64);
-        Res +=
-            appendSignedDecimal(&Buffer, BufferEnd, DVal, Width, PadWithZero);
+        appendSignedDecimal(DVal, Width, PadWithZero);
       } else {
         UVal = va_arg(Args, u64);
-        Res += appendUnsigned(&Buffer, BufferEnd, UVal, 10, Width, PadWithZero,
-                              false);
+        appendUnsigned(UVal, 10, Width, PadWithZero, false);
       }
 
       break;
     }
     case '%': {
       RAW_CHECK_MSG(!HaveFlags, PrintfFormatsHelp);
-      Res += appendChar(&Buffer, BufferEnd, '%');
+      String.push_back('%');
       break;
     }
     default: {
@@ -227,35 +213,13 @@ static int formatString(char *Buffer, uptr BufferLength, const char *Format,
     }
     }
   }
-  RAW_CHECK(Buffer <= BufferEnd);
-  appendChar(&Buffer, BufferEnd + 1, '\0');
-  return Res;
-}
-
-int formatString(char *Buffer, uptr BufferLength, const char *Format, ...) {
-  va_list Args;
-  va_start(Args, Format);
-  int Res = formatString(Buffer, BufferLength, Format, Args);
-  va_end(Args);
-  return Res;
-}
-
-void ScopedString::vappend(const char *Format, va_list Args) {
-  va_list ArgsCopy;
-  va_copy(ArgsCopy, Args);
-  // formatString doesn't currently support a null buffer or zero buffer length,
-  // so in order to get the resulting formatted string length, we use a one-char
-  // buffer.
-  char C[1];
-  const uptr AdditionalLength =
-      static_cast<uptr>(formatString(C, sizeof(C), Format, Args)) + 1;
-  const uptr Length = length();
-  String.resize(Length + AdditionalLength);
-  const uptr FormattedLength = static_cast<uptr>(formatString(
-      String.data() + Length, String.size() - Length, Format, ArgsCopy));
-  RAW_CHECK(data()[length()] == '\0');
-  RAW_CHECK(FormattedLength + 1 == AdditionalLength);
-  va_end(ArgsCopy);
+  String.push_back('\0');
+  if (String.back() != '\0') {
+    // String truncated, make sure the string is terminated properly.
+    // This can happen if there is no more memory when trying to resize
+    // the string.
+    String.back() = '\0';
+  }
 }
 
 void ScopedString::append(const char *Format, ...) {
diff --git a/compiler-rt/lib/scudo/standalone/string_utils.h b/compiler-rt/lib/scudo/standalone/string_utils.h
index a4cab5268ede..6e00b6377973 100644
--- a/compiler-rt/lib/scudo/standalone/string_utils.h
+++ b/compiler-rt/lib/scudo/standalone/string_utils.h
@@ -25,17 +25,24 @@ public:
     String.clear();
     String.push_back('\0');
   }
-  void vappend(const char *Format, va_list Args);
+  void vappend(const char *Format, va_list &Args);
   void append(const char *Format, ...) FORMAT(2, 3);
   void output() const { outputRaw(String.data()); }
   void reserve(size_t Size) { String.reserve(Size + 1); }
+  uptr capacity() { return String.capacity() - 1; }
 
 private:
+  void appendNumber(u64 AbsoluteValue, u8 Base, u8 MinNumberLength,
+                    bool PadWithZero, bool Negative, bool Upper);
+  void appendUnsigned(u64 Num, u8 Base, u8 MinNumberLength, bool PadWithZero,
+                      bool Upper);
+  void appendSignedDecimal(s64 Num, u8 MinNumberLength, bool PadWithZero);
+  void appendString(int Width, int MaxChars, const char *S);
+  void appendPointer(u64 ptr_value);
+
   Vector<char> String;
 };
 
-int formatString(char *Buffer, uptr BufferLength, const char *Format, ...)
-    FORMAT(3, 4);
 void Printf(const char *Format, ...) FORMAT(1, 2);
 
 } // namespace scudo
diff --git a/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp b/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp
index 7a69ffd9762c..e068c48fc97c 100644
--- a/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/strings_test.cpp
@@ -66,6 +66,10 @@ TEST(ScudoStringsTest, Precision) {
   Str.append("%-6s", "12345");
   EXPECT_EQ(Str.length(), strlen(Str.data()));
   EXPECT_STREQ("12345 ", Str.data());
+  Str.clear();
+  Str.append("%-8s", "12345");
+  EXPECT_EQ(Str.length(), strlen(Str.data()));
+  EXPECT_STREQ("12345   ", Str.data());
 }
 
 static void fillString(scudo::ScopedString &Str, scudo::uptr Size) {
@@ -123,3 +127,31 @@ TEST(ScudoStringsTest, Padding) {
   testAgainstLibc<int>("%03d - %03d", 12, 1234);
   testAgainstLibc<int>("%03d - %03d", -12, -1234);
 }
+
+#if defined(__linux__)
+#include <sys/resource.h>
+
+TEST(ScudoStringsTest, CapacityIncreaseFails) {
+  scudo::ScopedString Str;
+
+  rlimit Limit = {};
+  EXPECT_EQ(0, getrlimit(RLIMIT_AS, &Limit));
+  rlimit EmptyLimit = {.rlim_max = Limit.rlim_max};
+  EXPECT_EQ(0, setrlimit(RLIMIT_AS, &EmptyLimit));
+
+  // Test requires that the default length is at least 6 characters.
+  scudo::uptr MaxSize = Str.capacity();
+  EXPECT_LE(6, MaxSize);
+
+  for (size_t i = 0; i < MaxSize - 5; i++) {
+    Str.append("B");
+  }
+
+  // Attempt to append past the end of the current capacity.
+  Str.append("%d", 12345678);
+  EXPECT_EQ(MaxSize, Str.capacity());
+  EXPECT_STREQ("B12345", &Str.data()[MaxSize - 6]);
+
+  EXPECT_EQ(0, setrlimit(RLIMIT_AS, &Limit));
+}
+#endif
diff --git a/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp b/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp
index dc23c2a34713..b7678678d8a2 100644
--- a/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/vector_test.cpp
@@ -41,3 +41,38 @@ TEST(ScudoVectorTest, ResizeReduction) {
   V.resize(1);
   EXPECT_EQ(V.size(), 1U);
 }
+
+#if defined(__linux__)
+
+#include <sys/resource.h>
+
+// Verify that if the reallocate fails, nothing new is added.
+TEST(ScudoVectorTest, ReallocateFails) {
+  scudo::Vector<char> V;
+  scudo::uptr capacity = V.capacity();
+
+  // Get the current address space size.
+  rlimit Limit = {};
+  EXPECT_EQ(0, getrlimit(RLIMIT_AS, &Limit));
+
+  rlimit EmptyLimit = {.rlim_max = Limit.rlim_max};
+  EXPECT_EQ(0, setrlimit(RLIMIT_AS, &EmptyLimit));
+
+  V.resize(capacity);
+  // Set the last element so we can check it later.
+  V.back() = '\0';
+
+  // The reallocate should fail, so the capacity should not change.
+  V.reserve(capacity + 1000);
+  EXPECT_EQ(capacity, V.capacity());
+
+  // Now try to do a push back and verify that the size does not change.
+  scudo::uptr Size = V.size();
+  V.push_back('2');
+  EXPECT_EQ(Size, V.size());
+  // Verify that the last element in the vector did not change.
+  EXPECT_EQ('\0', V.back());
+
+  EXPECT_EQ(0, setrlimit(RLIMIT_AS, &Limit));
+}
+#endif
diff --git a/compiler-rt/lib/scudo/standalone/vector.h b/compiler-rt/lib/scudo/standalone/vector.h
index c0f1ba0eddfa..ca10cc281d77 100644
--- a/compiler-rt/lib/scudo/standalone/vector.h
+++ b/compiler-rt/lib/scudo/standalone/vector.h
@@ -35,7 +35,9 @@ public:
     DCHECK_LE(Size, capacity());
     if (Size == capacity()) {
       const uptr NewCapacity = roundUpPowerOfTwo(Size + 1);
-      reallocate(NewCapacity);
+      if (!reallocate(NewCapacity)) {
+        return;
+      }
     }
     memcpy(&Data[Size++], &Element, sizeof(T));
   }
@@ -51,14 +53,17 @@ public:
   const T *data() const { return Data; }
   T *data() { return Data; }
   constexpr uptr capacity() const { return CapacityBytes / sizeof(T); }
-  void reserve(uptr NewSize) {
+  bool reserve(uptr NewSize) {
     // Never downsize internal buffer.
     if (NewSize > capacity())
-      reallocate(NewSize);
+      return reallocate(NewSize);
+    return true;
   }
   void resize(uptr NewSize) {
     if (NewSize > Size) {
-      reserve(NewSize);
+      if (!reserve(NewSize)) {
+        return;
+      }
       memset(&Data[Size], 0, sizeof(T) * (NewSize - Size));
     }
     Size = NewSize;
@@ -86,13 +91,16 @@ protected:
   }
 
 private:
-  void reallocate(uptr NewCapacity) {
+  bool reallocate(uptr NewCapacity) {
     DCHECK_GT(NewCapacity, 0);
     DCHECK_LE(Size, NewCapacity);
 
     MemMapT NewExternalBuffer;
     NewCapacity = roundUp(NewCapacity * sizeof(T), getPageSizeCached());
-    NewExternalBuffer.map(/*Addr=*/0U, NewCapacity, "scudo:vector");
+    if (!NewExternalBuffer.map(/*Addr=*/0U, NewCapacity, "scudo:vector",
+                               MAP_ALLOWNOMEM)) {
+      return false;
+    }
     T *NewExternalData = reinterpret_cast<T *>(NewExternalBuffer.getBase());
 
     memcpy(NewExternalData, Data, Size * sizeof(T));
@@ -101,6 +109,7 @@ private:
     Data = NewExternalData;
     CapacityBytes = NewCapacity;
     ExternalBuffer = NewExternalBuffer;
+    return true;
   }
 
   T *Data = nullptr;
-- 
GitLab


From 80bba17914dec52789d2e75ed560acb11cce959a Mon Sep 17 00:00:00 2001
From: Nick Desaulniers <nickdesaulniers@users.noreply.github.com>
Date: Tue, 26 Mar 2024 14:53:04 -0700
Subject: [PATCH 353/404] [libc][FPUtil] fixup missing explicit cast (#86736)

The arm32 buildbot reports an error because UInt::operator bool() is explicit,
thus an explicit cast is necessary.

Link: #85940
---
 libc/src/__support/FPUtil/BasicOperations.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libc/src/__support/FPUtil/BasicOperations.h b/libc/src/__support/FPUtil/BasicOperations.h
index 405755f8b57d..f746d7ac6ad4 100644
--- a/libc/src/__support/FPUtil/BasicOperations.h
+++ b/libc/src/__support/FPUtil/BasicOperations.h
@@ -199,7 +199,7 @@ LIBC_INLINE int canonicalize(T &cx, const T &x) {
     //  Values    |            |           | (−1)**s × m × 2**−16382
     bool bit63 = sx.get_implicit_bit();
     UInt128 mantissa = sx.get_explicit_mantissa();
-    bool bit62 = ((mantissa & (1ULL << 62)) >> 62);
+    bool bit62 = static_cast<bool>((mantissa & (1ULL << 62)) >> 62);
     int exponent = sx.get_biased_exponent();
     if (exponent == 0x7FFF) {
       if (!bit63 && !bit62) {
-- 
GitLab


From 86692258637549ed9f863c3d2ba47b49f61bbc1f Mon Sep 17 00:00:00 2001
From: Jon Chesterfield <jonathanchesterfield@gmail.com>
Date: Tue, 26 Mar 2024 22:33:03 +0000
Subject: [PATCH 354/404] [compiler-rt] Allow building builtins.a without a
 libc (#86737)

compiler-rt may depend on libc (memset etc). Likewise a libc built by
clang may depend on compiler-rt builtins.

This circular dependency doesn't matter much once they're both compiled.
The easy compilation order to build both from source is:

1. install libc headers somewhere
2. build compiler-rt builtins against those headers
3. build libc against compiler-rt builtins

This patch relaxes the cmake sanity check to pass without requiring a
libc library. That allows the above sequence to work. Otherwise one
needs to build a static libc, then use that to pass the compiler-rt
cmake check, then build a normal libc.
---
 compiler-rt/cmake/config-ix.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake
index 46a6fdf8728f..911f48fa1381 100644
--- a/compiler-rt/cmake/config-ix.cmake
+++ b/compiler-rt/cmake/config-ix.cmake
@@ -235,9 +235,9 @@ set(COMPILER_RT_SUPPORTED_ARCH)
 # Try to compile a very simple source file to ensure we can target the given
 # platform. We use the results of these tests to build only the various target
 # runtime libraries supported by our current compilers cross-compiling
-# abilities.
+# abilities. Avoids using libc as that may not be available yet.
 set(SIMPLE_SOURCE ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/simple.cc)
-file(WRITE ${SIMPLE_SOURCE} "#include <stdlib.h>\n#include <stdio.h>\nint main(void) { printf(\"hello, world\"); }\n")
+file(WRITE ${SIMPLE_SOURCE} "int main(void) { return 0; }\n")
 
 # Detect whether the current target platform is 32-bit or 64-bit, and setup
 # the correct commandline flags needed to attempt to target 32-bit and 64-bit.
-- 
GitLab


From d31278896208d856b277e34bd7e2a63899f0b57b Mon Sep 17 00:00:00 2001
From: "Xiangyang (Mark) Guo" <helloguo@fb.com>
Date: Tue, 26 Mar 2024 15:59:31 -0700
Subject: [PATCH 355/404] [InlineOrder] fix the calculation of Cost for
 CostBenefitPriority (#86630)

getCost() expects that isVariable() is true.
https://github.com/llvm/llvm-project/blob/main/llvm/include/llvm/Analysis/InlineCost.h#L146

Co-authored-by: helloguo <helloguo@meta.com>
---
 llvm/lib/Analysis/InlineOrder.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Analysis/InlineOrder.cpp b/llvm/lib/Analysis/InlineOrder.cpp
index 09fc4f9a00f4..f156daa2f126 100644
--- a/llvm/lib/Analysis/InlineOrder.cpp
+++ b/llvm/lib/Analysis/InlineOrder.cpp
@@ -114,7 +114,10 @@ public:
   CostBenefitPriority(const CallBase *CB, FunctionAnalysisManager &FAM,
                       const InlineParams &Params) {
     auto IC = getInlineCostWrapper(const_cast<CallBase &>(*CB), FAM, Params);
-    Cost = IC.getCost();
+    if (IC.isVariable())
+      Cost = IC.getCost();
+    else
+      Cost = IC.isNever() ? INT_MAX : INT_MIN;
     StaticBonusApplied = IC.getStaticBonusApplied();
     CostBenefit = IC.getCostBenefit();
   }
-- 
GitLab


From 09155ac290b0906ac8011c87ee43d7c7a2710387 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 26 Mar 2024 16:23:26 -0700
Subject: [PATCH 356/404] [LegalizeDAG] Remove unneeded temporary SDValues from
 PerformInsertVectorEltInMemory. NFC

There were 3 temporaries that just renamed the 3 well name arguments to the
function to Tmp1-3. Looks like this was done when the code was extracted from
elsewhere into a separate function 15 years ago.
---
 llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 808e3c622033..b1f6fd6f3c72 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -386,17 +386,13 @@ SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
                                                              SDValue Val,
                                                              SDValue Idx,
                                                              const SDLoc &dl) {
-  SDValue Tmp1 = Vec;
-  SDValue Tmp2 = Val;
-  SDValue Tmp3 = Idx;
-
   // If the target doesn't support this, we have to spill the input vector
   // to a temporary stack slot, update the element, then reload it.  This is
   // badness.  We could also load the value into a vector register (either
   // with a "move to register" or "extload into register" instruction, then
   // permute it into place, if the idx is a constant and if the idx is
   // supported by the target.
-  EVT VT    = Tmp1.getValueType();
+  EVT VT    = Vec.getValueType();
   EVT EltVT = VT.getVectorElementType();
   SDValue StackPtr = DAG.CreateStackTemporary(VT);
 
@@ -404,14 +400,14 @@ SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
 
   // Store the vector.
   SDValue Ch = DAG.getStore(
-      DAG.getEntryNode(), dl, Tmp1, StackPtr,
+      DAG.getEntryNode(), dl, Vec, StackPtr,
       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
 
-  SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
+  SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Idx);
 
   // Store the scalar value.
   Ch = DAG.getTruncStore(
-      Ch, dl, Tmp2, StackPtr2,
+      Ch, dl, Val, StackPtr2,
       MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT);
   // Load the updated vector.
   return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
-- 
GitLab


From b1a633bc014a845a58f0f3c4e442219051646c19 Mon Sep 17 00:00:00 2001
From: Alex MacLean <amaclean@nvidia.com>
Date: Tue, 26 Mar 2024 16:39:32 -0700
Subject: [PATCH 357/404] [NFC][NVPTX] remove truncating c-style cast (#85889)

While a stack size large enough to cause this truncation to be a problem
would certainly cause other issues and not produce a valid program
anyway, this cast is triggering our Coverity static analysis. Removing
it seems cleaner.
---
 llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
index 2783b1e0ec73..ece9821a2d0d 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
@@ -1719,7 +1719,7 @@ void NVPTXAsmPrinter::setAndEmitFunctionVirtualRegisters(
 
   // Emit the Fake Stack Object
   const MachineFrameInfo &MFI = MF.getFrameInfo();
-  int NumBytes = (int) MFI.getStackSize();
+  int64_t NumBytes = MFI.getStackSize();
   if (NumBytes) {
     O << "\t.local .align " << MFI.getMaxAlign().value() << " .b8 \t"
       << DEPOTNAME << getFunctionNumber() << "[" << NumBytes << "];\n";
-- 
GitLab


From 7db40463229bb1c9fb15b2107d878fe70d1eda65 Mon Sep 17 00:00:00 2001
From: Vadim Paretsky <vadim.paretsky@intel.com>
Date: Tue, 26 Mar 2024 16:41:31 -0700
Subject: [PATCH 358/404] [OpenMP] add loop collapse tests (#86243)

This PR adds loop collapse tests ported from MSVC.

---------

Co-authored-by: Vadim Paretsky <b-vadipa@microsoft.com>
---
 openmp/runtime/src/kmp_collapse.cpp           |  11 +-
 .../test/worksharing/for/collapse_test.inc    | 201 ++++++++++++++++++
 .../for/omp_collapse_many_GELTGT_int.c        |  65 ++++++
 .../for/omp_collapse_many_GTGEGT_int.c        |  71 +++++++
 .../for/omp_collapse_many_LTLEGE_int.c        |  66 ++++++
 .../worksharing/for/omp_collapse_many_int.c   |  73 +++++++
 .../worksharing/for/omp_collapse_one_int.c    |  32 +++
 7 files changed, 511 insertions(+), 8 deletions(-)
 create mode 100644 openmp/runtime/test/worksharing/for/collapse_test.inc
 create mode 100644 openmp/runtime/test/worksharing/for/omp_collapse_many_GELTGT_int.c
 create mode 100644 openmp/runtime/test/worksharing/for/omp_collapse_many_GTGEGT_int.c
 create mode 100644 openmp/runtime/test/worksharing/for/omp_collapse_many_LTLEGE_int.c
 create mode 100644 openmp/runtime/test/worksharing/for/omp_collapse_many_int.c
 create mode 100644 openmp/runtime/test/worksharing/for/omp_collapse_one_int.c

diff --git a/openmp/runtime/src/kmp_collapse.cpp b/openmp/runtime/src/kmp_collapse.cpp
index 569d2c150831..e63a98081db9 100644
--- a/openmp/runtime/src/kmp_collapse.cpp
+++ b/openmp/runtime/src/kmp_collapse.cpp
@@ -1517,16 +1517,11 @@ void kmp_handle_upper_triangle_matrix(
   kmp_uint64 iter_with_current = iter_before_current + iter_current;
   // calculate the outer loop lower bound (lbo) which is the max outer iv value
   // that gives the number of iterations that is equal or just below the total
-  // number of iterations executed by the previous threads, for less_than
-  // (1-based) inner loops (inner_ub0 == -1) it will be i.e.
-  // lbo*(lbo-1)/2<=iter_before_current => lbo^2-lbo-2*iter_before_current<=0
-  // for less_than_equal (0-based) inner loops (inner_ub == 0) it will be:
-  // i.e. lbo*(lbo+1)/2<=iter_before_current =>
-  // lbo^2+lbo-2*iter_before_current<=0 both cases can be handled similarily
-  // using a parameter to control the equatio sign
+  // number of iterations executed by the previous threads:
+  // lbo*(lbo+1)/2<=iter_before_current =>
+  // lbo^2+lbo-2*iter_before_current<=0
   kmp_uint64 lower_bound_outer =
       (kmp_uint64)(sqrt_newton_approx(1 + 8 * iter_before_current) + 1) / 2 - 1;
-  ;
   // calculate the inner loop lower bound which is the remaining number of
   // iterations required to hit the total number of iterations executed by the
   // previous threads giving the starting point of this thread
diff --git a/openmp/runtime/test/worksharing/for/collapse_test.inc b/openmp/runtime/test/worksharing/for/collapse_test.inc
new file mode 100644
index 000000000000..de0e7e4e57f3
--- /dev/null
+++ b/openmp/runtime/test/worksharing/for/collapse_test.inc
@@ -0,0 +1,201 @@
+#include <omp.h>
+#include <malloc.h>
+#include <stdio.h>
+#include <memory.h>
+
+#define LOOP_IV_TYPE0 LOOP_TYPES
+#define LOOP_TYPE0 LOOP_TYPES
+#define LOOP_STYPE0 LOOP_TYPES
+
+#define LOOP_IV_TYPE1 LOOP_TYPES
+#define LOOP_TYPE1 LOOP_TYPES
+#define LOOP_STYPE1 LOOP_TYPES
+
+#define LOOP_IV_TYPE2 LOOP_TYPES
+#define LOOP_TYPE2 LOOP_TYPES
+#define LOOP_STYPE2 LOOP_TYPES
+
+#define MAX_THREADS 256
+
+#if defined VERBOSE
+#define PRINTF printf
+#else
+#define PRINTF
+#endif
+
+LOOP_TYPE0 iLB, iUB;
+LOOP_TYPE1 jA0, jB0;
+LOOP_TYPE2 kA0, kB0;
+
+LOOP_STYPE0 iStep;
+LOOP_STYPE1 jA1, jB1, jStep;
+LOOP_STYPE2 kA1, kB1, kStep;
+
+// We can check <=, <, >=, > (!= has different pattern)
+// Additional definition of LOOP_LEi, LOOP_LTi, etc. is helpful to build calls
+// of the test from main
+
+#if defined LOOP_LE0
+#define COMPARE0 <=
+#elif defined LOOP_LT0
+#define COMPARE0 <
+#elif defined LOOP_GE0
+#define COMPARE0 >=
+#elif defined LOOP_GT0
+#define COMPARE0 >
+#endif
+
+#if defined LOOP_LE1
+#define COMPARE1 <=
+#elif defined LOOP_LT1
+#define COMPARE1 <
+#elif defined LOOP_GE1
+#define COMPARE1 >=
+#elif defined LOOP_GT1
+#define COMPARE1 >
+#endif
+
+#if defined LOOP_LE2
+#define COMPARE2 <=
+#elif defined LOOP_LT2
+#define COMPARE2 <
+#elif defined LOOP_GE2
+#define COMPARE2 >=
+#elif defined LOOP_GT2
+#define COMPARE2 >
+#endif
+
+typedef struct {
+  LOOP_IV_TYPE0 i;
+  LOOP_IV_TYPE1 j;
+  LOOP_IV_TYPE2 k;
+} spaceType;
+
+spaceType *AllocSpace(unsigned size) {
+
+  spaceType *p = (spaceType *)malloc(size * sizeof(spaceType));
+  memset(p, 0, size * sizeof(spaceType));
+  return p;
+}
+
+void FreeSpace(spaceType *space) { free(space); }
+
+// record an iteration
+void Set(spaceType *space, unsigned count, unsigned trueCount, LOOP_IV_TYPE0 i,
+         LOOP_IV_TYPE1 j, LOOP_IV_TYPE0 k) {
+  if (count > trueCount) {
+    // number of iterations exceeded
+    // will be reported with checks
+    return;
+  }
+  space[count - 1].i = i;
+  space[count - 1].j = j;
+  space[count - 1].k = k;
+}
+int test() {
+  int pass = 1;
+  LOOP_IV_TYPE0 i;
+  LOOP_IV_TYPE1 j;
+  LOOP_IV_TYPE2 k;
+
+  spaceType *openmpSpace;
+  spaceType *scalarSpace;
+
+  unsigned trueCount = 0;
+  unsigned openmpCount = 0;
+  unsigned scalarCount = 0;
+  unsigned uselessThreadsOpenMP = 0;
+  unsigned usefulThreadsOpenMP = 0;
+  unsigned chunkSizesOpenmp[MAX_THREADS] = {0};
+
+  unsigned num_threads = omp_get_max_threads();
+  if (num_threads > MAX_THREADS)
+    num_threads = MAX_THREADS;
+  omp_set_num_threads(num_threads);
+
+  // count iterations and allocate space
+  LOOP { ++trueCount; }
+
+  openmpSpace = AllocSpace(trueCount);
+  scalarSpace = AllocSpace(trueCount);
+
+  // fill the scalar (compare) space
+  LOOP {
+    ++scalarCount;
+    Set(scalarSpace, scalarCount, trueCount, i, j, k);
+  }
+
+  // test run body:
+  // perform and record OpenMP iterations and thread use
+#pragma omp parallel num_threads(num_threads)
+  {
+#pragma omp for collapse(3) private(i, j, k)
+    LOOP {
+      unsigned count;
+      unsigned gtid = omp_get_thread_num();
+#pragma omp atomic update
+      ++chunkSizesOpenmp[gtid];
+#pragma omp atomic capture
+      count = ++openmpCount;
+      Set(openmpSpace, count, trueCount, i, j, k);
+    }
+  }
+
+  // check for the right number of iterations processed
+  // (only need to check for less, greater is checked when recording)
+  if (openmpCount < trueCount) {
+    PRINTF("OpenMP FAILURE: Openmp processed fewer iterations: %d vs %d\n",
+           openmpCount, trueCount);
+    pass = 0;
+  } else if (openmpCount > trueCount) {
+    PRINTF("OpenMP FAILURE: Openmp processed more iterations: %d vs %d\n",
+           openmpCount, trueCount);
+    pass = 0;
+  }
+
+  // check openMP for iteration correctnes against scalar
+  for (unsigned i = 0; i < trueCount; i++) {
+    unsigned j;
+    for (j = 0; j < openmpCount; j++) {
+      if ((scalarSpace[i].i == openmpSpace[j].i) &&
+          (scalarSpace[i].j == openmpSpace[j].j) &&
+          (scalarSpace[i].k == openmpSpace[j].k)) {
+        break;
+      }
+    }
+    if (j == openmpCount) {
+      PRINTF("OpenMP FAILURE: (%d %d %d) not processed\n", scalarSpace[i].i,
+             scalarSpace[i].j, scalarSpace[i].k);
+      pass = 0;
+    }
+  }
+
+  // check for efficient thread use
+  for (unsigned i = 0; i < num_threads; ++i) {
+    if (chunkSizesOpenmp[i] == 0) {
+      ++uselessThreadsOpenMP;
+    }
+  }
+
+  // a check to see if at least more than one thread was used (weakish)
+  if ((uselessThreadsOpenMP == num_threads - 1) && (trueCount > 1)) {
+    PRINTF("OpenMP FAILURE: threads are not used\n");
+    pass = 0;
+  }
+
+#if 0
+    // a check to see if the load was spread more or less evenly so that
+    // when there was more work than threads each one got at least something 
+    // (stronger, but may currently fail for a general collapse case)
+    if ((trueCount >= num_threads) && (uselessThreadsOpenMP > 0)) {
+       PRINTF("OpenMP FAILURE: %d threads not used with %d iterations\n", 
+           uselessThreadsOpenMP, openmpCount);
+       pass = 0;
+    }
+#endif
+
+  // clean up space
+  FreeSpace(openmpSpace);
+  FreeSpace(scalarSpace);
+  return pass;
+}
diff --git a/openmp/runtime/test/worksharing/for/omp_collapse_many_GELTGT_int.c b/openmp/runtime/test/worksharing/for/omp_collapse_many_GELTGT_int.c
new file mode 100644
index 000000000000..77b2d6918d87
--- /dev/null
+++ b/openmp/runtime/test/worksharing/for/omp_collapse_many_GELTGT_int.c
@@ -0,0 +1,65 @@
+// RUN: %libomp-compile-and-run
+
+// Non-rectangular loop collapsing.
+//
+// Nested loops conform to OpenMP 5.2 standard,
+// inner loops bounds may depend on outer loops induction variables.
+
+#define LOOP_TYPES int
+#define COMPARE0 >=
+#define COMPARE1 <
+#define COMPARE2 >
+#define LOOP                                                                   \
+  for (i = iLB; i COMPARE0 iUB; i += iStep)                                    \
+    for (j = jA0; j COMPARE1 jB0; j += jStep)                                  \
+      for (k = kA0; k COMPARE2 kB0; k += kStep)
+#include "collapse_test.inc"
+
+int main() {
+  int fail;
+
+  iLB = 3;
+  iUB = -2;
+  jA0 = -3;
+  jA1 = 0;
+  jB0 = -6;
+  jB1 = 0;
+  kA0 = -2;
+  kA1 = 0;
+  kB0 = -4;
+  kB1 = 0;
+  iStep = -1;
+  jStep = -1;
+  kStep = -4;
+  PRINTF("\nOne off iLB=%d; iUB=%d; jA0=%d; jA1=%d; jB0=%d; jB1=%d; kA0=%d; "
+         "kA1=%d; kB0=%d; kB1=%d; iStep=%d; jStep=%d; kStep=%d;\n",
+         iLB, iUB, jA0, jA1, jB0, jB1, kA0, kA1, kB0, kB1, iStep, jStep, kStep);
+  fail = (test() == 0);
+
+  if (!fail) {
+    for (iStep = -3; iStep >= -6; iStep -= 2) {
+      for (jA0 = -6; jA0 <= 6; jA0 += 3) {
+        for (jB0 = -3; jB0 <= 10; jB0 += 3) {
+          for (jStep = 1; jStep <= 10; jStep += 2) {
+            for (kA0 = -2; kA0 <= 4; ++kA0) {
+              for (kB0 = -4; kB0 <= 2; ++kB0) {
+                for (kStep = -2; kStep >= -10; kStep -= 4) {
+                  {
+                    PRINTF("\nTrying iLB=%d; iUB=%d; jA0=%d; jA1=%d; jB0=%d; "
+                           "jB1=%d; kA0=%d; kA1=%d; kB0=%d; kB1=%d; iStep=%d; "
+                           "jStep=%d; kStep=%d;\n",
+                           iLB, iUB, jA0, jA1, jB0, jB1, kA0, kA1, kB0, kB1,
+                           iStep, jStep, kStep);
+                    fail = fail || (test() == 0);
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return fail;
+}
diff --git a/openmp/runtime/test/worksharing/for/omp_collapse_many_GTGEGT_int.c b/openmp/runtime/test/worksharing/for/omp_collapse_many_GTGEGT_int.c
new file mode 100644
index 000000000000..985211172e62
--- /dev/null
+++ b/openmp/runtime/test/worksharing/for/omp_collapse_many_GTGEGT_int.c
@@ -0,0 +1,71 @@
+// RUN: %libomp-compile-and-run
+
+// Non-rectangular loop collapsing.
+//
+// Nested loops conform to OpenMP 5.2 standard,
+// inner loops bounds may depend on outer loops induction variables.
+
+#define LOOP_TYPES int
+#define COMPARE0 >
+#define COMPARE1 >=
+#define COMPARE2 >
+
+#define DLOOP_GT0
+#define DLOOP_GE1
+#define DLOOP_GT2
+
+#define LOOP                                                                   \
+  for (i = iLB; i COMPARE0 iUB; i += iStep)                                    \
+    for (j = jA0; j COMPARE1 jB0; j += jStep)                                  \
+      for (k = kA0; k COMPARE2 kB0; k += kStep)
+#include "collapse_test.inc"
+
+int main() {
+  int fail;
+
+  iLB = 3;
+  iUB = -2;
+  jA0 = -3;
+  jA1 = 0;
+  jB0 = -6;
+  jB1 = 0;
+  kA0 = -2;
+  kA1 = 0;
+  kB0 = -4;
+  kB1 = 0;
+  iStep = -1;
+  jStep = -1;
+  kStep = -4;
+  PRINTF("\nOne off iLB=%d; iUB=%d; jA0=%d; jA1=%d; jB0=%d; jB1=%d; kA0=%d; "
+         "kA1=%d; kB0=%d; kB1=%d; iStep=%d; jStep=%d; kStep=%d;\n",
+         iLB, iUB, jA0, jA1, jB0, jB1, kA0, kA1, kB0, kB1, iStep, jStep, kStep);
+  fail = (test() == 0);
+
+  if (!fail) {
+
+    for (iStep = -3; iStep >= -6; iStep -= 2) {
+      for (jA0 = -3; jA0 <= 10; jA0 += 3) {
+        for (jB0 = -6; jB0 <= 6; jB0 += 3) {
+          for (jStep = -1; jStep >= -10; jStep -= 2) {
+            for (kA0 = -2; kA0 <= 4; ++kA0) {
+              for (kB0 = -4; kB0 <= 2; ++kB0) {
+                for (kStep = -2; kStep >= -10; kStep -= 4) {
+                  {
+                    PRINTF("\nTrying iLB=%d; iUB=%d; jA0=%d; jA1=%d; jB0=%d; "
+                           "jB1=%d; kA0=%d; kA1=%d; kB0=%d; kB1=%d; iStep=%d; "
+                           "jStep=%d; kStep=%d;\n",
+                           iLB, iUB, jA0, jA1, jB0, jB1, kA0, kA1, kB0, kB1,
+                           iStep, jStep, kStep);
+                    fail = fail || (test() == 0);
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return fail;
+}
diff --git a/openmp/runtime/test/worksharing/for/omp_collapse_many_LTLEGE_int.c b/openmp/runtime/test/worksharing/for/omp_collapse_many_LTLEGE_int.c
new file mode 100644
index 000000000000..47e3b42226c8
--- /dev/null
+++ b/openmp/runtime/test/worksharing/for/omp_collapse_many_LTLEGE_int.c
@@ -0,0 +1,66 @@
+// RUN: %libomp-compile-and-run
+
+// Non-rectangular loop collapsing.
+//
+// Nested loops conform to OpenMP 5.2 standard,
+// inner loops bounds may depend on outer loops induction variables.
+
+#define LOOP_TYPES int
+#define COMPARE0 <
+#define COMPARE1 <=
+#define COMPARE2 >=
+#define LOOP                                                                   \
+  for (i = iLB; i COMPARE0 iUB; i += iStep)                                    \
+    for (j = jA0; j COMPARE1 jB0; j += jStep)                                  \
+      for (k = kA0; k COMPARE2 kB0; k += kStep)
+#include "collapse_test.inc"
+
+int main() {
+  int fail;
+
+  iLB = -2;
+  iUB = 3;
+  jA0 = -3;
+  jA1 = 0;
+  jB0 = -6;
+  jB1 = 0;
+  kA0 = -2;
+  kA1 = 0;
+  kB0 = -4;
+  kB1 = 0;
+  iStep = -1;
+  jStep = -1;
+  kStep = -4;
+  PRINTF("\nOne off iLB=%d; iUB=%d; jA0=%d; jA1=%d; jB0=%d; jB1=%d; kA0=%d; "
+         "kA1=%d; kB0=%d; kB1=%d; iStep=%d; jStep=%d; kStep=%d;\n",
+         iLB, iUB, jA0, jA1, jB0, jB1, kA0, kA1, kB0, kB1, iStep, jStep, kStep);
+  fail = (test() == 0);
+
+  if (!fail) {
+
+    for (iStep = 2; iStep <= 6; iStep += 2) {
+      for (jA0 = -6; jA0 <= 6; jA0 += 3) {
+        for (jB0 = -3; jB0 <= 10; jB0 += 3) {
+          for (jStep = 1; jStep <= 10; jStep += 2) {
+            for (kA0 = -2; kA0 <= 4; ++kA0) {
+              for (kB0 = -4; kB0 <= 2; ++kB0) {
+                for (kStep = -2; kStep >= -10; kStep -= 4) {
+                  {
+                    PRINTF("\nTrying iLB=%d; iUB=%d; jA0=%d; jA1=%d; jB0=%d; "
+                           "jB1=%d; kA0=%d; kA1=%d; kB0=%d; kB1=%d; iStep=%d; "
+                           "jStep=%d; kStep=%d;\n",
+                           iLB, iUB, jA0, jA1, jB0, jB1, kA0, kA1, kB0, kB1,
+                           iStep, jStep, kStep);
+                    fail = fail || (test() == 0);
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return fail;
+}
diff --git a/openmp/runtime/test/worksharing/for/omp_collapse_many_int.c b/openmp/runtime/test/worksharing/for/omp_collapse_many_int.c
new file mode 100644
index 000000000000..4455602df8a2
--- /dev/null
+++ b/openmp/runtime/test/worksharing/for/omp_collapse_many_int.c
@@ -0,0 +1,73 @@
+// RUN: %libomp-compile-and-run
+// XFAIL: true
+
+// Non-rectangular loop collapsing.
+//
+// Nested loops conform to OpenMP 5.2 standard,
+// inner loops bounds may depend on outer loops induction variables.
+
+#define LOOP_TYPES int
+#define LOOP                                                                   \
+  for (i = iLB; i <= iUB; i += iStep)                                          \
+    for (j = i * jA1 + jA0; j <= i * jB1 + jB0; j += jStep)                    \
+      for (k = j * kA1 + kA0; k <= j * kB1 + kB0; k += kStep)
+#include "collapse_test.inc"
+
+int main() {
+  int fail = 0;
+
+  iLB = -2;
+  iUB = 3;
+  jA0 = -7;
+  jA1 = -1;
+  jB0 = 13;
+  jB1 = 3;
+  kA0 = -20;
+  kA1 = -2;
+  kB0 = 111;
+  kB1 = -1;
+  iStep = 5;
+  jStep = 9;
+  kStep = 10;
+  PRINTF("\nOne off iLB=%d; iUB=%d; jA0=%d; jA1=%d; jB0=%d; jB1=%d; kA0=%d; "
+         "kA1=%d; kB0=%d; kB1=%d; iStep=%d; jStep=%d; kStep=%d;\n",
+         iLB, iUB, jA0, jA1, jB0, jB1, kA0, kA1, kB0, kB1, iStep, jStep, kStep);
+  fail = fail || (test() == 0);
+
+  if (!fail) {
+
+    // NOTE: if a loop on some level won't execute  for all iterations of an
+    // outer loop, it still should work. Runtime doesn't require lower bounds to
+    // be <= upper bounds for all possible i, j, k.
+
+    iLB = -2;
+    iUB = 3;
+    jA0 = -7;
+    jB0 = 5;
+    kA0 = -13;
+    kB0 = 37;
+
+    for (kA1 = -2; kA1 <= 2; ++kA1) { // <=
+      for (kB1 = -2; kB1 <= 2; ++kB1) {
+        for (jA1 = -3; jA1 <= 3; ++jA1) {
+          for (jB1 = -3; jB1 <= 3; ++jB1) {
+            for (iStep = 1; iStep <= 3; ++iStep) {
+              for (jStep = 2; jStep <= 6; jStep += 2) {
+                for (kStep = 2; kStep <= 8; kStep += 3) {
+                  PRINTF("\nTrying iLB=%d; iUB=%d; jA0=%d; jA1=%d; jB0=%d; "
+                         "jB1=%d; kA0=%d; kA1=%d; kB0=%d; kB1=%d; iStep=%d; "
+                         "jStep=%d; kStep=%d;\n",
+                         iLB, iUB, jA0, jA1, jB0, jB1, kA0, kA1, kB0, kB1,
+                         iStep, jStep, kStep);
+                  fail = fail || (test() == 0);
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return fail;
+}
diff --git a/openmp/runtime/test/worksharing/for/omp_collapse_one_int.c b/openmp/runtime/test/worksharing/for/omp_collapse_one_int.c
new file mode 100644
index 000000000000..437d4bff31eb
--- /dev/null
+++ b/openmp/runtime/test/worksharing/for/omp_collapse_one_int.c
@@ -0,0 +1,32 @@
+// RUN: %libomp-compile-and-run
+
+// Non-rectangular loop collapsing.
+//
+// Nested loops conform to OpenMP 5.2 standard,
+// inner loops bounds may depend on outer loops induction variables.
+
+#define LOOP_TYPES int
+#define LOOP                                                                   \
+  for (i = iLB; i <= iUB; i += iStep)                                          \
+    for (j = i + jA0; j <= i + jB0; j += jStep)                                \
+      for (k = j + kA0; k <= j + kB0; k += kStep)
+
+#include "collapse_test.inc"
+
+int main() {
+  int fail;
+  iLB = -2;
+  iUB = 3;
+  jA0 = -7;
+  jB0 = 13;
+  kA0 = -20;
+  kB0 = 111;
+  iStep = 5;
+  jStep = 9;
+  kStep = 10;
+  PRINTF("\nOne off iLB=%d; iUB=%d; jA0=%d; jB0=%d; kA0=%d; kB0=%d; iStep=%d; "
+         "jStep=%d; kStep=%d;\n",
+         iLB, iUB, jA0, jB0, kA0, kB0, iStep, jStep, kStep);
+  fail = (test() == 0);
+  return fail;
+}
-- 
GitLab


From c43932ebdc407ed9633f7468dcb061b635e99a8d Mon Sep 17 00:00:00 2001
From: SevenIsSeven <150881358+SevenIsSeven@users.noreply.github.com>
Date: Wed, 27 Mar 2024 09:05:08 +0900
Subject: [PATCH 359/404] fix build error in MSVC build (#86622)

This commit(https://github.com/llvm/llvm-project/pull/84738) introduced
following compile warning then treated-as-error in MSVC build.

>>>"'^': unsafe mix of type 'unsigned int' and type 'bool' in operation"

---------

Co-authored-by: Seven <Seven.Li@amd.com>
---
 llvm/include/llvm/IR/BasicBlock.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/IR/BasicBlock.h b/llvm/include/llvm/IR/BasicBlock.h
index 0eea4cdccca5..0c5a07bde4ec 100644
--- a/llvm/include/llvm/IR/BasicBlock.h
+++ b/llvm/include/llvm/IR/BasicBlock.h
@@ -791,7 +791,7 @@ template <> struct DenseMapInfo<BasicBlock::iterator> {
   static unsigned getHashValue(const BasicBlock::iterator &It) {
     return DenseMapInfo<void *>::getHashValue(
                reinterpret_cast<void *>(It.getNodePtr())) ^
-           It.getHeadBit();
+           (unsigned)It.getHeadBit();
   }
 
   static bool isEqual(const BasicBlock::iterator &LHS,
-- 
GitLab


From 3324f4d4f4bd82bc9fd43062d21a450671a3531b Mon Sep 17 00:00:00 2001
From: Aart Bik <ajcbik@google.com>
Date: Tue, 26 Mar 2024 17:16:03 -0700
Subject: [PATCH 360/404] [mlir][sparse] avoid incompatible linalg
 fuse-into-consumer (#86752)

This fixes an "infinite" loop bug, where the incoming IR was repeatedly
rewritten while adding identical cast operations. The test for
compatible types should include the notion of an encoding. If it
differs, then a
naive fusion into the consumer is invalid.
---
 mlir/lib/Dialect/Tensor/IR/TensorOps.cpp      |  4 ++
 .../SparseTensor/no_fold_into_consumer.mlir   | 47 +++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 mlir/test/Dialect/SparseTensor/no_fold_into_consumer.mlir

diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index dc8843aa4e1e..38a9ad60bb79 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -276,6 +276,10 @@ bool mlir::tensor::preservesStaticInformation(Type source, Type target) {
   if (sourceType.getRank() != targetType.getRank())
     return false;
 
+  // Requires same encoding.
+  if (sourceType.getEncoding() != targetType.getEncoding())
+    return false;
+
   // If cast is towards more static sizes along any dimension, don't fold.
   for (auto t : llvm::zip(sourceType.getShape(), targetType.getShape())) {
     if (!ShapedType::isDynamic(std::get<0>(t)) &&
diff --git a/mlir/test/Dialect/SparseTensor/no_fold_into_consumer.mlir b/mlir/test/Dialect/SparseTensor/no_fold_into_consumer.mlir
new file mode 100644
index 000000000000..bbc7f397e793
--- /dev/null
+++ b/mlir/test/Dialect/SparseTensor/no_fold_into_consumer.mlir
@@ -0,0 +1,47 @@
+// RUN: mlir-opt %s --canonicalize --pre-sparsification-rewrite | FileCheck %s
+
+#map = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
+
+#sparse = #sparse_tensor.encoding<{
+  map = (d0, d1, d2) ->
+          (d0 : compressed(nonunique),
+	   d1 : singleton(nonunique, soa),
+	   d2 : singleton(soa)),
+  posWidth = 64,
+  crdWidth = 64
+}>
+
+
+module {
+  //
+  // This IR should not end up in an infinite loop trying to fold
+  // the linalg producer into the tensor cast consumer (even though
+  // static sizes can fold, the different encodings cannot). The
+  // cast was sloppy to begin with (but it has been observed by
+  // external sources) and can be easily repaired by the sparsifier.
+  //
+  // CHECK-LABEL: func @avoid_fold
+  // CHECK:       arith.constant
+  // CHECK:       tensor.empty()
+  // CHECK:       linalg.generic
+  // CHECK:       sparse_tensor.convert
+  // CHECK:       return
+  //
+  func.func @avoid_fold(%0: tensor<10x20x30xf64, #sparse>) -> tensor<10x20x30xf64, #sparse> {
+    %1 = tensor.empty() : tensor<10x20x30xf64>
+    %2 = linalg.generic { indexing_maps = [#map, #map],
+                          iterator_types = ["parallel", "parallel", "parallel"]
+                        }
+    ins (%0 : tensor<10x20x30xf64, #sparse>)
+    outs(%1 : tensor<10x20x30xf64>) {
+        ^bb0(%in: f64, %out: f64):
+          %cst = arith.constant 0.000000e+00 : f64
+          %4 = arith.cmpf ugt, %in, %cst : f64
+          %5 = arith.select %4, %in, %cst : f64
+          linalg.yield %5 : f64
+    } -> tensor<10x20x30xf64>
+    %cast = tensor.cast %2 : tensor<10x20x30xf64> to tensor<10x20x30xf64, #sparse>
+    return %cast : tensor<10x20x30xf64, #sparse>
+  }
+}
+
-- 
GitLab


From 54a9f0e441c7cc3c954d24cfde00cb933306a9e9 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Tue, 26 Mar 2024 20:17:22 -0400
Subject: [PATCH 361/404] [RISCV][GISEL] Legalize, regbankselect, and
 instruction-select G_VSCALE (#85967)

G_VSCALE should be lowered using VLENB. If the type is not sXLen it
should be lowered using a G_VSCALE on the narrow type and a G_MUL.
regbank select and instruction select are straightforward so we really
only need to add tests to show it works.
---
 .../CodeGen/GlobalISel/MachineIRBuilder.h     |  11 +
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    |  29 +-
 .../CodeGen/GlobalISel/MachineIRBuilder.cpp   |   7 +
 .../Target/RISCV/GISel/RISCVLegalizerInfo.cpp |  48 +++
 .../Target/RISCV/GISel/RISCVLegalizerInfo.h   |   1 +
 llvm/lib/Target/RISCV/RISCVInstrGISel.td      |   8 +
 .../instruction-select/rvv/vscale32.mir       | 300 ++++++++++++++++++
 .../instruction-select/rvv/vscale64.mir       | 139 ++++++++
 .../legalizer/rvv/legalize-vscale-rv32.mir    | 228 +++++++++++++
 .../legalizer/rvv/legalize-vscale-rv64.mir    | 110 +++++++
 .../regbankselect/rvv/vscale-rv32.mir         |  48 +++
 .../regbankselect/rvv/vscale-rv64.mir         |  25 ++
 12 files changed, 953 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale32.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale64.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv32.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv64.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv32.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv64.mir

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 3ba036ae713f..16a7fc446fbe 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -1165,6 +1165,17 @@ public:
   /// \return a MachineInstrBuilder for the newly created instruction.
   MachineInstrBuilder buildVScale(const DstOp &Res, const ConstantInt &MinElts);
 
+  /// Build and insert \p Res = G_VSCALE \p MinElts
+  ///
+  /// G_VSCALE puts the value of the runtime vscale multiplied by \p MinElts
+  /// into \p Res.
+  ///
+  /// \pre setBasicBlock or setMI must have been called.
+  /// \pre \p Res must be a generic virtual register with scalar type.
+  ///
+  /// \return a MachineInstrBuilder for the newly created instruction.
+  MachineInstrBuilder buildVScale(const DstOp &Res, const APInt &MinElts);
+
   /// Build and insert a G_INTRINSIC instruction.
   ///
   /// There are four different opcodes based on combinations of whether the
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index c3a23ea0ad37..4981d7b80b0b 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -1699,6 +1699,20 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
   case TargetOpcode::G_FLDEXP:
   case TargetOpcode::G_STRICT_FLDEXP:
     return narrowScalarFLDEXP(MI, TypeIdx, NarrowTy);
+  case TargetOpcode::G_VSCALE: {
+    Register Dst = MI.getOperand(0).getReg();
+    LLT Ty = MRI.getType(Dst);
+
+    // Assume VSCALE(1) fits into a legal integer
+    const APInt One(NarrowTy.getSizeInBits(), 1);
+    auto VScaleBase = MIRBuilder.buildVScale(NarrowTy, One);
+    auto ZExt = MIRBuilder.buildZExt(Ty, VScaleBase);
+    auto C = MIRBuilder.buildConstant(Ty, *MI.getOperand(1).getCImm());
+    MIRBuilder.buildMul(Dst, ZExt, C);
+
+    MI.eraseFromParent();
+    return Legalized;
+  }
   }
 }
 
@@ -2966,7 +2980,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
   case TargetOpcode::G_VECREDUCE_FMIN:
   case TargetOpcode::G_VECREDUCE_FMAX:
   case TargetOpcode::G_VECREDUCE_FMINIMUM:
-  case TargetOpcode::G_VECREDUCE_FMAXIMUM:
+  case TargetOpcode::G_VECREDUCE_FMAXIMUM: {
     if (TypeIdx != 0)
       return UnableToLegalize;
     Observer.changingInstr(MI);
@@ -2980,6 +2994,19 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     Observer.changedInstr(MI);
     return Legalized;
   }
+  case TargetOpcode::G_VSCALE: {
+    MachineOperand &SrcMO = MI.getOperand(1);
+    LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
+    const APInt &SrcVal = SrcMO.getCImm()->getValue();
+    // The CImm is always a signed value
+    const APInt Val = SrcVal.sext(WideTy.getSizeInBits());
+    Observer.changingInstr(MI);
+    SrcMO.setCImm(ConstantInt::get(Ctx, Val));
+    widenScalarDst(MI, WideTy);
+    Observer.changedInstr(MI);
+    return Legalized;
+  }
+  }
 }
 
 static void getUnmergePieces(SmallVectorImpl<Register> &Pieces,
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index f7aaa0f02efc..f2665e25d195 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -811,6 +811,13 @@ MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
   return VScale;
 }
 
+MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
+                                                  const APInt &MinElts) {
+  ConstantInt *CI =
+      ConstantInt::get(getMF().getFunction().getContext(), MinElts);
+  return buildVScale(Res, *CI);
+}
+
 static unsigned getIntrinsicOpcode(bool HasSideEffects, bool IsConvergent) {
   if (HasSideEffects && IsConvergent)
     return TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS;
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 941ad752bfd9..9a388f4cd271 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -408,6 +408,10 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
       .clampScalar(0, s32, sXLen)
       .lowerForCartesianProduct({s32, sXLen, p0}, {p0});
 
+  getActionDefinitionsBuilder(G_VSCALE)
+      .clampScalar(0, sXLen, sXLen)
+      .customFor({sXLen});
+
   getLegacyLegalizerInfo().computeTables();
 }
 
@@ -529,6 +533,48 @@ bool RISCVLegalizerInfo::shouldBeInConstantPool(APInt APImm,
   return !(!SeqLo.empty() && (SeqLo.size() + 2) <= STI.getMaxBuildIntsCost());
 }
 
+bool RISCVLegalizerInfo::legalizeVScale(MachineInstr &MI,
+                                        MachineIRBuilder &MIB) const {
+  const LLT XLenTy(STI.getXLenVT());
+  Register Dst = MI.getOperand(0).getReg();
+
+  // We define our scalable vector types for lmul=1 to use a 64 bit known
+  // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
+  // vscale as VLENB / 8.
+  static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
+  if (STI.getRealMinVLen() < RISCV::RVVBitsPerBlock)
+    // Support for VLEN==32 is incomplete.
+    return false;
+
+  // We assume VLENB is a multiple of 8. We manually choose the best shift
+  // here because SimplifyDemandedBits isn't always able to simplify it.
+  uint64_t Val = MI.getOperand(1).getCImm()->getZExtValue();
+  if (isPowerOf2_64(Val)) {
+    uint64_t Log2 = Log2_64(Val);
+    if (Log2 < 3) {
+      auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {XLenTy}, {});
+      MIB.buildLShr(Dst, VLENB, MIB.buildConstant(XLenTy, 3 - Log2));
+    } else if (Log2 > 3) {
+      auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {XLenTy}, {});
+      MIB.buildShl(Dst, VLENB, MIB.buildConstant(XLenTy, Log2 - 3));
+    } else {
+      MIB.buildInstr(RISCV::G_READ_VLENB, {Dst}, {});
+    }
+  } else if ((Val % 8) == 0) {
+    // If the multiplier is a multiple of 8, scale it down to avoid needing
+    // to shift the VLENB value.
+    auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {XLenTy}, {});
+    MIB.buildMul(Dst, VLENB, MIB.buildConstant(XLenTy, Val / 8));
+  } else {
+    auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {XLenTy}, {});
+    auto VScale = MIB.buildLShr(XLenTy, VLENB, MIB.buildConstant(XLenTy, 3));
+    MIB.buildMul(Dst, VScale, MIB.buildConstant(XLenTy, Val));
+  }
+
+  MI.eraseFromParent();
+  return true;
+}
+
 bool RISCVLegalizerInfo::legalizeCustom(
     LegalizerHelper &Helper, MachineInstr &MI,
     LostDebugLocObserver &LocObserver) const {
@@ -586,6 +632,8 @@ bool RISCVLegalizerInfo::legalizeCustom(
   }
   case TargetOpcode::G_VASTART:
     return legalizeVAStart(MI, MIRBuilder);
+  case TargetOpcode::G_VSCALE:
+    return legalizeVScale(MI, MIRBuilder);
   }
 
   llvm_unreachable("expected switch to return");
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
index 323426034827..e2a98c8d2c73 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
@@ -42,6 +42,7 @@ private:
                            GISelChangeObserver &Observer) const;
 
   bool legalizeVAStart(MachineInstr &MI, MachineIRBuilder &MIRBuilder) const;
+  bool legalizeVScale(MachineInstr &MI, MachineIRBuilder &MIB) const;
 };
 } // end namespace llvm
 #endif
diff --git a/llvm/lib/Target/RISCV/RISCVInstrGISel.td b/llvm/lib/Target/RISCV/RISCVInstrGISel.td
index ede8c9809833..54e22d625781 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrGISel.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrGISel.td
@@ -24,3 +24,11 @@ def G_FCLASS : RISCVGenericInstruction {
   let hasSideEffects = false;
 }
 def : GINodeEquiv<G_FCLASS, riscv_fclass>;
+
+// Pseudo equivalent to a RISCVISD::READ_VLENB.
+def G_READ_VLENB : RISCVGenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins);
+  let hasSideEffects = false;
+}
+def : GINodeEquiv<G_READ_VLENB, riscv_read_vlenb>;
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale32.mir
new file mode 100644
index 000000000000..27dfb3fdda9d
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale32.mir
@@ -0,0 +1,300 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+v,+m -run-pass=instruction-select \
+# RUN:   -simplify-mir -verify-machineinstrs %s -o - | FileCheck  %s
+
+---
+name:            test_1_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_1_s32
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
+    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:gprb(s32) = G_READ_VLENB
+    %2:gprb(s32) = G_CONSTANT i32 3
+    %0:gprb(s32) = G_LSHR %1, %2(s32)
+    $x10 = COPY %0(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_2_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_2_s32
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 2
+    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:gprb(s32) = G_READ_VLENB
+    %2:gprb(s32) = G_CONSTANT i32 2
+    %0:gprb(s32) = G_LSHR %1, %2(s32)
+    $x10 = COPY %0(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_3_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_3_s32
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 3
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:gpr = MUL [[SRLI]], [[ADDI]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:gprb(s32) = G_READ_VLENB
+    %2:gprb(s32) = G_CONSTANT i32 3
+    %3:gprb(s32) = G_LSHR %1, %2(s32)
+    %4:gprb(s32) = G_CONSTANT i32 3
+    %0:gprb(s32) = G_MUL %3, %4
+    $x10 = COPY %0(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_4_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_4_s32
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 1
+    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:gprb(s32) = G_READ_VLENB
+    %2:gprb(s32) = G_CONSTANT i32 1
+    %0:gprb(s32) = G_LSHR %1, %2(s32)
+    $x10 = COPY %0(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_8_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_8_s32
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s32) = G_READ_VLENB
+    $x10 = COPY %0(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_16_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_16_s32
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SLLI:%[0-9]+]]:gpr = SLLI [[PseudoReadVLENB]], 1
+    ; CHECK-NEXT: $x10 = COPY [[SLLI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:gprb(s32) = G_READ_VLENB
+    %2:gprb(s32) = G_CONSTANT i32 1
+    %0:gprb(s32) = G_SHL %1, %2(s32)
+    $x10 = COPY %0(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_40_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_40_s32
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 5
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:gpr = MUL [[PseudoReadVLENB]], [[ADDI]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:gprb(s32) = G_READ_VLENB
+    %2:gprb(s32) = G_CONSTANT i32 5
+    %0:gprb(s32) = G_MUL %1, %2
+    $x10 = COPY %0(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_1_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_1_s64
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:gpr = MUL [[SRLI]], [[ADDI]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %17:gprb(s32) = G_READ_VLENB
+    %18:gprb(s32) = G_CONSTANT i32 3
+    %2:gprb(s32) = G_LSHR %17, %18(s32)
+    %15:gprb(s32) = G_CONSTANT i32 1
+    %9:gprb(s32) = G_MUL %2, %15
+    $x10 = COPY %9(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_2_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_2_s64
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 2
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:gpr = MUL [[SRLI]], [[ADDI]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %17:gprb(s32) = G_READ_VLENB
+    %18:gprb(s32) = G_CONSTANT i32 3
+    %2:gprb(s32) = G_LSHR %17, %18(s32)
+    %15:gprb(s32) = G_CONSTANT i32 2
+    %9:gprb(s32) = G_MUL %2, %15
+    $x10 = COPY %9(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_3_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_3_s64
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 3
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:gpr = MUL [[SRLI]], [[ADDI]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %17:gprb(s32) = G_READ_VLENB
+    %18:gprb(s32) = G_CONSTANT i32 3
+    %2:gprb(s32) = G_LSHR %17, %18(s32)
+    %15:gprb(s32) = G_CONSTANT i32 3
+    %9:gprb(s32) = G_MUL %2, %15
+    $x10 = COPY %9(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_4_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_4_s64
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 4
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:gpr = MUL [[SRLI]], [[ADDI]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %17:gprb(s32) = G_READ_VLENB
+    %18:gprb(s32) = G_CONSTANT i32 3
+    %2:gprb(s32) = G_LSHR %17, %18(s32)
+    %15:gprb(s32) = G_CONSTANT i32 4
+    %9:gprb(s32) = G_MUL %2, %15
+    $x10 = COPY %9(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_8_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_8_s64
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 8
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:gpr = MUL [[SRLI]], [[ADDI]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %17:gprb(s32) = G_READ_VLENB
+    %18:gprb(s32) = G_CONSTANT i32 3
+    %2:gprb(s32) = G_LSHR %17, %18(s32)
+    %15:gprb(s32) = G_CONSTANT i32 8
+    %9:gprb(s32) = G_MUL %2, %15
+    $x10 = COPY %9(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_16_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_16_s64
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 16
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:gpr = MUL [[SRLI]], [[ADDI]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %17:gprb(s32) = G_READ_VLENB
+    %18:gprb(s32) = G_CONSTANT i32 3
+    %2:gprb(s32) = G_LSHR %17, %18(s32)
+    %15:gprb(s32) = G_CONSTANT i32 16
+    %9:gprb(s32) = G_MUL %2, %15
+    $x10 = COPY %9(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_40_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_40_s64
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 40
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:gpr = MUL [[SRLI]], [[ADDI]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %17:gprb(s32) = G_READ_VLENB
+    %18:gprb(s32) = G_CONSTANT i32 3
+    %2:gprb(s32) = G_LSHR %17, %18(s32)
+    %15:gprb(s32) = G_CONSTANT i32 40
+    %9:gprb(s32) = G_MUL %2, %15
+    $x10 = COPY %9(s32)
+    PseudoRET implicit $x10
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale64.mir
new file mode 100644
index 000000000000..4a96be2471f8
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/vscale64.mir
@@ -0,0 +1,139 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -mattr=+v,+m -run-pass=instruction-select \
+# RUN:   -simplify-mir -verify-machineinstrs %s -o - | FileCheck  %s
+
+---
+name:            test_1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_1
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
+    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = G_READ_VLENB
+    %1:gprb(s64) = G_CONSTANT i64 3
+    %2:gprb(s64) = G_LSHR %0, %1(s64)
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_2
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 2
+    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = G_READ_VLENB
+    %1:gprb(s64) = G_CONSTANT i64 2
+    %2:gprb(s64) = G_LSHR %0, %1(s64)
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_3
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_3
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 3
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 3
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:gpr = MUL [[SRLI]], [[ADDI]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = G_READ_VLENB
+    %1:gprb(s64) = G_CONSTANT i64 3
+    %2:gprb(s64) = G_LSHR %0, %1(s64)
+    %3:gprb(s64) = G_CONSTANT i64 3
+    %4:gprb(s64) = G_MUL %2, %3
+    $x10 = COPY %4(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_4
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[PseudoReadVLENB]], 1
+    ; CHECK-NEXT: $x10 = COPY [[SRLI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = G_READ_VLENB
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s64) = G_LSHR %0, %1(s64)
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_8
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_8
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: $x10 = COPY [[PseudoReadVLENB]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = G_READ_VLENB
+    $x10 = COPY %0(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_16
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[SLLI:%[0-9]+]]:gpr = SLLI [[PseudoReadVLENB]], 1
+    ; CHECK-NEXT: $x10 = COPY [[SLLI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = G_READ_VLENB
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s64) = G_SHL %0, %1(s64)
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_40
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_40
+    ; CHECK: [[PseudoReadVLENB:%[0-9]+]]:gpr = PseudoReadVLENB
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 5
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:gpr = MUL [[PseudoReadVLENB]], [[ADDI]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = G_READ_VLENB
+    %1:gprb(s64) = G_CONSTANT i64 5
+    %2:gprb(s64) = G_MUL %0, %1
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv32.mir
new file mode 100644
index 000000000000..899f7955a273
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv32.mir
@@ -0,0 +1,228 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+v,+m -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            test_1_s32
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_1_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = G_VSCALE i32 1
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_2_s32
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_2_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = G_VSCALE i32 2
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_3_s32
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_3_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[LSHR]], [[C1]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = G_VSCALE i32 3
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_4_s32
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_4_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = G_VSCALE i32 4
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_8_s32
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_8_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = G_VSCALE i32 8
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_16_s32
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_16_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[SHL]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = G_VSCALE i32 16
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_40_s32
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_40_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[READ_VLENB]], [[C]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = G_VSCALE i32 40
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+
+---
+name:            test_1_s64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_1_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[LSHR]], [[C1]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 1
+    %1:_(s32) = G_TRUNC %0
+    $x10 = COPY %1
+    PseudoRET implicit $x10
+...
+---
+name:            test_2_s64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_2_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[LSHR]], [[C1]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 2
+    %1:_(s32) = G_TRUNC %0
+    $x10 = COPY %1
+    PseudoRET implicit $x10
+...
+---
+name:            test_3_s64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_3_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[LSHR]], [[C1]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 3
+    %1:_(s32) = G_TRUNC %0
+    $x10 = COPY %1
+    PseudoRET implicit $x10
+...
+---
+name:            test_4_s64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_4_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[LSHR]], [[C1]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 4
+    %1:_(s32) = G_TRUNC %0
+    $x10 = COPY %1
+    PseudoRET implicit $x10
+...
+---
+name:            test_8_s64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_8_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[LSHR]], [[C1]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 8
+    %1:_(s32) = G_TRUNC %0
+    $x10 = COPY %1
+    PseudoRET implicit $x10
+...
+---
+name:            test_16_s64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_16_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[LSHR]], [[C1]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 16
+    %1:_(s32) = G_TRUNC %0
+    $x10 = COPY %1
+    PseudoRET implicit $x10
+...
+---
+name:            test_40_s64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_40_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 40
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[LSHR]], [[C1]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 40
+    %1:_(s32) = G_TRUNC %0
+    $x10 = COPY %1
+    PseudoRET implicit $x10
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv64.mir
new file mode 100644
index 000000000000..c0453a04a181
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-vscale-rv64.mir
@@ -0,0 +1,110 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -mattr=+v,+m -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            test_1
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_1
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 1
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_2
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_2
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 2
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_3
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_3
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[LSHR]], [[C1]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 3
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_4
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_4
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 4
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_8
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_8
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; CHECK-NEXT: $x10 = COPY [[READ_VLENB]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 8
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_16
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_16
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[READ_VLENB]], [[C]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[SHL]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 16
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+---
+name:            test_40
+body:             |
+  bb.0.entry:
+
+    ; CHECK-LABEL: name: test_40
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:_(s64) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 5
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[READ_VLENB]], [[C]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = G_VSCALE i64 40
+    $x10 = COPY %0
+    PseudoRET implicit $x10
+...
+
+
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv32.mir
new file mode 100644
index 000000000000..ae3bb0a18020
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv32.mir
@@ -0,0 +1,48 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+# RUN: llc -mtriple=riscv32 -mattr=+m,+v -run-pass=regbankselect \
+# RUN:   -disable-gisel-legality-check -simplify-mir -verify-machineinstrs %s \
+# RUN:   -o - | FileCheck %s
+
+---
+name:            test_s32
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_s32
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = G_READ_VLENB
+    %2:_(s32) = G_CONSTANT i32 3
+    %0:_(s32) = G_LSHR %1, %2(s32)
+    $x10 = COPY %0(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            test_s64
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_s64
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s32) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s32) = G_LSHR [[READ_VLENB]], [[C]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:gprb(s32) = G_MUL [[LSHR]], [[C1]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %17:_(s32) = G_READ_VLENB
+    %18:_(s32) = G_CONSTANT i32 3
+    %2:_(s32) = G_LSHR %17, %18(s32)
+    %15:_(s32) = G_CONSTANT i32 1
+    %9:_(s32) = G_MUL %2, %15
+    $x10 = COPY %9(s32)
+    PseudoRET implicit $x10
+
+...
+
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv64.mir
new file mode 100644
index 000000000000..a7446d976f25
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/rvv/vscale-rv64.mir
@@ -0,0 +1,25 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -mattr=+m,+v -run-pass=regbankselect \
+# RUN:   -disable-gisel-legality-check -simplify-mir -verify-machineinstrs %s \
+# RUN:   -o - | FileCheck %s
+
+---
+name:            test
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test
+    ; CHECK: [[READ_VLENB:%[0-9]+]]:gprb(s64) = G_READ_VLENB
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s64) = G_CONSTANT i64 3
+    ; CHECK-NEXT: [[LSHR:%[0-9]+]]:gprb(s64) = G_LSHR [[READ_VLENB]], [[C]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[LSHR]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = G_READ_VLENB
+    %2:_(s64) = G_CONSTANT i64 3
+    %0:_(s64) = G_LSHR %1, %2(s64)
+    $x10 = COPY %0(s64)
+    PseudoRET implicit $x10
+
+...
+
-- 
GitLab


From 373d8755140df0c760e9c292c5f88479cdda6f4c Mon Sep 17 00:00:00 2001
From: Pavel Kosov <kpdev42@gmail.com>
Date: Wed, 27 Mar 2024 03:25:33 +0300
Subject: [PATCH 362/404] =?UTF-8?q?[cfi][CodeGen]=20Call=20SetLLVMFunction?=
 =?UTF-8?q?Attributes{,ForDefinition}=20on=20=5F=5Fcf=E2=80=A6=20(#78253)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…i_check

This causes __cfi_check, just as __cfi_check_fail, to get the proper
target-specific attributes, in particular uwtable for unwind table
generation. Previously, nounwind attribute could be inferred for
__cfi_check, which caused it to lose its unwind table even with
-funwind-table option.

~~

Huawei RRI, OS Lab

Co-authored-by: Nikolai Kholiavin <kholiavin.nikolai@huawei-partners.com>
---
 clang/docs/ReleaseNotes.rst          |  4 ++++
 clang/lib/CodeGen/CGExpr.cpp         | 21 +++++++++++++++++++--
 clang/test/CodeGen/cfi-check-attrs.c |  5 +++++
 clang/test/CodeGen/cfi-check-fail.c  |  2 +-
 4 files changed, 29 insertions(+), 3 deletions(-)
 create mode 100644 clang/test/CodeGen/cfi-check-attrs.c

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 01abe2b39e87..0dd026a5de5c 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -626,6 +626,10 @@ Sanitizers
   manually disable potentially noisy signed integer overflow checks with
   ``-fno-sanitize=signed-integer-overflow``
 
+- ``-fsanitize=cfi -fsanitize-cfi-cross-dso`` (cross-DSO CFI instrumentation)
+  now generates the ``__cfi_check`` function with proper target-specific
+  attributes, for example allowing unwind table generation.
+
 Python Binding Changes
 ----------------------
 
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 85f5d739cef4..6491835cf8ef 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -3663,12 +3663,29 @@ void CodeGenFunction::EmitCfiSlowPathCheck(
 // symbol in LTO mode.
 void CodeGenFunction::EmitCfiCheckStub() {
   llvm::Module *M = &CGM.getModule();
-  auto &Ctx = M->getContext();
+  ASTContext &C = getContext();
+  QualType QInt64Ty = C.getIntTypeForBitwidth(64, false);
+
+  FunctionArgList FnArgs;
+  ImplicitParamDecl ArgCallsiteTypeId(C, QInt64Ty, ImplicitParamKind::Other);
+  ImplicitParamDecl ArgAddr(C, C.VoidPtrTy, ImplicitParamKind::Other);
+  ImplicitParamDecl ArgCFICheckFailData(C, C.VoidPtrTy,
+                                        ImplicitParamKind::Other);
+  FnArgs.push_back(&ArgCallsiteTypeId);
+  FnArgs.push_back(&ArgAddr);
+  FnArgs.push_back(&ArgCFICheckFailData);
+  const CGFunctionInfo &FI =
+      CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, FnArgs);
+
   llvm::Function *F = llvm::Function::Create(
-      llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy}, false),
+      llvm::FunctionType::get(VoidTy, {Int64Ty, VoidPtrTy, VoidPtrTy}, false),
       llvm::GlobalValue::WeakAnyLinkage, "__cfi_check", M);
+  CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, F, /*IsThunk=*/false);
+  CGM.SetLLVMFunctionAttributesForDefinition(nullptr, F);
   F->setAlignment(llvm::Align(4096));
   CGM.setDSOLocal(F);
+
+  llvm::LLVMContext &Ctx = M->getContext();
   llvm::BasicBlock *BB = llvm::BasicBlock::Create(Ctx, "entry", F);
   // CrossDSOCFI pass is not executed if there is no executable code.
   SmallVector<llvm::Value*> Args{F->getArg(2), F->getArg(1)};
diff --git a/clang/test/CodeGen/cfi-check-attrs.c b/clang/test/CodeGen/cfi-check-attrs.c
new file mode 100644
index 000000000000..375aa30074d8
--- /dev/null
+++ b/clang/test/CodeGen/cfi-check-attrs.c
@@ -0,0 +1,5 @@
+// RUN: %clang_cc1 -triple arm-unknown-linux -funwind-tables=1 -fsanitize-cfi-cross-dso -emit-llvm -o - %s | FileCheck %s
+
+// CHECK: define weak {{.*}}void @__cfi_check({{.*}} [[ATTR:#[0-9]*]]
+
+// CHECK: attributes [[ATTR]] = {{.*}} uwtable(sync)
diff --git a/clang/test/CodeGen/cfi-check-fail.c b/clang/test/CodeGen/cfi-check-fail.c
index 2f12cee9dec6..15f6c77abf2b 100644
--- a/clang/test/CodeGen/cfi-check-fail.c
+++ b/clang/test/CodeGen/cfi-check-fail.c
@@ -72,7 +72,7 @@ void caller(void (*f)(void)) {
 // CHECK: [[CONT5]]:
 // CHECK:   ret void
 
-// CHECK: define weak void @__cfi_check(i64 %[[TYPE:.*]], ptr %[[ADDR:.*]], ptr %[[DATA:.*]]) align 4096
+// CHECK: define weak void @__cfi_check(i64 noundef %[[TYPE:.*]], ptr noundef %[[ADDR:.*]], ptr noundef %[[DATA:.*]]){{.*}} align 4096
 // CHECK-NOT: }
 // CHECK: call void @__cfi_check_fail(ptr %[[DATA]], ptr %[[ADDR]])
 // CHECK-NEXT: ret void
-- 
GitLab


From d345599c2851c7ef25d2350244cbfe7cfef36386 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Tue, 26 Mar 2024 17:41:46 -0700
Subject: [PATCH 363/404] [GISEL][NFC] Use getElementCount instead of
 getNumElements in more places

These cases in particular are  done as a precommit to support
legalization, regbank selection, and instruction selection for extends,
splat vectors, and integer compares in #85938.
---
 llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp    | 3 ++-
 llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp | 2 +-
 llvm/lib/CodeGen/LowLevelTypeUtils.cpp           | 2 +-
 llvm/lib/CodeGen/MachineVerifier.cpp             | 3 ++-
 llvm/lib/CodeGen/RegisterBankInfo.cpp            | 7 ++++---
 5 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
index 4ee1793d33d2..c9ee35373cd4 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
@@ -154,7 +154,8 @@ static bool mutationIsSane(const LegalizeRule &Rule,
   case WidenScalar: {
     if (OldTy.isVector()) {
       // Number of elements should not change.
-      if (!NewTy.isVector() || OldTy.getNumElements() != NewTy.getNumElements())
+      if (!NewTy.isVector() ||
+          OldTy.getElementCount() != NewTy.getElementCount())
         return false;
     } else {
       // Both types must be vectors
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index f2665e25d195..07d4cb5eaa23 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -1160,7 +1160,7 @@ void MachineIRBuilder::validateSelectOp(const LLT ResTy, const LLT TstTy,
   else
     assert((TstTy.isScalar() ||
             (TstTy.isVector() &&
-             TstTy.getNumElements() == Op0Ty.getNumElements())) &&
+             TstTy.getElementCount() == Op0Ty.getElementCount())) &&
            "type mismatch");
 #endif
 }
diff --git a/llvm/lib/CodeGen/LowLevelTypeUtils.cpp b/llvm/lib/CodeGen/LowLevelTypeUtils.cpp
index 5caf20add2a1..1602cd99c383 100644
--- a/llvm/lib/CodeGen/LowLevelTypeUtils.cpp
+++ b/llvm/lib/CodeGen/LowLevelTypeUtils.cpp
@@ -51,7 +51,7 @@ MVT llvm::getMVTForLLT(LLT Ty) {
 
   return MVT::getVectorVT(
       MVT::getIntegerVT(Ty.getElementType().getSizeInBits()),
-      Ty.getNumElements());
+      Ty.getElementCount());
 }
 
 EVT llvm::getApproximateEVTForLLT(LLT Ty, const DataLayout &DL,
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index 9a1498d4070e..e4e05ce9278c 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -1506,7 +1506,8 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
     LLT SrcTy = MRI->getType(MI->getOperand(2).getReg());
 
     if ((DstTy.isVector() != SrcTy.isVector()) ||
-        (DstTy.isVector() && DstTy.getNumElements() != SrcTy.getNumElements()))
+        (DstTy.isVector() &&
+         DstTy.getElementCount() != SrcTy.getElementCount()))
       report("Generic vector icmp/fcmp must preserve number of lanes", MI);
 
     break;
diff --git a/llvm/lib/CodeGen/RegisterBankInfo.cpp b/llvm/lib/CodeGen/RegisterBankInfo.cpp
index 5548430d1b0a..72b07eb1902d 100644
--- a/llvm/lib/CodeGen/RegisterBankInfo.cpp
+++ b/llvm/lib/CodeGen/RegisterBankInfo.cpp
@@ -484,9 +484,10 @@ void RegisterBankInfo::applyDefaultMapping(const OperandsMapper &OpdMapper) {
       // the storage. However, right now we don't necessarily bump all
       // the types to storage size. For instance, we can consider
       // s16 G_AND legal whereas the storage size is going to be 32.
-      assert(OrigTy.getSizeInBits() <= NewTy.getSizeInBits() &&
-             "Types with difference size cannot be handled by the default "
-             "mapping");
+      assert(
+          TypeSize::isKnownLE(OrigTy.getSizeInBits(), NewTy.getSizeInBits()) &&
+          "Types with difference size cannot be handled by the default "
+          "mapping");
       LLVM_DEBUG(dbgs() << "\nChange type of new opd from " << NewTy << " to "
                         << OrigTy);
       MRI.setType(NewReg, OrigTy);
-- 
GitLab


From d023995ae2cc6ab968ec305ea7b6f11b6ac2e78f Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang@amd.com>
Date: Tue, 26 Mar 2024 17:51:03 -0700
Subject: [PATCH 364/404] AMDGPU: Simplify EmitAMDGPUBuiltinExpr for load
 transposes, NFC (#86707)

We should not manually get the types of the loading data.
Instead, we can get the types from the intrinsics directly.
---
 clang/lib/CodeGen/CGBuiltin.cpp | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 46a815155e7b..3cfdb261a0ea 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -18544,31 +18544,19 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
   case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16: {
 
     Intrinsic::ID IID;
-    llvm::Type *ArgTy;
     switch (BuiltinID) {
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
-      ArgTy = llvm::Type::getInt32Ty(getLLVMContext());
-      IID = Intrinsic::amdgcn_global_load_tr_b64;
-      break;
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
-      ArgTy = llvm::FixedVectorType::get(
-          llvm::Type::getInt32Ty(getLLVMContext()), 2);
       IID = Intrinsic::amdgcn_global_load_tr_b64;
       break;
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
-      ArgTy = llvm::FixedVectorType::get(
-          llvm::Type::getInt16Ty(getLLVMContext()), 4);
-      IID = Intrinsic::amdgcn_global_load_tr_b128;
-      break;
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
-      ArgTy = llvm::FixedVectorType::get(
-          llvm::Type::getInt16Ty(getLLVMContext()), 8);
       IID = Intrinsic::amdgcn_global_load_tr_b128;
       break;
     }
-
+    llvm::Type *LoadTy = ConvertType(E->getType());
     llvm::Value *Addr = EmitScalarExpr(E->getArg(0));
-    llvm::Function *F = CGM.getIntrinsic(IID, {ArgTy});
+    llvm::Function *F = CGM.getIntrinsic(IID, {LoadTy});
     return Builder.CreateCall(F, {Addr});
   }
   case AMDGPU::BI__builtin_amdgcn_get_fpenv: {
-- 
GitLab


From 4720e3831b814fdd2e8441ee0ac05b6934fdf533 Mon Sep 17 00:00:00 2001
From: Enna1 <xumingjie.enna1@bytedance.com>
Date: Wed, 27 Mar 2024 09:06:45 +0800
Subject: [PATCH 365/404] [NFC][Sanitizer] Refine the restriction on
 SizeClassAllocator64::kRegionSize (#86270)

This patch replaces the `SANITIZER_WORDSIZE / 2` with
`sizeof(CompactPtrT) * 8`, replaces hardcoded `4` with
`kCompactPtrScale` in assertion.
---
 .../lib/sanitizer_common/sanitizer_allocator_primary64.h       | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
index 34a64f26478f..6e73065d7f53 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
@@ -639,7 +639,8 @@ class SizeClassAllocator64 {
   static_assert(kRegionSize >= SizeClassMap::kMaxSize,
                 "Region size exceed largest size");
   // kRegionSize must be <= 2^36, see CompactPtrT.
-  COMPILER_CHECK((kRegionSize) <= (1ULL << (SANITIZER_WORDSIZE / 2 + 4)));
+  COMPILER_CHECK((kRegionSize) <=
+                 (1ULL << (sizeof(CompactPtrT) * 8 + kCompactPtrScale)));
   // Call mmap for user memory with at least this size.
   static const uptr kUserMapSize = 1 << 18;
   // Call mmap for metadata memory with at least this size.
-- 
GitLab


From 10b07f2324aa990664c4141ffab534603af35c7a Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Wed, 27 Mar 2024 10:49:24 +0900
Subject: [PATCH 366/404] [mlir][Interfaces] `ValueBoundsConstraintSet`: Add
 `dump` helper function (#86634)

This commit adds a helper function that dumps the constraint set and the
mapping of columns to values/dims. For debugging only.

Example output:
```
==========
Columns:
(column	dim	value)
 0	1	linalg.fill (result 0)
 1	1	tensor.extract_slice (result 0)
 2	n/a	affine.min (result 0)
 3	n/a	scf.for (bbarg 0)
 4	n/a	func.func (bbarg 2)

Constraint set:
Domain: 0, Range: 1, Symbols: 4, Locals: 0
6 constraints
(None	None	None	None	None	const)
 1	-1	0	0	0	0	= 0
 0	1	-1	0	0	0	= 0
 0	0	-1	-1	1	0	>= 0
 0	0	-1	0	0	4	>= 0
 0	0	0	1	0	0	>= 0
 0	0	0	-1	1	-1	>= 0
==========
```
---
 .../mlir/Interfaces/ValueBoundsOpInterface.h  |  4 +++
 .../lib/Interfaces/ValueBoundsOpInterface.cpp | 29 +++++++++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h b/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h
index b4ed0967e63f..bdfd689c7ac4 100644
--- a/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h
+++ b/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h
@@ -259,6 +259,10 @@ public:
   /// Return an expression that represents a constant.
   AffineExpr getExpr(int64_t constant);
 
+  /// Debugging only: Dump the constraint set and the column-to-value/dim
+  /// mapping to llvm::errs.
+  void dump() const;
+
 protected:
   /// Dimension identifier to indicate a value is index-typed. This is used for
   /// internal data structures/API only.
diff --git a/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp b/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp
index 06ec3f4e135e..99598f2e89d9 100644
--- a/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp
+++ b/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp
@@ -715,6 +715,35 @@ ValueBoundsConstraintSet::areEquivalentSlices(MLIRContext *ctx,
   return true;
 }
 
+void ValueBoundsConstraintSet::dump() const {
+  llvm::errs() << "==========\nColumns:\n";
+  llvm::errs() << "(column\tdim\tvalue)\n";
+  for (auto [index, valueDim] : llvm::enumerate(positionToValueDim)) {
+    llvm::errs() << " " << index << "\t";
+    if (valueDim) {
+      if (valueDim->second == kIndexValue) {
+        llvm::errs() << "n/a\t";
+      } else {
+        llvm::errs() << valueDim->second << "\t";
+      }
+      llvm::errs() << getOwnerOfValue(valueDim->first)->getName() << " ";
+      if (OpResult result = dyn_cast<OpResult>(valueDim->first)) {
+        llvm::errs() << "(result " << result.getResultNumber() << ")";
+      } else {
+        llvm::errs() << "(bbarg "
+                     << cast<BlockArgument>(valueDim->first).getArgNumber()
+                     << ")";
+      }
+      llvm::errs() << "\n";
+    } else {
+      llvm::errs() << "n/a\tn/a\n";
+    }
+  }
+  llvm::errs() << "\nConstraint set:\n";
+  cstr.dump();
+  llvm::errs() << "==========\n";
+}
+
 ValueBoundsConstraintSet::BoundBuilder &
 ValueBoundsConstraintSet::BoundBuilder::operator[](int64_t dim) {
   assert(!this->dim.has_value() && "dim was already set");
-- 
GitLab


From fa1b807befdc7b31b1c0e0ab625170924f7b4951 Mon Sep 17 00:00:00 2001
From: "Oleksandr \"Alex\" Zinenko" <zinenko@google.com>
Date: Wed, 27 Mar 2024 02:58:06 +0100
Subject: [PATCH 367/404] [mlir] fix crash in transform.print verification
 (#86679)

Transform op trait verification calls `getEffects`, and since trait
verification runs before op verification, this call cannot assume the op
to be valid. However, the operand getters now return a `TypedValue` that
unconditionally casts the value to the expected type, leading to an
assertion failure. Use the untyped mechanism instead.

Fixes #84701.
---
 mlir/lib/Dialect/Transform/IR/TransformOps.cpp |  6 +++++-
 mlir/test/Dialect/Transform/ops-invalid.mlir   | 11 +++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
index 8d2ed8f6d737..abd557a508a1 100644
--- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
+++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
@@ -2628,7 +2628,11 @@ transform::PrintOp::apply(transform::TransformRewriter &rewriter,
 
 void transform::PrintOp::getEffects(
     SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
-  onlyReadsHandle(getTarget(), effects);
+  // We don't really care about mutability here, but `getTarget` now
+  // unconditionally casts to a specific type before verification could run
+  // here.
+  if (!getTargetMutable().empty())
+    onlyReadsHandle(getTargetMutable()[0].get(), effects);
   onlyReadsPayload(effects);
 
   // There is no resource for stderr file descriptor, so just declare print
diff --git a/mlir/test/Dialect/Transform/ops-invalid.mlir b/mlir/test/Dialect/Transform/ops-invalid.mlir
index 73a5f36af929..cc04e65420c5 100644
--- a/mlir/test/Dialect/Transform/ops-invalid.mlir
+++ b/mlir/test/Dialect/Transform/ops-invalid.mlir
@@ -771,3 +771,14 @@ module attributes { transform.with_named_sequence } {
     transform.yield %arg0 : !transform.any_op
   }
 }
+
+// -----
+
+module attributes { transform.with_named_sequence } {
+  transform.named_sequence @match_matmul(%entry: !transform.any_op) -> () {
+    %c3 = transform.param.constant 1 : i64 -> !transform.param<i64>
+    // expected-error @below {{op operand #0 must be TransformHandleTypeInterface instance}}
+    transform.print %c3 : !transform.param<i64>
+    transform.yield
+  }
+}
-- 
GitLab


From 7545c635729a2055a429c5decd26a619a8d6e74b Mon Sep 17 00:00:00 2001
From: Shih-Po Hung <shihpo.hung@sifive.com>
Date: Wed, 27 Mar 2024 10:58:17 +0800
Subject: [PATCH 368/404] [RISCV][TTI] Scale the cost of the sext/zext with
 LMUL (#86617)

Use the destination data type to measure the LMUL size for
latency/throughput cost
---
 .../Target/RISCV/RISCVTargetTransformInfo.cpp |  20 +-
 llvm/test/Analysis/CostModel/RISCV/cast.ll    | 920 +++++++++---------
 .../CostModel/RISCV/reduce-scalable-int.ll    |  12 +-
 .../CostModel/RISCV/rvv-extractelement.ll     |  84 +-
 .../CostModel/RISCV/rvv-insertelement.ll      |  84 +-
 .../CostModel/RISCV/shuffle-broadcast.ll      |   2 +-
 6 files changed, 566 insertions(+), 556 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index df289be18932..a0652d41e9f2 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -909,23 +909,33 @@ InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
   if (!IsTypeLegal)
     return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
 
+  std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(Dst);
+
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
   assert(ISD && "Invalid opcode");
 
-  // FIXME: Need to consider vsetvli and lmul.
   int PowDiff = (int)Log2_32(Dst->getScalarSizeInBits()) -
                 (int)Log2_32(Src->getScalarSizeInBits());
   switch (ISD) {
   case ISD::SIGN_EXTEND:
-  case ISD::ZERO_EXTEND:
-    if (Src->getScalarSizeInBits() == 1) {
+  case ISD::ZERO_EXTEND: {
+    const unsigned SrcEltSize = Src->getScalarSizeInBits();
+    if (SrcEltSize == 1) {
       // We do not use vsext/vzext to extend from mask vector.
       // Instead we use the following instructions to extend from mask vector:
       // vmv.v.i v8, 0
       // vmerge.vim v8, v8, -1, v0
-      return 2;
+      return getRISCVInstructionCost({RISCV::VMV_V_I, RISCV::VMERGE_VIM},
+                                     DstLT.second, CostKind);
     }
-    return 1;
+    if (PowDiff > 3)
+      return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
+    unsigned SExtOp[] = {RISCV::VSEXT_VF2, RISCV::VSEXT_VF4, RISCV::VSEXT_VF8};
+    unsigned ZExtOp[] = {RISCV::VZEXT_VF2, RISCV::VZEXT_VF4, RISCV::VZEXT_VF8};
+    unsigned Op =
+        (ISD == ISD::SIGN_EXTEND) ? SExtOp[PowDiff - 1] : ZExtOp[PowDiff - 1];
+    return getRISCVInstructionCost(Op, DstLT.second, CostKind);
+  }
   case ISD::TRUNCATE:
     if (Dst->getScalarSizeInBits() == 1) {
       // We do not use several vncvt to truncate to mask vector. So we could
diff --git a/llvm/test/Analysis/CostModel/RISCV/cast.ll b/llvm/test/Analysis/CostModel/RISCV/cast.ll
index bd26c19c2f2c..14da9a3f79d7 100644
--- a/llvm/test/Analysis/CostModel/RISCV/cast.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/cast.ll
@@ -16,74 +16,74 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i1_v2i64 = sext <2 x i1> undef to <2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i16 = sext <4 x i8> undef to <4 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i32 = sext <4 x i8> undef to <4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i64 = sext <4 x i8> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_v4i64 = sext <4 x i8> undef to <4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i32 = sext <4 x i16> undef to <4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i64 = sext <4 x i16> undef to <4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_v4i64 = sext <4 x i32> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16_v4i64 = sext <4 x i16> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32_v4i64 = sext <4 x i32> undef to <4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i8 = sext <4 x i1> undef to <4 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i16 = sext <4 x i1> undef to <4 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i32 = sext <4 x i1> undef to <4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i64 = sext <4 x i1> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_v4i64 = sext <4 x i1> undef to <4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i16 = sext <8 x i8> undef to <8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i32 = sext <8 x i8> undef to <8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i64 = sext <8 x i8> undef to <8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i32 = sext <8 x i16> undef to <8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i64 = sext <8 x i16> undef to <8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_v8i64 = sext <8 x i32> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_v8i32 = sext <8 x i8> undef to <8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_v8i64 = sext <8 x i8> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16_v8i32 = sext <8 x i16> undef to <8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i16_v8i64 = sext <8 x i16> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32_v8i64 = sext <8 x i32> undef to <8 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i8 = sext <8 x i1> undef to <8 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i16 = sext <8 x i1> undef to <8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i32 = sext <8 x i1> undef to <8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i64 = sext <8 x i1> undef to <8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i16 = sext <16 x i8> undef to <16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i32 = sext <16 x i8> undef to <16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i64 = sext <16 x i8> undef to <16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i32 = sext <16 x i16> undef to <16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i64 = sext <16 x i16> undef to <16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_v16i64 = sext <16 x i32> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_v8i32 = sext <8 x i1> undef to <8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i1_v8i64 = sext <8 x i1> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8_v16i16 = sext <16 x i8> undef to <16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i8_v16i32 = sext <16 x i8> undef to <16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_v16i64 = sext <16 x i8> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16_v16i32 = sext <16 x i16> undef to <16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i16_v16i64 = sext <16 x i16> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32_v16i64 = sext <16 x i32> undef to <16 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i8 = sext <16 x i1> undef to <16 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i16 = sext <16 x i1> undef to <16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i32 = sext <16 x i1> undef to <16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i64 = sext <16 x i1> undef to <16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i16 = sext <32 x i8> undef to <32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i32 = sext <32 x i8> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i8_v32i64 = sext <32 x i8> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_v32i32 = sext <32 x i16> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i16_v32i64 = sext <32 x i16> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i32_v32i64 = sext <32 x i32> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i8 = sext <32 x i1> undef to <32 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i16 = sext <32 x i1> undef to <32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i32 = sext <32 x i1> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_v32i64 = sext <32 x i1> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_v64i16 = sext <64 x i8> undef to <64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i8_v64i32 = sext <64 x i8> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i8_v64i64 = sext <64 x i8> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i16_v64i32 = sext <64 x i16> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i16_v64i64 = sext <64 x i16> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v64i32_v64i64 = sext <64 x i32> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i8 = sext <64 x i1> undef to <64 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i16 = sext <64 x i1> undef to <64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v64i1_v64i32 = sext <64 x i1> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v64i1_v64i64 = sext <64 x i1> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v128i8_v128i16 = sext <128 x i8> undef to <128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v128i8_v128i32 = sext <128 x i8> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v128i8_v128i64 = sext <128 x i8> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v128i16_v128i32 = sext <128 x i16> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v128i16_v128i64 = sext <128 x i16> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v128i32_v128i64 = sext <128 x i32> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v128i1_v128i8 = sext <128 x i1> undef to <128 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v128i1_v128i16 = sext <128 x i1> undef to <128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v128i1_v128i32 = sext <128 x i1> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v128i1_v128i64 = sext <128 x i1> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v256i8_v256i16 = sext <256 x i8> undef to <256 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v256i8_v256i32 = sext <256 x i8> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %v256i8_v256i64 = sext <256 x i8> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v256i16_v256i32 = sext <256 x i16> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v256i16_v256i64 = sext <256 x i16> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v256i32_v256i64 = sext <256 x i32> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v256i1_v256i8 = sext <256 x i1> undef to <256 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v256i1_v256i16 = sext <256 x i1> undef to <256 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %v256i1_v256i32 = sext <256 x i1> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %v256i1_v256i64 = sext <256 x i1> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_v16i16 = sext <16 x i1> undef to <16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_v16i32 = sext <16 x i1> undef to <16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i1_v16i64 = sext <16 x i1> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i8_v32i16 = sext <32 x i8> undef to <32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i8_v32i32 = sext <32 x i8> undef to <32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i8_v32i64 = sext <32 x i8> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i16_v32i32 = sext <32 x i16> undef to <32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i16_v32i64 = sext <32 x i16> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i32_v32i64 = sext <32 x i32> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_v32i8 = sext <32 x i1> undef to <32 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_v32i16 = sext <32 x i1> undef to <32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32i32 = sext <32 x i1> undef to <32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32i64 = sext <32 x i1> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i16 = sext <64 x i8> undef to <64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i8_v64i32 = sext <64 x i8> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i8_v64i64 = sext <64 x i8> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i16_v64i32 = sext <64 x i16> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i16_v64i64 = sext <64 x i16> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64i64 = sext <64 x i32> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i1_v64i8 = sext <64 x i1> undef to <64 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i1_v64i16 = sext <64 x i1> undef to <64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64i32 = sext <64 x i1> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64i64 = sext <64 x i1> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v128i8_v128i16 = sext <128 x i8> undef to <128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v128i8_v128i32 = sext <128 x i8> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %v128i8_v128i64 = sext <128 x i8> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v128i16_v128i32 = sext <128 x i16> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v128i16_v128i64 = sext <128 x i16> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128i64 = sext <128 x i32> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v128i1_v128i8 = sext <128 x i1> undef to <128 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v128i1_v128i16 = sext <128 x i1> undef to <128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128i32 = sext <128 x i1> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128i64 = sext <128 x i1> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v256i8_v256i16 = sext <256 x i8> undef to <256 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v256i8_v256i32 = sext <256 x i8> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %v256i8_v256i64 = sext <256 x i8> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v256i16_v256i32 = sext <256 x i16> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %v256i16_v256i64 = sext <256 x i16> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %v256i32_v256i64 = sext <256 x i32> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v256i1_v256i8 = sext <256 x i1> undef to <256 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v256i1_v256i16 = sext <256 x i1> undef to <256 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %v256i1_v256i32 = sext <256 x i1> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %v256i1_v256i64 = sext <256 x i1> undef to <256 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i16 = sext <vscale x 1 x i8> undef to <vscale x 1 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i32 = sext <vscale x 1 x i8> undef to <vscale x 1 x i32>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i64 = sext <vscale x 1 x i8> undef to <vscale x 1 x i64>
@@ -96,73 +96,73 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i1_nxv1i64 = sext <vscale x 1 x i1> undef to <vscale x 1 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i16 = sext <vscale x 2 x i8> undef to <vscale x 2 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i32 = sext <vscale x 2 x i8> undef to <vscale x 2 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i64 = sext <vscale x 2 x i8> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i8_nxv2i64 = sext <vscale x 2 x i8> undef to <vscale x 2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i32 = sext <vscale x 2 x i16> undef to <vscale x 2 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i64 = sext <vscale x 2 x i16> undef to <vscale x 2 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32_nxv2i64 = sext <vscale x 2 x i32> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i16_nxv2i64 = sext <vscale x 2 x i16> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i32_nxv2i64 = sext <vscale x 2 x i32> undef to <vscale x 2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i8 = sext <vscale x 2 x i1> undef to <vscale x 2 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i16 = sext <vscale x 2 x i1> undef to <vscale x 2 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i32 = sext <vscale x 2 x i1> undef to <vscale x 2 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i64 = sext <vscale x 2 x i1> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_nxv2i64 = sext <vscale x 2 x i1> undef to <vscale x 2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i16 = sext <vscale x 4 x i8> undef to <vscale x 4 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i32 = sext <vscale x 4 x i8> undef to <vscale x 4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i64 = sext <vscale x 4 x i8> undef to <vscale x 4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i32 = sext <vscale x 4 x i16> undef to <vscale x 4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i64 = sext <vscale x 4 x i16> undef to <vscale x 4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32_nxv4i64 = sext <vscale x 4 x i32> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i8_nxv4i32 = sext <vscale x 4 x i8> undef to <vscale x 4 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i8_nxv4i64 = sext <vscale x 4 x i8> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i16_nxv4i32 = sext <vscale x 4 x i16> undef to <vscale x 4 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i16_nxv4i64 = sext <vscale x 4 x i16> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i32_nxv4i64 = sext <vscale x 4 x i32> undef to <vscale x 4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i8 = sext <vscale x 4 x i1> undef to <vscale x 4 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i16 = sext <vscale x 4 x i1> undef to <vscale x 4 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i32 = sext <vscale x 4 x i1> undef to <vscale x 4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i64 = sext <vscale x 4 x i1> undef to <vscale x 4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i16 = sext <vscale x 8 x i8> undef to <vscale x 8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i32 = sext <vscale x 8 x i8> undef to <vscale x 8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i64 = sext <vscale x 8 x i8> undef to <vscale x 8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i32 = sext <vscale x 8 x i16> undef to <vscale x 8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i64 = sext <vscale x 8 x i16> undef to <vscale x 8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32_nxv8i64 = sext <vscale x 8 x i32> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_nxv4i32 = sext <vscale x 4 x i1> undef to <vscale x 4 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv4i1_nxv4i64 = sext <vscale x 4 x i1> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i8_nxv8i16 = sext <vscale x 8 x i8> undef to <vscale x 8 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i8_nxv8i32 = sext <vscale x 8 x i8> undef to <vscale x 8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i8_nxv8i64 = sext <vscale x 8 x i8> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i16_nxv8i32 = sext <vscale x 8 x i16> undef to <vscale x 8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i16_nxv8i64 = sext <vscale x 8 x i16> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i32_nxv8i64 = sext <vscale x 8 x i32> undef to <vscale x 8 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i8 = sext <vscale x 8 x i1> undef to <vscale x 8 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i16 = sext <vscale x 8 x i1> undef to <vscale x 8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i32 = sext <vscale x 8 x i1> undef to <vscale x 8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i64 = sext <vscale x 8 x i1> undef to <vscale x 8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i16 = sext <vscale x 16 x i8> undef to <vscale x 16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i32 = sext <vscale x 16 x i8> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i8_nxv16i64 = sext <vscale x 16 x i8> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16_nxv16i32 = sext <vscale x 16 x i16> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i16_nxv16i64 = sext <vscale x 16 x i16> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i32_nxv16i64 = sext <vscale x 16 x i32> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i8 = sext <vscale x 16 x i1> undef to <vscale x 16 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i16 = sext <vscale x 16 x i1> undef to <vscale x 16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i32 = sext <vscale x 16 x i1> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_nxv16i64 = sext <vscale x 16 x i1> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8_nxv32i16 = sext <vscale x 32 x i8> undef to <vscale x 32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i8_nxv32i32 = sext <vscale x 32 x i8> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i8_nxv32i64 = sext <vscale x 32 x i8> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i16_nxv32i32 = sext <vscale x 32 x i16> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i16_nxv32i64 = sext <vscale x 32 x i16> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i32_nxv32i64 = sext <vscale x 32 x i32> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i8 = sext <vscale x 32 x i1> undef to <vscale x 32 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i16 = sext <vscale x 32 x i1> undef to <vscale x 32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_nxv32i32 = sext <vscale x 32 x i1> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_nxv32i64 = sext <vscale x 32 x i1> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv64i8_nxv64i16 = sext <vscale x 64 x i8> undef to <vscale x 64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv64i8_nxv64i32 = sext <vscale x 64 x i8> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_nxv8i16 = sext <vscale x 8 x i1> undef to <vscale x 8 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i1_nxv8i32 = sext <vscale x 8 x i1> undef to <vscale x 8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv8i1_nxv8i64 = sext <vscale x 8 x i1> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i8_nxv16i16 = sext <vscale x 16 x i8> undef to <vscale x 16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i8_nxv16i32 = sext <vscale x 16 x i8> undef to <vscale x 16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i8_nxv16i64 = sext <vscale x 16 x i8> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i16_nxv16i32 = sext <vscale x 16 x i16> undef to <vscale x 16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i16_nxv16i64 = sext <vscale x 16 x i16> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i32_nxv16i64 = sext <vscale x 16 x i32> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_nxv16i8 = sext <vscale x 16 x i1> undef to <vscale x 16 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_nxv16i16 = sext <vscale x 16 x i1> undef to <vscale x 16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16i32 = sext <vscale x 16 x i1> undef to <vscale x 16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16i64 = sext <vscale x 16 x i1> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i8_nxv32i16 = sext <vscale x 32 x i8> undef to <vscale x 32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i8_nxv32i32 = sext <vscale x 32 x i8> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i8_nxv32i64 = sext <vscale x 32 x i8> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i16_nxv32i32 = sext <vscale x 32 x i16> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i16_nxv32i64 = sext <vscale x 32 x i16> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32i64 = sext <vscale x 32 x i32> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i1_nxv32i8 = sext <vscale x 32 x i1> undef to <vscale x 32 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv32i1_nxv32i16 = sext <vscale x 32 x i1> undef to <vscale x 32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32i32 = sext <vscale x 32 x i1> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32i64 = sext <vscale x 32 x i1> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv64i8_nxv64i16 = sext <vscale x 64 x i8> undef to <vscale x 64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv64i8_nxv64i32 = sext <vscale x 64 x i8> undef to <vscale x 64 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i8_nxv64i64 = sext <vscale x 64 x i8> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv64i16_nxv64i32 = sext <vscale x 64 x i16> undef to <vscale x 64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %nxv64i16_nxv64i64 = sext <vscale x 64 x i16> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv64i32_nxv64i64 = sext <vscale x 64 x i32> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv64i1_nxv64i8 = sext <vscale x 64 x i1> undef to <vscale x 64 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv64i1_nxv64i16 = sext <vscale x 64 x i1> undef to <vscale x 64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv64i1_nxv64i32 = sext <vscale x 64 x i1> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv64i16_nxv64i32 = sext <vscale x 64 x i16> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %nxv64i16_nxv64i64 = sext <vscale x 64 x i16> undef to <vscale x 64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %nxv64i32_nxv64i64 = sext <vscale x 64 x i32> undef to <vscale x 64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv64i1_nxv64i8 = sext <vscale x 64 x i1> undef to <vscale x 64 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv64i1_nxv64i16 = sext <vscale x 64 x i1> undef to <vscale x 64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64i32 = sext <vscale x 64 x i1> undef to <vscale x 64 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i1_nxv64i64 = sext <vscale x 64 x i1> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv128i8_nxv128i16 = sext <vscale x 128 x i8> undef to <vscale x 128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %nxv128i8_nxv128i32 = sext <vscale x 128 x i8> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv128i8_nxv128i16 = sext <vscale x 128 x i8> undef to <vscale x 128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv128i8_nxv128i32 = sext <vscale x 128 x i8> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i8_nxv128i128 = sext <vscale x 128 x i8> undef to <vscale x 128 x i128>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv128i16_nxv128i32 = sext <vscale x 128 x i16> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv128i16_nxv128i32 = sext <vscale x 128 x i16> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i16_nxv128i128 = sext <vscale x 128 x i16> undef to <vscale x 128 x i128>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i32_nxv128i128 = sext <vscale x 128 x i32> undef to <vscale x 128 x i128>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv128i1_nxv128i8 = sext <vscale x 128 x i1> undef to <vscale x 128 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv128i1_nxv128i16 = sext <vscale x 128 x i1> undef to <vscale x 128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %nxv128i1_nxv128i32 = sext <vscale x 128 x i1> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %nxv128i1_nxv128i8 = sext <vscale x 128 x i1> undef to <vscale x 128 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %nxv128i1_nxv128i16 = sext <vscale x 128 x i1> undef to <vscale x 128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %nxv128i1_nxv128i32 = sext <vscale x 128 x i1> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i1_nxv128i128 = sext <vscale x 128 x i1> undef to <vscale x 128 x i128>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -179,74 +179,74 @@ define void @sext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i1_v2i64 = sext <2 x i1> undef to <2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i16 = sext <4 x i8> undef to <4 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i32 = sext <4 x i8> undef to <4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i64 = sext <4 x i8> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_v4i64 = sext <4 x i8> undef to <4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i32 = sext <4 x i16> undef to <4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i64 = sext <4 x i16> undef to <4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_v4i64 = sext <4 x i32> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16_v4i64 = sext <4 x i16> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32_v4i64 = sext <4 x i32> undef to <4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i8 = sext <4 x i1> undef to <4 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i16 = sext <4 x i1> undef to <4 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i32 = sext <4 x i1> undef to <4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i64 = sext <4 x i1> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_v4i64 = sext <4 x i1> undef to <4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i16 = sext <8 x i8> undef to <8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i32 = sext <8 x i8> undef to <8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i64 = sext <8 x i8> undef to <8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i32 = sext <8 x i16> undef to <8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i64 = sext <8 x i16> undef to <8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_v8i64 = sext <8 x i32> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_v8i32 = sext <8 x i8> undef to <8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_v8i64 = sext <8 x i8> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16_v8i32 = sext <8 x i16> undef to <8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i16_v8i64 = sext <8 x i16> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32_v8i64 = sext <8 x i32> undef to <8 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i8 = sext <8 x i1> undef to <8 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i16 = sext <8 x i1> undef to <8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i32 = sext <8 x i1> undef to <8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i64 = sext <8 x i1> undef to <8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i16 = sext <16 x i8> undef to <16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i32 = sext <16 x i8> undef to <16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i64 = sext <16 x i8> undef to <16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i32 = sext <16 x i16> undef to <16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i64 = sext <16 x i16> undef to <16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_v16i64 = sext <16 x i32> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_v8i32 = sext <8 x i1> undef to <8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i1_v8i64 = sext <8 x i1> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8_v16i16 = sext <16 x i8> undef to <16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i8_v16i32 = sext <16 x i8> undef to <16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_v16i64 = sext <16 x i8> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16_v16i32 = sext <16 x i16> undef to <16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i16_v16i64 = sext <16 x i16> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32_v16i64 = sext <16 x i32> undef to <16 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i8 = sext <16 x i1> undef to <16 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i16 = sext <16 x i1> undef to <16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i32 = sext <16 x i1> undef to <16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i64 = sext <16 x i1> undef to <16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i16 = sext <32 x i8> undef to <32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i32 = sext <32 x i8> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i8_v32i64 = sext <32 x i8> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_v32i32 = sext <32 x i16> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i16_v32i64 = sext <32 x i16> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i32_v32i64 = sext <32 x i32> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i8 = sext <32 x i1> undef to <32 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i16 = sext <32 x i1> undef to <32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i32 = sext <32 x i1> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_v32i64 = sext <32 x i1> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_v64i16 = sext <64 x i8> undef to <64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i8_v64i32 = sext <64 x i8> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i8_v64i64 = sext <64 x i8> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i16_v64i32 = sext <64 x i16> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i16_v64i64 = sext <64 x i16> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v64i32_v64i64 = sext <64 x i32> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i8 = sext <64 x i1> undef to <64 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i16 = sext <64 x i1> undef to <64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v64i1_v64i32 = sext <64 x i1> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v64i1_v64i64 = sext <64 x i1> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v128i8_v128i16 = sext <128 x i8> undef to <128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v128i8_v128i32 = sext <128 x i8> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v128i8_v128i64 = sext <128 x i8> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v128i16_v128i32 = sext <128 x i16> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v128i16_v128i64 = sext <128 x i16> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v128i32_v128i64 = sext <128 x i32> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v128i1_v128i8 = sext <128 x i1> undef to <128 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v128i1_v128i16 = sext <128 x i1> undef to <128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v128i1_v128i32 = sext <128 x i1> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v128i1_v128i64 = sext <128 x i1> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v256i8_v256i16 = sext <256 x i8> undef to <256 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v256i8_v256i32 = sext <256 x i8> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %v256i8_v256i64 = sext <256 x i8> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v256i16_v256i32 = sext <256 x i16> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v256i16_v256i64 = sext <256 x i16> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v256i32_v256i64 = sext <256 x i32> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v256i1_v256i8 = sext <256 x i1> undef to <256 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v256i1_v256i16 = sext <256 x i1> undef to <256 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %v256i1_v256i32 = sext <256 x i1> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %v256i1_v256i64 = sext <256 x i1> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_v16i16 = sext <16 x i1> undef to <16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_v16i32 = sext <16 x i1> undef to <16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i1_v16i64 = sext <16 x i1> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i8_v32i16 = sext <32 x i8> undef to <32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i8_v32i32 = sext <32 x i8> undef to <32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i8_v32i64 = sext <32 x i8> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i16_v32i32 = sext <32 x i16> undef to <32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i16_v32i64 = sext <32 x i16> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i32_v32i64 = sext <32 x i32> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_v32i8 = sext <32 x i1> undef to <32 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_v32i16 = sext <32 x i1> undef to <32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32i32 = sext <32 x i1> undef to <32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32i64 = sext <32 x i1> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i16 = sext <64 x i8> undef to <64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i8_v64i32 = sext <64 x i8> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i8_v64i64 = sext <64 x i8> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i16_v64i32 = sext <64 x i16> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i16_v64i64 = sext <64 x i16> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64i64 = sext <64 x i32> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i1_v64i8 = sext <64 x i1> undef to <64 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i1_v64i16 = sext <64 x i1> undef to <64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64i32 = sext <64 x i1> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64i64 = sext <64 x i1> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v128i8_v128i16 = sext <128 x i8> undef to <128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v128i8_v128i32 = sext <128 x i8> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %v128i8_v128i64 = sext <128 x i8> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v128i16_v128i32 = sext <128 x i16> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v128i16_v128i64 = sext <128 x i16> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128i64 = sext <128 x i32> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v128i1_v128i8 = sext <128 x i1> undef to <128 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v128i1_v128i16 = sext <128 x i1> undef to <128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128i32 = sext <128 x i1> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128i64 = sext <128 x i1> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v256i8_v256i16 = sext <256 x i8> undef to <256 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v256i8_v256i32 = sext <256 x i8> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %v256i8_v256i64 = sext <256 x i8> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v256i16_v256i32 = sext <256 x i16> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %v256i16_v256i64 = sext <256 x i16> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %v256i32_v256i64 = sext <256 x i32> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v256i1_v256i8 = sext <256 x i1> undef to <256 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v256i1_v256i16 = sext <256 x i1> undef to <256 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %v256i1_v256i32 = sext <256 x i1> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %v256i1_v256i64 = sext <256 x i1> undef to <256 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i16 = sext <vscale x 1 x i8> undef to <vscale x 1 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i32 = sext <vscale x 1 x i8> undef to <vscale x 1 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i64 = sext <vscale x 1 x i8> undef to <vscale x 1 x i64>
@@ -259,73 +259,73 @@ define void @sext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i1_nxv1i64 = sext <vscale x 1 x i1> undef to <vscale x 1 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i16 = sext <vscale x 2 x i8> undef to <vscale x 2 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i32 = sext <vscale x 2 x i8> undef to <vscale x 2 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i64 = sext <vscale x 2 x i8> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i8_nxv2i64 = sext <vscale x 2 x i8> undef to <vscale x 2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i32 = sext <vscale x 2 x i16> undef to <vscale x 2 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i64 = sext <vscale x 2 x i16> undef to <vscale x 2 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32_nxv2i64 = sext <vscale x 2 x i32> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i16_nxv2i64 = sext <vscale x 2 x i16> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i32_nxv2i64 = sext <vscale x 2 x i32> undef to <vscale x 2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i8 = sext <vscale x 2 x i1> undef to <vscale x 2 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i16 = sext <vscale x 2 x i1> undef to <vscale x 2 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i32 = sext <vscale x 2 x i1> undef to <vscale x 2 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i64 = sext <vscale x 2 x i1> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_nxv2i64 = sext <vscale x 2 x i1> undef to <vscale x 2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i16 = sext <vscale x 4 x i8> undef to <vscale x 4 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i32 = sext <vscale x 4 x i8> undef to <vscale x 4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i64 = sext <vscale x 4 x i8> undef to <vscale x 4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i32 = sext <vscale x 4 x i16> undef to <vscale x 4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i64 = sext <vscale x 4 x i16> undef to <vscale x 4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32_nxv4i64 = sext <vscale x 4 x i32> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i8_nxv4i32 = sext <vscale x 4 x i8> undef to <vscale x 4 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i8_nxv4i64 = sext <vscale x 4 x i8> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i16_nxv4i32 = sext <vscale x 4 x i16> undef to <vscale x 4 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i16_nxv4i64 = sext <vscale x 4 x i16> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i32_nxv4i64 = sext <vscale x 4 x i32> undef to <vscale x 4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i8 = sext <vscale x 4 x i1> undef to <vscale x 4 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i16 = sext <vscale x 4 x i1> undef to <vscale x 4 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i32 = sext <vscale x 4 x i1> undef to <vscale x 4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i64 = sext <vscale x 4 x i1> undef to <vscale x 4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i16 = sext <vscale x 8 x i8> undef to <vscale x 8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i32 = sext <vscale x 8 x i8> undef to <vscale x 8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i64 = sext <vscale x 8 x i8> undef to <vscale x 8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i32 = sext <vscale x 8 x i16> undef to <vscale x 8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i64 = sext <vscale x 8 x i16> undef to <vscale x 8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32_nxv8i64 = sext <vscale x 8 x i32> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_nxv4i32 = sext <vscale x 4 x i1> undef to <vscale x 4 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv4i1_nxv4i64 = sext <vscale x 4 x i1> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i8_nxv8i16 = sext <vscale x 8 x i8> undef to <vscale x 8 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i8_nxv8i32 = sext <vscale x 8 x i8> undef to <vscale x 8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i8_nxv8i64 = sext <vscale x 8 x i8> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i16_nxv8i32 = sext <vscale x 8 x i16> undef to <vscale x 8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i16_nxv8i64 = sext <vscale x 8 x i16> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i32_nxv8i64 = sext <vscale x 8 x i32> undef to <vscale x 8 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i8 = sext <vscale x 8 x i1> undef to <vscale x 8 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i16 = sext <vscale x 8 x i1> undef to <vscale x 8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i32 = sext <vscale x 8 x i1> undef to <vscale x 8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i64 = sext <vscale x 8 x i1> undef to <vscale x 8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i16 = sext <vscale x 16 x i8> undef to <vscale x 16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i32 = sext <vscale x 16 x i8> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i8_nxv16i64 = sext <vscale x 16 x i8> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16_nxv16i32 = sext <vscale x 16 x i16> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i16_nxv16i64 = sext <vscale x 16 x i16> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i32_nxv16i64 = sext <vscale x 16 x i32> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i8 = sext <vscale x 16 x i1> undef to <vscale x 16 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i16 = sext <vscale x 16 x i1> undef to <vscale x 16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i32 = sext <vscale x 16 x i1> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_nxv16i64 = sext <vscale x 16 x i1> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8_nxv32i16 = sext <vscale x 32 x i8> undef to <vscale x 32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i8_nxv32i32 = sext <vscale x 32 x i8> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i8_nxv32i64 = sext <vscale x 32 x i8> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i16_nxv32i32 = sext <vscale x 32 x i16> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i16_nxv32i64 = sext <vscale x 32 x i16> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i32_nxv32i64 = sext <vscale x 32 x i32> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i8 = sext <vscale x 32 x i1> undef to <vscale x 32 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i16 = sext <vscale x 32 x i1> undef to <vscale x 32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_nxv32i32 = sext <vscale x 32 x i1> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_nxv32i64 = sext <vscale x 32 x i1> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv64i8_nxv64i16 = sext <vscale x 64 x i8> undef to <vscale x 64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv64i8_nxv64i32 = sext <vscale x 64 x i8> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %nxv64i8_nxv64i64 = sext <vscale x 64 x i8> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv64i16_nxv64i32 = sext <vscale x 64 x i16> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %nxv64i16_nxv64i64 = sext <vscale x 64 x i16> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv64i32_nxv64i64 = sext <vscale x 64 x i32> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv64i1_nxv64i8 = sext <vscale x 64 x i1> undef to <vscale x 64 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv64i1_nxv64i16 = sext <vscale x 64 x i1> undef to <vscale x 64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv64i1_nxv64i32 = sext <vscale x 64 x i1> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %nxv64i1_nxv64i64 = sext <vscale x 64 x i1> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv128i8_nxv128i16 = sext <vscale x 128 x i8> undef to <vscale x 128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %nxv128i8_nxv128i32 = sext <vscale x 128 x i8> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_nxv8i16 = sext <vscale x 8 x i1> undef to <vscale x 8 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i1_nxv8i32 = sext <vscale x 8 x i1> undef to <vscale x 8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv8i1_nxv8i64 = sext <vscale x 8 x i1> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i8_nxv16i16 = sext <vscale x 16 x i8> undef to <vscale x 16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i8_nxv16i32 = sext <vscale x 16 x i8> undef to <vscale x 16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i8_nxv16i64 = sext <vscale x 16 x i8> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i16_nxv16i32 = sext <vscale x 16 x i16> undef to <vscale x 16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i16_nxv16i64 = sext <vscale x 16 x i16> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i32_nxv16i64 = sext <vscale x 16 x i32> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_nxv16i8 = sext <vscale x 16 x i1> undef to <vscale x 16 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_nxv16i16 = sext <vscale x 16 x i1> undef to <vscale x 16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16i32 = sext <vscale x 16 x i1> undef to <vscale x 16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16i64 = sext <vscale x 16 x i1> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i8_nxv32i16 = sext <vscale x 32 x i8> undef to <vscale x 32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i8_nxv32i32 = sext <vscale x 32 x i8> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i8_nxv32i64 = sext <vscale x 32 x i8> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i16_nxv32i32 = sext <vscale x 32 x i16> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i16_nxv32i64 = sext <vscale x 32 x i16> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32i64 = sext <vscale x 32 x i32> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i1_nxv32i8 = sext <vscale x 32 x i1> undef to <vscale x 32 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv32i1_nxv32i16 = sext <vscale x 32 x i1> undef to <vscale x 32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32i32 = sext <vscale x 32 x i1> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32i64 = sext <vscale x 32 x i1> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv64i8_nxv64i16 = sext <vscale x 64 x i8> undef to <vscale x 64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv64i8_nxv64i32 = sext <vscale x 64 x i8> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %nxv64i8_nxv64i64 = sext <vscale x 64 x i8> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv64i16_nxv64i32 = sext <vscale x 64 x i16> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv64i16_nxv64i64 = sext <vscale x 64 x i16> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv64i32_nxv64i64 = sext <vscale x 64 x i32> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv64i1_nxv64i8 = sext <vscale x 64 x i1> undef to <vscale x 64 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv64i1_nxv64i16 = sext <vscale x 64 x i1> undef to <vscale x 64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64i32 = sext <vscale x 64 x i1> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %nxv64i1_nxv64i64 = sext <vscale x 64 x i1> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv128i8_nxv128i16 = sext <vscale x 128 x i8> undef to <vscale x 128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv128i8_nxv128i32 = sext <vscale x 128 x i8> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i8_nxv128i128 = sext <vscale x 128 x i8> undef to <vscale x 128 x i128>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv128i16_nxv128i32 = sext <vscale x 128 x i16> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv128i16_nxv128i32 = sext <vscale x 128 x i16> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i16_nxv128i128 = sext <vscale x 128 x i16> undef to <vscale x 128 x i128>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i32_nxv128i128 = sext <vscale x 128 x i32> undef to <vscale x 128 x i128>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv128i1_nxv128i8 = sext <vscale x 128 x i1> undef to <vscale x 128 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv128i1_nxv128i16 = sext <vscale x 128 x i1> undef to <vscale x 128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %nxv128i1_nxv128i32 = sext <vscale x 128 x i1> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %nxv128i1_nxv128i8 = sext <vscale x 128 x i1> undef to <vscale x 128 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %nxv128i1_nxv128i16 = sext <vscale x 128 x i1> undef to <vscale x 128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %nxv128i1_nxv128i32 = sext <vscale x 128 x i1> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i1_nxv128i128 = sext <vscale x 128 x i1> undef to <vscale x 128 x i128>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -522,74 +522,74 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i1_v2i64 = zext <2 x i1> undef to <2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i16 = zext <4 x i8> undef to <4 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i32 = zext <4 x i8> undef to <4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i64 = zext <4 x i8> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_v4i64 = zext <4 x i8> undef to <4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i32 = zext <4 x i16> undef to <4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i64 = zext <4 x i16> undef to <4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_v4i64 = zext <4 x i32> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16_v4i64 = zext <4 x i16> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32_v4i64 = zext <4 x i32> undef to <4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i8 = zext <4 x i1> undef to <4 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i16 = zext <4 x i1> undef to <4 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i32 = zext <4 x i1> undef to <4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i64 = zext <4 x i1> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_v4i64 = zext <4 x i1> undef to <4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i16 = zext <8 x i8> undef to <8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i32 = zext <8 x i8> undef to <8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i64 = zext <8 x i8> undef to <8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i32 = zext <8 x i16> undef to <8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i64 = zext <8 x i16> undef to <8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_v8i64 = zext <8 x i32> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_v8i32 = zext <8 x i8> undef to <8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_v8i64 = zext <8 x i8> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16_v8i32 = zext <8 x i16> undef to <8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i16_v8i64 = zext <8 x i16> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32_v8i64 = zext <8 x i32> undef to <8 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i8 = zext <8 x i1> undef to <8 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i16 = zext <8 x i1> undef to <8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i32 = zext <8 x i1> undef to <8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i64 = zext <8 x i1> undef to <8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i16 = zext <16 x i8> undef to <16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i32 = zext <16 x i8> undef to <16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i64 = zext <16 x i8> undef to <16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i32 = zext <16 x i16> undef to <16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i64 = zext <16 x i16> undef to <16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_v16i64 = zext <16 x i32> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_v8i32 = zext <8 x i1> undef to <8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i1_v8i64 = zext <8 x i1> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8_v16i16 = zext <16 x i8> undef to <16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i8_v16i32 = zext <16 x i8> undef to <16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_v16i64 = zext <16 x i8> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16_v16i32 = zext <16 x i16> undef to <16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i16_v16i64 = zext <16 x i16> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32_v16i64 = zext <16 x i32> undef to <16 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i8 = zext <16 x i1> undef to <16 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i16 = zext <16 x i1> undef to <16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i32 = zext <16 x i1> undef to <16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i64 = zext <16 x i1> undef to <16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i16 = zext <32 x i8> undef to <32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i32 = zext <32 x i8> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i8_v32i64 = zext <32 x i8> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_v32i32 = zext <32 x i16> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i16_v32i64 = zext <32 x i16> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i32_v32i64 = zext <32 x i32> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i8 = zext <32 x i1> undef to <32 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i16 = zext <32 x i1> undef to <32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i32 = zext <32 x i1> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_v32i64 = zext <32 x i1> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_v64i16 = zext <64 x i8> undef to <64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i8_v64i32 = zext <64 x i8> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i8_v64i64 = zext <64 x i8> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i16_v64i32 = zext <64 x i16> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i16_v64i64 = zext <64 x i16> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v64i32_v64i64 = zext <64 x i32> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i8 = zext <64 x i1> undef to <64 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i16 = zext <64 x i1> undef to <64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v64i1_v64i32 = zext <64 x i1> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v64i1_v64i64 = zext <64 x i1> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v128i8_v128i16 = zext <128 x i8> undef to <128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v128i8_v128i32 = zext <128 x i8> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v128i8_v128i64 = zext <128 x i8> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v128i16_v128i32 = zext <128 x i16> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v128i16_v128i64 = zext <128 x i16> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v128i32_v128i64 = zext <128 x i32> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v128i1_v128i8 = zext <128 x i1> undef to <128 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v128i1_v128i16 = zext <128 x i1> undef to <128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v128i1_v128i32 = zext <128 x i1> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v128i1_v128i64 = zext <128 x i1> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v256i8_v256i16 = zext <256 x i8> undef to <256 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v256i8_v256i32 = zext <256 x i8> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %v256i8_v256i64 = zext <256 x i8> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v256i16_v256i32 = zext <256 x i16> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v256i16_v256i64 = zext <256 x i16> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v256i32_v256i64 = zext <256 x i32> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v256i1_v256i8 = zext <256 x i1> undef to <256 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v256i1_v256i16 = zext <256 x i1> undef to <256 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %v256i1_v256i32 = zext <256 x i1> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %v256i1_v256i64 = zext <256 x i1> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_v16i16 = zext <16 x i1> undef to <16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_v16i32 = zext <16 x i1> undef to <16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i1_v16i64 = zext <16 x i1> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i8_v32i16 = zext <32 x i8> undef to <32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i8_v32i32 = zext <32 x i8> undef to <32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i8_v32i64 = zext <32 x i8> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i16_v32i32 = zext <32 x i16> undef to <32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i16_v32i64 = zext <32 x i16> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i32_v32i64 = zext <32 x i32> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_v32i8 = zext <32 x i1> undef to <32 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_v32i16 = zext <32 x i1> undef to <32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32i32 = zext <32 x i1> undef to <32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32i64 = zext <32 x i1> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i16 = zext <64 x i8> undef to <64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i8_v64i32 = zext <64 x i8> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i8_v64i64 = zext <64 x i8> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i16_v64i32 = zext <64 x i16> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i16_v64i64 = zext <64 x i16> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64i64 = zext <64 x i32> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i1_v64i8 = zext <64 x i1> undef to <64 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i1_v64i16 = zext <64 x i1> undef to <64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64i32 = zext <64 x i1> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64i64 = zext <64 x i1> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v128i8_v128i16 = zext <128 x i8> undef to <128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v128i8_v128i32 = zext <128 x i8> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %v128i8_v128i64 = zext <128 x i8> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v128i16_v128i32 = zext <128 x i16> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v128i16_v128i64 = zext <128 x i16> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128i64 = zext <128 x i32> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v128i1_v128i8 = zext <128 x i1> undef to <128 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v128i1_v128i16 = zext <128 x i1> undef to <128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128i32 = zext <128 x i1> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128i64 = zext <128 x i1> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v256i8_v256i16 = zext <256 x i8> undef to <256 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v256i8_v256i32 = zext <256 x i8> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %v256i8_v256i64 = zext <256 x i8> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v256i16_v256i32 = zext <256 x i16> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %v256i16_v256i64 = zext <256 x i16> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %v256i32_v256i64 = zext <256 x i32> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v256i1_v256i8 = zext <256 x i1> undef to <256 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v256i1_v256i16 = zext <256 x i1> undef to <256 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %v256i1_v256i32 = zext <256 x i1> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %v256i1_v256i64 = zext <256 x i1> undef to <256 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i16 = zext <vscale x 1 x i8> undef to <vscale x 1 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i32 = zext <vscale x 1 x i8> undef to <vscale x 1 x i32>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i64 = zext <vscale x 1 x i8> undef to <vscale x 1 x i64>
@@ -602,73 +602,73 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i1_nxv1i64 = zext <vscale x 1 x i1> undef to <vscale x 1 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i16 = zext <vscale x 2 x i8> undef to <vscale x 2 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i32 = zext <vscale x 2 x i8> undef to <vscale x 2 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i64 = zext <vscale x 2 x i8> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i8_nxv2i64 = zext <vscale x 2 x i8> undef to <vscale x 2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i32 = zext <vscale x 2 x i16> undef to <vscale x 2 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i64 = zext <vscale x 2 x i16> undef to <vscale x 2 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32_nxv2i64 = zext <vscale x 2 x i32> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i16_nxv2i64 = zext <vscale x 2 x i16> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i32_nxv2i64 = zext <vscale x 2 x i32> undef to <vscale x 2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i8 = zext <vscale x 2 x i1> undef to <vscale x 2 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i16 = zext <vscale x 2 x i1> undef to <vscale x 2 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i32 = zext <vscale x 2 x i1> undef to <vscale x 2 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i64 = zext <vscale x 2 x i1> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_nxv2i64 = zext <vscale x 2 x i1> undef to <vscale x 2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i16 = zext <vscale x 4 x i8> undef to <vscale x 4 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i32 = zext <vscale x 4 x i8> undef to <vscale x 4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i64 = zext <vscale x 4 x i8> undef to <vscale x 4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i32 = zext <vscale x 4 x i16> undef to <vscale x 4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i64 = zext <vscale x 4 x i16> undef to <vscale x 4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32_nxv4i64 = zext <vscale x 4 x i32> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i8_nxv4i32 = zext <vscale x 4 x i8> undef to <vscale x 4 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i8_nxv4i64 = zext <vscale x 4 x i8> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i16_nxv4i32 = zext <vscale x 4 x i16> undef to <vscale x 4 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i16_nxv4i64 = zext <vscale x 4 x i16> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i32_nxv4i64 = zext <vscale x 4 x i32> undef to <vscale x 4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i8 = zext <vscale x 4 x i1> undef to <vscale x 4 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i16 = zext <vscale x 4 x i1> undef to <vscale x 4 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i32 = zext <vscale x 4 x i1> undef to <vscale x 4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i64 = zext <vscale x 4 x i1> undef to <vscale x 4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i16 = zext <vscale x 8 x i8> undef to <vscale x 8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i32 = zext <vscale x 8 x i8> undef to <vscale x 8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i64 = zext <vscale x 8 x i8> undef to <vscale x 8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i32 = zext <vscale x 8 x i16> undef to <vscale x 8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i64 = zext <vscale x 8 x i16> undef to <vscale x 8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32_nxv8i64 = zext <vscale x 8 x i32> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_nxv4i32 = zext <vscale x 4 x i1> undef to <vscale x 4 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv4i1_nxv4i64 = zext <vscale x 4 x i1> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i8_nxv8i16 = zext <vscale x 8 x i8> undef to <vscale x 8 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i8_nxv8i32 = zext <vscale x 8 x i8> undef to <vscale x 8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i8_nxv8i64 = zext <vscale x 8 x i8> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i16_nxv8i32 = zext <vscale x 8 x i16> undef to <vscale x 8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i16_nxv8i64 = zext <vscale x 8 x i16> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i32_nxv8i64 = zext <vscale x 8 x i32> undef to <vscale x 8 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i8 = zext <vscale x 8 x i1> undef to <vscale x 8 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i16 = zext <vscale x 8 x i1> undef to <vscale x 8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i32 = zext <vscale x 8 x i1> undef to <vscale x 8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i64 = zext <vscale x 8 x i1> undef to <vscale x 8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i16 = zext <vscale x 16 x i8> undef to <vscale x 16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i32 = zext <vscale x 16 x i8> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i8_nxv16i64 = zext <vscale x 16 x i8> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16_nxv16i32 = zext <vscale x 16 x i16> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i16_nxv16i64 = zext <vscale x 16 x i16> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i32_nxv16i64 = zext <vscale x 16 x i32> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i8 = zext <vscale x 16 x i1> undef to <vscale x 16 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i16 = zext <vscale x 16 x i1> undef to <vscale x 16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i32 = zext <vscale x 16 x i1> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_nxv16i64 = zext <vscale x 16 x i1> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8_nxv32i16 = zext <vscale x 32 x i8> undef to <vscale x 32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i8_nxv32i32 = zext <vscale x 32 x i8> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i8_nxv32i64 = zext <vscale x 32 x i8> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i16_nxv32i32 = zext <vscale x 32 x i16> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i16_nxv32i64 = zext <vscale x 32 x i16> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i32_nxv32i64 = zext <vscale x 32 x i32> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i8 = zext <vscale x 32 x i1> undef to <vscale x 32 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i16 = zext <vscale x 32 x i1> undef to <vscale x 32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_nxv32i32 = zext <vscale x 32 x i1> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_nxv32i64 = zext <vscale x 32 x i1> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv64i8_nxv64i16 = zext <vscale x 64 x i8> undef to <vscale x 64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv64i8_nxv64i32 = zext <vscale x 64 x i8> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_nxv8i16 = zext <vscale x 8 x i1> undef to <vscale x 8 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i1_nxv8i32 = zext <vscale x 8 x i1> undef to <vscale x 8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv8i1_nxv8i64 = zext <vscale x 8 x i1> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i8_nxv16i16 = zext <vscale x 16 x i8> undef to <vscale x 16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i8_nxv16i32 = zext <vscale x 16 x i8> undef to <vscale x 16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i8_nxv16i64 = zext <vscale x 16 x i8> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i16_nxv16i32 = zext <vscale x 16 x i16> undef to <vscale x 16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i16_nxv16i64 = zext <vscale x 16 x i16> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i32_nxv16i64 = zext <vscale x 16 x i32> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_nxv16i8 = zext <vscale x 16 x i1> undef to <vscale x 16 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_nxv16i16 = zext <vscale x 16 x i1> undef to <vscale x 16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16i32 = zext <vscale x 16 x i1> undef to <vscale x 16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16i64 = zext <vscale x 16 x i1> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i8_nxv32i16 = zext <vscale x 32 x i8> undef to <vscale x 32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i8_nxv32i32 = zext <vscale x 32 x i8> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i8_nxv32i64 = zext <vscale x 32 x i8> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i16_nxv32i32 = zext <vscale x 32 x i16> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i16_nxv32i64 = zext <vscale x 32 x i16> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32i64 = zext <vscale x 32 x i32> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i1_nxv32i8 = zext <vscale x 32 x i1> undef to <vscale x 32 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv32i1_nxv32i16 = zext <vscale x 32 x i1> undef to <vscale x 32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32i32 = zext <vscale x 32 x i1> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32i64 = zext <vscale x 32 x i1> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv64i8_nxv64i16 = zext <vscale x 64 x i8> undef to <vscale x 64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv64i8_nxv64i32 = zext <vscale x 64 x i8> undef to <vscale x 64 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i8_nxv64i64 = zext <vscale x 64 x i8> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv64i16_nxv64i32 = zext <vscale x 64 x i16> undef to <vscale x 64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %nxv64i16_nxv64i64 = zext <vscale x 64 x i16> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv64i32_nxv64i64 = zext <vscale x 64 x i32> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv64i1_nxv64i8 = zext <vscale x 64 x i1> undef to <vscale x 64 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv64i1_nxv64i16 = zext <vscale x 64 x i1> undef to <vscale x 64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv64i1_nxv64i32 = zext <vscale x 64 x i1> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv64i16_nxv64i32 = zext <vscale x 64 x i16> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %nxv64i16_nxv64i64 = zext <vscale x 64 x i16> undef to <vscale x 64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %nxv64i32_nxv64i64 = zext <vscale x 64 x i32> undef to <vscale x 64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv64i1_nxv64i8 = zext <vscale x 64 x i1> undef to <vscale x 64 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv64i1_nxv64i16 = zext <vscale x 64 x i1> undef to <vscale x 64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64i32 = zext <vscale x 64 x i1> undef to <vscale x 64 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i1_nxv64i64 = zext <vscale x 64 x i1> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv128i8_nxv128i16 = zext <vscale x 128 x i8> undef to <vscale x 128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %nxv128i8_nxv128i32 = zext <vscale x 128 x i8> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv128i8_nxv128i16 = zext <vscale x 128 x i8> undef to <vscale x 128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv128i8_nxv128i32 = zext <vscale x 128 x i8> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i8_nxv128i128 = zext <vscale x 128 x i8> undef to <vscale x 128 x i128>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv128i16_nxv128i32 = zext <vscale x 128 x i16> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv128i16_nxv128i32 = zext <vscale x 128 x i16> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i16_nxv128i128 = zext <vscale x 128 x i16> undef to <vscale x 128 x i128>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i32_nxv128i128 = zext <vscale x 128 x i32> undef to <vscale x 128 x i128>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv128i1_nxv128i8 = zext <vscale x 128 x i1> undef to <vscale x 128 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv128i1_nxv128i16 = zext <vscale x 128 x i1> undef to <vscale x 128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %nxv128i1_nxv128i32 = zext <vscale x 128 x i1> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %nxv128i1_nxv128i8 = zext <vscale x 128 x i1> undef to <vscale x 128 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %nxv128i1_nxv128i16 = zext <vscale x 128 x i1> undef to <vscale x 128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %nxv128i1_nxv128i32 = zext <vscale x 128 x i1> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i1_nxv128i128 = zext <vscale x 128 x i1> undef to <vscale x 128 x i128>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -685,74 +685,74 @@ define void @zext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i1_v2i64 = zext <2 x i1> undef to <2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i16 = zext <4 x i8> undef to <4 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i32 = zext <4 x i8> undef to <4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i64 = zext <4 x i8> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_v4i64 = zext <4 x i8> undef to <4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i32 = zext <4 x i16> undef to <4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i64 = zext <4 x i16> undef to <4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_v4i64 = zext <4 x i32> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16_v4i64 = zext <4 x i16> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32_v4i64 = zext <4 x i32> undef to <4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i8 = zext <4 x i1> undef to <4 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i16 = zext <4 x i1> undef to <4 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i32 = zext <4 x i1> undef to <4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i64 = zext <4 x i1> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_v4i64 = zext <4 x i1> undef to <4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i16 = zext <8 x i8> undef to <8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i32 = zext <8 x i8> undef to <8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i64 = zext <8 x i8> undef to <8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i32 = zext <8 x i16> undef to <8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i64 = zext <8 x i16> undef to <8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_v8i64 = zext <8 x i32> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_v8i32 = zext <8 x i8> undef to <8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_v8i64 = zext <8 x i8> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16_v8i32 = zext <8 x i16> undef to <8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i16_v8i64 = zext <8 x i16> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32_v8i64 = zext <8 x i32> undef to <8 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i8 = zext <8 x i1> undef to <8 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i16 = zext <8 x i1> undef to <8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i32 = zext <8 x i1> undef to <8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i64 = zext <8 x i1> undef to <8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i16 = zext <16 x i8> undef to <16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i32 = zext <16 x i8> undef to <16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i64 = zext <16 x i8> undef to <16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i32 = zext <16 x i16> undef to <16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i64 = zext <16 x i16> undef to <16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_v16i64 = zext <16 x i32> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_v8i32 = zext <8 x i1> undef to <8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i1_v8i64 = zext <8 x i1> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8_v16i16 = zext <16 x i8> undef to <16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i8_v16i32 = zext <16 x i8> undef to <16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_v16i64 = zext <16 x i8> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16_v16i32 = zext <16 x i16> undef to <16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i16_v16i64 = zext <16 x i16> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32_v16i64 = zext <16 x i32> undef to <16 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i8 = zext <16 x i1> undef to <16 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i16 = zext <16 x i1> undef to <16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i32 = zext <16 x i1> undef to <16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i64 = zext <16 x i1> undef to <16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i16 = zext <32 x i8> undef to <32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i32 = zext <32 x i8> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i8_v32i64 = zext <32 x i8> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_v32i32 = zext <32 x i16> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i16_v32i64 = zext <32 x i16> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i32_v32i64 = zext <32 x i32> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i8 = zext <32 x i1> undef to <32 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i16 = zext <32 x i1> undef to <32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i32 = zext <32 x i1> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_v32i64 = zext <32 x i1> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_v64i16 = zext <64 x i8> undef to <64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i8_v64i32 = zext <64 x i8> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i8_v64i64 = zext <64 x i8> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i16_v64i32 = zext <64 x i16> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i16_v64i64 = zext <64 x i16> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v64i32_v64i64 = zext <64 x i32> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i8 = zext <64 x i1> undef to <64 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i16 = zext <64 x i1> undef to <64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v64i1_v64i32 = zext <64 x i1> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v64i1_v64i64 = zext <64 x i1> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v128i8_v128i16 = zext <128 x i8> undef to <128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v128i8_v128i32 = zext <128 x i8> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v128i8_v128i64 = zext <128 x i8> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v128i16_v128i32 = zext <128 x i16> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v128i16_v128i64 = zext <128 x i16> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v128i32_v128i64 = zext <128 x i32> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v128i1_v128i8 = zext <128 x i1> undef to <128 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v128i1_v128i16 = zext <128 x i1> undef to <128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v128i1_v128i32 = zext <128 x i1> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v128i1_v128i64 = zext <128 x i1> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v256i8_v256i16 = zext <256 x i8> undef to <256 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v256i8_v256i32 = zext <256 x i8> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %v256i8_v256i64 = zext <256 x i8> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v256i16_v256i32 = zext <256 x i16> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v256i16_v256i64 = zext <256 x i16> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v256i32_v256i64 = zext <256 x i32> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v256i1_v256i8 = zext <256 x i1> undef to <256 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v256i1_v256i16 = zext <256 x i1> undef to <256 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %v256i1_v256i32 = zext <256 x i1> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %v256i1_v256i64 = zext <256 x i1> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_v16i16 = zext <16 x i1> undef to <16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_v16i32 = zext <16 x i1> undef to <16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i1_v16i64 = zext <16 x i1> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i8_v32i16 = zext <32 x i8> undef to <32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i8_v32i32 = zext <32 x i8> undef to <32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i8_v32i64 = zext <32 x i8> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i16_v32i32 = zext <32 x i16> undef to <32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i16_v32i64 = zext <32 x i16> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i32_v32i64 = zext <32 x i32> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_v32i8 = zext <32 x i1> undef to <32 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_v32i16 = zext <32 x i1> undef to <32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32i32 = zext <32 x i1> undef to <32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32i64 = zext <32 x i1> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i16 = zext <64 x i8> undef to <64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i8_v64i32 = zext <64 x i8> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i8_v64i64 = zext <64 x i8> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i16_v64i32 = zext <64 x i16> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i16_v64i64 = zext <64 x i16> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64i64 = zext <64 x i32> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i1_v64i8 = zext <64 x i1> undef to <64 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i1_v64i16 = zext <64 x i1> undef to <64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64i32 = zext <64 x i1> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64i64 = zext <64 x i1> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v128i8_v128i16 = zext <128 x i8> undef to <128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v128i8_v128i32 = zext <128 x i8> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %v128i8_v128i64 = zext <128 x i8> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v128i16_v128i32 = zext <128 x i16> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v128i16_v128i64 = zext <128 x i16> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128i64 = zext <128 x i32> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v128i1_v128i8 = zext <128 x i1> undef to <128 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v128i1_v128i16 = zext <128 x i1> undef to <128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128i32 = zext <128 x i1> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128i64 = zext <128 x i1> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v256i8_v256i16 = zext <256 x i8> undef to <256 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v256i8_v256i32 = zext <256 x i8> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %v256i8_v256i64 = zext <256 x i8> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v256i16_v256i32 = zext <256 x i16> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %v256i16_v256i64 = zext <256 x i16> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %v256i32_v256i64 = zext <256 x i32> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v256i1_v256i8 = zext <256 x i1> undef to <256 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v256i1_v256i16 = zext <256 x i1> undef to <256 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %v256i1_v256i32 = zext <256 x i1> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %v256i1_v256i64 = zext <256 x i1> undef to <256 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i16 = zext <vscale x 1 x i8> undef to <vscale x 1 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i32 = zext <vscale x 1 x i8> undef to <vscale x 1 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i64 = zext <vscale x 1 x i8> undef to <vscale x 1 x i64>
@@ -765,73 +765,73 @@ define void @zext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i1_nxv1i64 = zext <vscale x 1 x i1> undef to <vscale x 1 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i16 = zext <vscale x 2 x i8> undef to <vscale x 2 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i32 = zext <vscale x 2 x i8> undef to <vscale x 2 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i64 = zext <vscale x 2 x i8> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i8_nxv2i64 = zext <vscale x 2 x i8> undef to <vscale x 2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i32 = zext <vscale x 2 x i16> undef to <vscale x 2 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i64 = zext <vscale x 2 x i16> undef to <vscale x 2 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32_nxv2i64 = zext <vscale x 2 x i32> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i16_nxv2i64 = zext <vscale x 2 x i16> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i32_nxv2i64 = zext <vscale x 2 x i32> undef to <vscale x 2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i8 = zext <vscale x 2 x i1> undef to <vscale x 2 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i16 = zext <vscale x 2 x i1> undef to <vscale x 2 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i32 = zext <vscale x 2 x i1> undef to <vscale x 2 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i64 = zext <vscale x 2 x i1> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_nxv2i64 = zext <vscale x 2 x i1> undef to <vscale x 2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i16 = zext <vscale x 4 x i8> undef to <vscale x 4 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i32 = zext <vscale x 4 x i8> undef to <vscale x 4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i64 = zext <vscale x 4 x i8> undef to <vscale x 4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i32 = zext <vscale x 4 x i16> undef to <vscale x 4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i64 = zext <vscale x 4 x i16> undef to <vscale x 4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32_nxv4i64 = zext <vscale x 4 x i32> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i8_nxv4i32 = zext <vscale x 4 x i8> undef to <vscale x 4 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i8_nxv4i64 = zext <vscale x 4 x i8> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i16_nxv4i32 = zext <vscale x 4 x i16> undef to <vscale x 4 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i16_nxv4i64 = zext <vscale x 4 x i16> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i32_nxv4i64 = zext <vscale x 4 x i32> undef to <vscale x 4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i8 = zext <vscale x 4 x i1> undef to <vscale x 4 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i16 = zext <vscale x 4 x i1> undef to <vscale x 4 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i32 = zext <vscale x 4 x i1> undef to <vscale x 4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i64 = zext <vscale x 4 x i1> undef to <vscale x 4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i16 = zext <vscale x 8 x i8> undef to <vscale x 8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i32 = zext <vscale x 8 x i8> undef to <vscale x 8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i64 = zext <vscale x 8 x i8> undef to <vscale x 8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i32 = zext <vscale x 8 x i16> undef to <vscale x 8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i64 = zext <vscale x 8 x i16> undef to <vscale x 8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32_nxv8i64 = zext <vscale x 8 x i32> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_nxv4i32 = zext <vscale x 4 x i1> undef to <vscale x 4 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv4i1_nxv4i64 = zext <vscale x 4 x i1> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i8_nxv8i16 = zext <vscale x 8 x i8> undef to <vscale x 8 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i8_nxv8i32 = zext <vscale x 8 x i8> undef to <vscale x 8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i8_nxv8i64 = zext <vscale x 8 x i8> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i16_nxv8i32 = zext <vscale x 8 x i16> undef to <vscale x 8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i16_nxv8i64 = zext <vscale x 8 x i16> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i32_nxv8i64 = zext <vscale x 8 x i32> undef to <vscale x 8 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i8 = zext <vscale x 8 x i1> undef to <vscale x 8 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i16 = zext <vscale x 8 x i1> undef to <vscale x 8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i32 = zext <vscale x 8 x i1> undef to <vscale x 8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i64 = zext <vscale x 8 x i1> undef to <vscale x 8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i16 = zext <vscale x 16 x i8> undef to <vscale x 16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i32 = zext <vscale x 16 x i8> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i8_nxv16i64 = zext <vscale x 16 x i8> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16_nxv16i32 = zext <vscale x 16 x i16> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i16_nxv16i64 = zext <vscale x 16 x i16> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i32_nxv16i64 = zext <vscale x 16 x i32> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i8 = zext <vscale x 16 x i1> undef to <vscale x 16 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i16 = zext <vscale x 16 x i1> undef to <vscale x 16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i32 = zext <vscale x 16 x i1> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_nxv16i64 = zext <vscale x 16 x i1> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8_nxv32i16 = zext <vscale x 32 x i8> undef to <vscale x 32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i8_nxv32i32 = zext <vscale x 32 x i8> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i8_nxv32i64 = zext <vscale x 32 x i8> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i16_nxv32i32 = zext <vscale x 32 x i16> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i16_nxv32i64 = zext <vscale x 32 x i16> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i32_nxv32i64 = zext <vscale x 32 x i32> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i8 = zext <vscale x 32 x i1> undef to <vscale x 32 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i16 = zext <vscale x 32 x i1> undef to <vscale x 32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_nxv32i32 = zext <vscale x 32 x i1> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_nxv32i64 = zext <vscale x 32 x i1> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv64i8_nxv64i16 = zext <vscale x 64 x i8> undef to <vscale x 64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv64i8_nxv64i32 = zext <vscale x 64 x i8> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %nxv64i8_nxv64i64 = zext <vscale x 64 x i8> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv64i16_nxv64i32 = zext <vscale x 64 x i16> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %nxv64i16_nxv64i64 = zext <vscale x 64 x i16> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv64i32_nxv64i64 = zext <vscale x 64 x i32> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv64i1_nxv64i8 = zext <vscale x 64 x i1> undef to <vscale x 64 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv64i1_nxv64i16 = zext <vscale x 64 x i1> undef to <vscale x 64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv64i1_nxv64i32 = zext <vscale x 64 x i1> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %nxv64i1_nxv64i64 = zext <vscale x 64 x i1> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv128i8_nxv128i16 = zext <vscale x 128 x i8> undef to <vscale x 128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %nxv128i8_nxv128i32 = zext <vscale x 128 x i8> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_nxv8i16 = zext <vscale x 8 x i1> undef to <vscale x 8 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i1_nxv8i32 = zext <vscale x 8 x i1> undef to <vscale x 8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv8i1_nxv8i64 = zext <vscale x 8 x i1> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i8_nxv16i16 = zext <vscale x 16 x i8> undef to <vscale x 16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i8_nxv16i32 = zext <vscale x 16 x i8> undef to <vscale x 16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i8_nxv16i64 = zext <vscale x 16 x i8> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i16_nxv16i32 = zext <vscale x 16 x i16> undef to <vscale x 16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i16_nxv16i64 = zext <vscale x 16 x i16> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i32_nxv16i64 = zext <vscale x 16 x i32> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_nxv16i8 = zext <vscale x 16 x i1> undef to <vscale x 16 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_nxv16i16 = zext <vscale x 16 x i1> undef to <vscale x 16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16i32 = zext <vscale x 16 x i1> undef to <vscale x 16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16i64 = zext <vscale x 16 x i1> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i8_nxv32i16 = zext <vscale x 32 x i8> undef to <vscale x 32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i8_nxv32i32 = zext <vscale x 32 x i8> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i8_nxv32i64 = zext <vscale x 32 x i8> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i16_nxv32i32 = zext <vscale x 32 x i16> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i16_nxv32i64 = zext <vscale x 32 x i16> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32i64 = zext <vscale x 32 x i32> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i1_nxv32i8 = zext <vscale x 32 x i1> undef to <vscale x 32 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv32i1_nxv32i16 = zext <vscale x 32 x i1> undef to <vscale x 32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32i32 = zext <vscale x 32 x i1> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32i64 = zext <vscale x 32 x i1> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv64i8_nxv64i16 = zext <vscale x 64 x i8> undef to <vscale x 64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv64i8_nxv64i32 = zext <vscale x 64 x i8> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %nxv64i8_nxv64i64 = zext <vscale x 64 x i8> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv64i16_nxv64i32 = zext <vscale x 64 x i16> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv64i16_nxv64i64 = zext <vscale x 64 x i16> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv64i32_nxv64i64 = zext <vscale x 64 x i32> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv64i1_nxv64i8 = zext <vscale x 64 x i1> undef to <vscale x 64 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv64i1_nxv64i16 = zext <vscale x 64 x i1> undef to <vscale x 64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64i32 = zext <vscale x 64 x i1> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %nxv64i1_nxv64i64 = zext <vscale x 64 x i1> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv128i8_nxv128i16 = zext <vscale x 128 x i8> undef to <vscale x 128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv128i8_nxv128i32 = zext <vscale x 128 x i8> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i8_nxv128i128 = zext <vscale x 128 x i8> undef to <vscale x 128 x i128>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv128i16_nxv128i32 = zext <vscale x 128 x i16> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv128i16_nxv128i32 = zext <vscale x 128 x i16> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i16_nxv128i128 = zext <vscale x 128 x i16> undef to <vscale x 128 x i128>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i32_nxv128i128 = zext <vscale x 128 x i32> undef to <vscale x 128 x i128>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv128i1_nxv128i8 = zext <vscale x 128 x i1> undef to <vscale x 128 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv128i1_nxv128i16 = zext <vscale x 128 x i1> undef to <vscale x 128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %nxv128i1_nxv128i32 = zext <vscale x 128 x i1> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %nxv128i1_nxv128i8 = zext <vscale x 128 x i1> undef to <vscale x 128 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %nxv128i1_nxv128i16 = zext <vscale x 128 x i1> undef to <vscale x 128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %nxv128i1_nxv128i32 = zext <vscale x 128 x i1> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i1_nxv128i128 = zext <vscale x 128 x i1> undef to <vscale x 128 x i128>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll
index 80efe912c869..30cb32ce4eaf 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll
@@ -1141,7 +1141,7 @@ define signext i32 @vreduce_add_nxv4i32(<vscale x 4 x i32> %v) {
 
 define signext i32 @vwreduce_add_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vwreduce_add_nxv4i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = sext <vscale x 4 x i16> %v to <vscale x 4 x i32>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e = sext <vscale x 4 x i16> %v to <vscale x 4 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %e)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
@@ -1157,7 +1157,7 @@ define signext i32 @vwreduce_add_nxv4i16(<vscale x 4 x i16> %v) {
 
 define signext i32 @vwreduce_uadd_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vwreduce_uadd_nxv4i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = zext <vscale x 4 x i16> %v to <vscale x 4 x i32>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e = zext <vscale x 4 x i16> %v to <vscale x 4 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %e)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
@@ -1445,7 +1445,7 @@ define i64 @vreduce_add_nxv2i64(<vscale x 2 x i64> %v) {
 
 define i64 @vwreduce_add_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vwreduce_add_nxv2i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = sext <vscale x 2 x i32> %v to <vscale x 2 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e = sext <vscale x 2 x i32> %v to <vscale x 2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %e)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
@@ -1461,7 +1461,7 @@ define i64 @vwreduce_add_nxv2i32(<vscale x 2 x i32> %v) {
 
 define i64 @vwreduce_uadd_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vwreduce_uadd_nxv2i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = zext <vscale x 2 x i32> %v to <vscale x 2 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e = zext <vscale x 2 x i32> %v to <vscale x 2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %e)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
@@ -1597,7 +1597,7 @@ define i64 @vreduce_add_nxv4i64(<vscale x 4 x i64> %v) {
 
 define i64 @vwreduce_add_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vwreduce_add_nxv4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = sext <vscale x 4 x i32> %v to <vscale x 4 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e = sext <vscale x 4 x i32> %v to <vscale x 4 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> %e)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
@@ -1613,7 +1613,7 @@ define i64 @vwreduce_add_nxv4i32(<vscale x 4 x i32> %v) {
 
 define i64 @vwreduce_uadd_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vwreduce_uadd_nxv4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = zext <vscale x 4 x i32> %v to <vscale x 4 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e = zext <vscale x 4 x i32> %v to <vscale x 4 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> %e)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-extractelement.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-extractelement.ll
index 225bad6da591..aa7a90bece33 100644
--- a/llvm/test/Analysis/CostModel/RISCV/rvv-extractelement.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/rvv-extractelement.ll
@@ -12,12 +12,12 @@ define void @extractelement_int(i32 %x) {
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i1_0 = extractelement <4 x i1> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i1_0 = extractelement <8 x i1> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i1_0 = extractelement <16 x i1> undef, i32 0
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i1_0 = extractelement <vscale x 2 x i1> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4i1_0 = extractelement <vscale x 4 x i1> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv8i1_0 = extractelement <vscale x 8 x i1> undef, i32 0
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = extractelement <2 x i8> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = extractelement <4 x i8> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = extractelement <8 x i8> undef, i32 0
@@ -66,12 +66,12 @@ define void @extractelement_int(i32 %x) {
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_1 = extractelement <4 x i1> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_1 = extractelement <8 x i1> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_1 = extractelement <16 x i1> undef, i32 1
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_1 = extractelement <vscale x 2 x i1> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_1 = extractelement <vscale x 4 x i1> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_1 = extractelement <vscale x 8 x i1> undef, i32 1
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = extractelement <2 x i8> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = extractelement <4 x i8> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = extractelement <8 x i8> undef, i32 1
@@ -120,12 +120,12 @@ define void @extractelement_int(i32 %x) {
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_x = extractelement <4 x i1> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_x = extractelement <8 x i1> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_x = extractelement <16 x i1> undef, i32 %x
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_x = extractelement <vscale x 2 x i1> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_x = extractelement <vscale x 4 x i1> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_x = extractelement <vscale x 8 x i1> undef, i32 %x
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_x = extractelement <2 x i8> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_x = extractelement <4 x i8> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_x = extractelement <8 x i8> undef, i32 %x
@@ -177,12 +177,12 @@ define void @extractelement_int(i32 %x) {
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i1_0 = extractelement <4 x i1> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i1_0 = extractelement <8 x i1> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i1_0 = extractelement <16 x i1> undef, i32 0
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i1_0 = extractelement <vscale x 2 x i1> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4i1_0 = extractelement <vscale x 4 x i1> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv8i1_0 = extractelement <vscale x 8 x i1> undef, i32 0
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = extractelement <2 x i8> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = extractelement <4 x i8> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = extractelement <8 x i8> undef, i32 0
@@ -231,12 +231,12 @@ define void @extractelement_int(i32 %x) {
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_1 = extractelement <4 x i1> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_1 = extractelement <8 x i1> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_1 = extractelement <16 x i1> undef, i32 1
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_1 = extractelement <vscale x 2 x i1> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_1 = extractelement <vscale x 4 x i1> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_1 = extractelement <vscale x 8 x i1> undef, i32 1
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = extractelement <2 x i8> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = extractelement <4 x i8> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = extractelement <8 x i8> undef, i32 1
@@ -285,12 +285,12 @@ define void @extractelement_int(i32 %x) {
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_x = extractelement <4 x i1> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_x = extractelement <8 x i1> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_x = extractelement <16 x i1> undef, i32 %x
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_x = extractelement <vscale x 2 x i1> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_x = extractelement <vscale x 4 x i1> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_x = extractelement <vscale x 8 x i1> undef, i32 %x
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_x = extractelement <2 x i8> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_x = extractelement <4 x i8> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_x = extractelement <8 x i8> undef, i32 %x
@@ -341,13 +341,13 @@ define void @extractelement_int(i32 %x) {
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i1_0 = extractelement <2 x i1> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i1_0 = extractelement <4 x i1> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i1_0 = extractelement <8 x i1> undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i1_0 = extractelement <16 x i1> undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = extractelement <16 x i1> undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i1_0 = extractelement <vscale x 2 x i1> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4i1_0 = extractelement <vscale x 4 x i1> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv8i1_0 = extractelement <vscale x 8 x i1> undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = extractelement <2 x i8> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = extractelement <4 x i8> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = extractelement <8 x i8> undef, i32 0
@@ -395,13 +395,13 @@ define void @extractelement_int(i32 %x) {
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i1_1 = extractelement <2 x i1> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_1 = extractelement <4 x i1> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_1 = extractelement <8 x i1> undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_1 = extractelement <16 x i1> undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = extractelement <16 x i1> undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_1 = extractelement <vscale x 2 x i1> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_1 = extractelement <vscale x 4 x i1> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_1 = extractelement <vscale x 8 x i1> undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = extractelement <2 x i8> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = extractelement <4 x i8> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = extractelement <8 x i8> undef, i32 1
@@ -449,13 +449,13 @@ define void @extractelement_int(i32 %x) {
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i1_x = extractelement <2 x i1> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_x = extractelement <4 x i1> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_x = extractelement <8 x i1> undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_x = extractelement <16 x i1> undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_x = extractelement <16 x i1> undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_x = extractelement <vscale x 2 x i1> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_x = extractelement <vscale x 4 x i1> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_x = extractelement <vscale x 8 x i1> undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_x = extractelement <2 x i8> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_x = extractelement <4 x i8> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_x = extractelement <8 x i8> undef, i32 %x
@@ -506,13 +506,13 @@ define void @extractelement_int(i32 %x) {
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i1_0 = extractelement <2 x i1> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i1_0 = extractelement <4 x i1> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i1_0 = extractelement <8 x i1> undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i1_0 = extractelement <16 x i1> undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = extractelement <16 x i1> undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i1_0 = extractelement <vscale x 2 x i1> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4i1_0 = extractelement <vscale x 4 x i1> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv8i1_0 = extractelement <vscale x 8 x i1> undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = extractelement <2 x i8> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = extractelement <4 x i8> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = extractelement <8 x i8> undef, i32 0
@@ -560,13 +560,13 @@ define void @extractelement_int(i32 %x) {
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i1_1 = extractelement <2 x i1> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_1 = extractelement <4 x i1> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_1 = extractelement <8 x i1> undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_1 = extractelement <16 x i1> undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = extractelement <16 x i1> undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_1 = extractelement <vscale x 2 x i1> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_1 = extractelement <vscale x 4 x i1> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_1 = extractelement <vscale x 8 x i1> undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = extractelement <2 x i8> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = extractelement <4 x i8> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = extractelement <8 x i8> undef, i32 1
@@ -614,13 +614,13 @@ define void @extractelement_int(i32 %x) {
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i1_x = extractelement <2 x i1> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_x = extractelement <4 x i1> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_x = extractelement <8 x i1> undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_x = extractelement <16 x i1> undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_x = extractelement <16 x i1> undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_x = extractelement <vscale x 2 x i1> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_x = extractelement <vscale x 4 x i1> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_x = extractelement <vscale x 8 x i1> undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_x = extractelement <2 x i8> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_x = extractelement <4 x i8> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_x = extractelement <8 x i8> undef, i32 %x
diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll
index 5387c8dc3594..6e1ae0216f76 100644
--- a/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll
@@ -12,12 +12,12 @@ define void @insertelement_int(i32 %x) {
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v4i1_0 = insertelement <4 x i1> undef, i1 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v8i1_0 = insertelement <8 x i1> undef, i1 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv2i1_0 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv4i1_0 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv8i1_0 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 0
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = insertelement <2 x i8> undef, i8 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = insertelement <4 x i8> undef, i8 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = insertelement <8 x i8> undef, i8 undef, i32 0
@@ -66,12 +66,12 @@ define void @insertelement_int(i32 %x) {
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v4i1_1 = insertelement <4 x i1> undef, i1 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i1_1 = insertelement <8 x i1> undef, i1 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv2i1_1 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i1_1 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8i1_1 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 1
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = insertelement <2 x i8> undef, i8 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = insertelement <4 x i8> undef, i8 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = insertelement <8 x i8> undef, i8 undef, i32 1
@@ -120,12 +120,12 @@ define void @insertelement_int(i32 %x) {
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i1_x = insertelement <4 x i1> undef, i1 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8i1_x = insertelement <8 x i1> undef, i1 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv2i1_x = insertelement <vscale x 2 x i1> undef, i1 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv4i1_x = insertelement <vscale x 4 x i1> undef, i1 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv8i1_x = insertelement <vscale x 8 x i1> undef, i1 undef, i32 %x
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i8_x = insertelement <2 x i8> undef, i8 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i8_x = insertelement <4 x i8> undef, i8 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i8_x = insertelement <8 x i8> undef, i8 undef, i32 %x
@@ -177,12 +177,12 @@ define void @insertelement_int(i32 %x) {
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v4i1_0 = insertelement <4 x i1> undef, i1 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v8i1_0 = insertelement <8 x i1> undef, i1 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv2i1_0 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv4i1_0 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv8i1_0 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 0
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = insertelement <2 x i8> undef, i8 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = insertelement <4 x i8> undef, i8 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = insertelement <8 x i8> undef, i8 undef, i32 0
@@ -231,12 +231,12 @@ define void @insertelement_int(i32 %x) {
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v4i1_1 = insertelement <4 x i1> undef, i1 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i1_1 = insertelement <8 x i1> undef, i1 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv2i1_1 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i1_1 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8i1_1 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 1
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = insertelement <2 x i8> undef, i8 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = insertelement <4 x i8> undef, i8 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = insertelement <8 x i8> undef, i8 undef, i32 1
@@ -285,12 +285,12 @@ define void @insertelement_int(i32 %x) {
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i1_x = insertelement <4 x i1> undef, i1 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8i1_x = insertelement <8 x i1> undef, i1 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv2i1_x = insertelement <vscale x 2 x i1> undef, i1 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv4i1_x = insertelement <vscale x 4 x i1> undef, i1 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv8i1_x = insertelement <vscale x 8 x i1> undef, i1 undef, i32 %x
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i8_x = insertelement <2 x i8> undef, i8 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i8_x = insertelement <4 x i8> undef, i8 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i8_x = insertelement <8 x i8> undef, i8 undef, i32 %x
@@ -341,13 +341,13 @@ define void @insertelement_int(i32 %x) {
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v2i1_0 = insertelement <2 x i1> undef, i1 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v4i1_0 = insertelement <4 x i1> undef, i1 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v8i1_0 = insertelement <8 x i1> undef, i1 undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv2i1_0 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv4i1_0 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv8i1_0 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = insertelement <2 x i8> undef, i8 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = insertelement <4 x i8> undef, i8 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = insertelement <8 x i8> undef, i8 undef, i32 0
@@ -395,13 +395,13 @@ define void @insertelement_int(i32 %x) {
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v2i1_1 = insertelement <2 x i1> undef, i1 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v4i1_1 = insertelement <4 x i1> undef, i1 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i1_1 = insertelement <8 x i1> undef, i1 undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv2i1_1 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i1_1 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8i1_1 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = insertelement <2 x i8> undef, i8 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = insertelement <4 x i8> undef, i8 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = insertelement <8 x i8> undef, i8 undef, i32 1
@@ -449,13 +449,13 @@ define void @insertelement_int(i32 %x) {
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2i1_x = insertelement <2 x i1> undef, i1 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i1_x = insertelement <4 x i1> undef, i1 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8i1_x = insertelement <8 x i1> undef, i1 undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv2i1_x = insertelement <vscale x 2 x i1> undef, i1 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv4i1_x = insertelement <vscale x 4 x i1> undef, i1 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv8i1_x = insertelement <vscale x 8 x i1> undef, i1 undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i8_x = insertelement <2 x i8> undef, i8 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i8_x = insertelement <4 x i8> undef, i8 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i8_x = insertelement <8 x i8> undef, i8 undef, i32 %x
@@ -506,13 +506,13 @@ define void @insertelement_int(i32 %x) {
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v2i1_0 = insertelement <2 x i1> undef, i1 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v4i1_0 = insertelement <4 x i1> undef, i1 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v8i1_0 = insertelement <8 x i1> undef, i1 undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv2i1_0 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv4i1_0 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv8i1_0 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = insertelement <2 x i8> undef, i8 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = insertelement <4 x i8> undef, i8 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = insertelement <8 x i8> undef, i8 undef, i32 0
@@ -560,13 +560,13 @@ define void @insertelement_int(i32 %x) {
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v2i1_1 = insertelement <2 x i1> undef, i1 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v4i1_1 = insertelement <4 x i1> undef, i1 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i1_1 = insertelement <8 x i1> undef, i1 undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv2i1_1 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i1_1 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8i1_1 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = insertelement <2 x i8> undef, i8 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = insertelement <4 x i8> undef, i8 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = insertelement <8 x i8> undef, i8 undef, i32 1
@@ -614,13 +614,13 @@ define void @insertelement_int(i32 %x) {
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2i1_x = insertelement <2 x i1> undef, i1 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i1_x = insertelement <4 x i1> undef, i1 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8i1_x = insertelement <8 x i1> undef, i1 undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv2i1_x = insertelement <vscale x 2 x i1> undef, i1 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv4i1_x = insertelement <vscale x 4 x i1> undef, i1 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv8i1_x = insertelement <vscale x 8 x i1> undef, i1 undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i8_x = insertelement <2 x i8> undef, i8 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i8_x = insertelement <4 x i8> undef, i8 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i8_x = insertelement <8 x i8> undef, i8 undef, i32 %x
diff --git a/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll b/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll
index 46bf3152ac5b..b763198e98ba 100644
--- a/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll
@@ -197,7 +197,7 @@ define void  @broadcast_fixed() #0{
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %41 = shufflevector <32 x i1> undef, <32 x i1> undef, <32 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %42 = shufflevector <64 x i1> undef, <64 x i1> undef, <64 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %43 = shufflevector <128 x i1> undef, <128 x i1> undef, <128 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %ins1 = insertelement <128 x i1> poison, i1 poison, i32 0
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %ins1 = insertelement <128 x i1> poison, i1 poison, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %44 = shufflevector <128 x i1> %ins1, <128 x i1> poison, <128 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ins2 = insertelement <2 x i8> poison, i8 3, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %45 = shufflevector <2 x i8> %ins2, <2 x i8> undef, <2 x i32> zeroinitializer
-- 
GitLab


From 082e7c480e033c1b973b8379a31929d04e236868 Mon Sep 17 00:00:00 2001
From: Teresa Johnson <tejohnson@google.com>
Date: Tue, 26 Mar 2024 20:06:27 -0700
Subject: [PATCH 369/404] [MemProf] Remove empty edges once after cloning
 (#85320)

Restructure the handling of edges that become empty during the cloning
process. Instead of removing them as they become empty (no context ids
and alloc type), do this once after all cloning is complete.

This has no effect on the cloning result, but prepares for a follow on
change that does improve the cloning. The structural change here reduces
the diffs for the follow on change, which would be much more difficult
with the previous handling.
---
 .../IPO/MemProfContextDisambiguation.cpp      | 84 ++++++++++++-------
 1 file changed, 54 insertions(+), 30 deletions(-)

diff --git a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
index ba5e3b637db7..4e4a49997766 100644
--- a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
+++ b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
@@ -310,8 +310,12 @@ public:
     // True if this node was effectively removed from the graph, in which case
     // its context id set, caller edges, and callee edges should all be empty.
     bool isRemoved() const {
-      assert(ContextIds.empty() ==
-             (CalleeEdges.empty() && CallerEdges.empty()));
+      // Note that we can have non-empty context ids with empty caller and
+      // callee edges if the graph ends up with a single node.
+      if (ContextIds.empty())
+        assert(CalleeEdges.empty() && CallerEdges.empty() &&
+               "Context ids empty but at least one of callee and caller edges "
+               "were not!");
       return ContextIds.empty();
     }
 
@@ -353,9 +357,12 @@ public:
     }
   };
 
-  /// Helper to remove callee edges that have allocation type None (due to not
+  /// Helpers to remove callee edges that have allocation type None (due to not
   /// carrying any context ids) after transformations.
   void removeNoneTypeCalleeEdges(ContextNode *Node);
+  void
+  recursivelyRemoveNoneTypeCalleeEdges(ContextNode *Node,
+                                       DenseSet<const ContextNode *> &Visited);
 
 protected:
   /// Get a list of nodes corresponding to the stack ids in the given callsite
@@ -2431,11 +2438,43 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
   }
 }
 
+template <typename DerivedCCG, typename FuncTy, typename CallTy>
+void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
+    recursivelyRemoveNoneTypeCalleeEdges(
+        ContextNode *Node, DenseSet<const ContextNode *> &Visited) {
+  auto Inserted = Visited.insert(Node);
+  if (!Inserted.second)
+    return;
+
+  removeNoneTypeCalleeEdges(Node);
+
+  for (auto *Clone : Node->Clones)
+    recursivelyRemoveNoneTypeCalleeEdges(Clone, Visited);
+
+  // The recursive call may remove some of this Node's caller edges.
+  // Iterate over a copy and skip any that were removed.
+  auto CallerEdges = Node->CallerEdges;
+  for (auto &Edge : CallerEdges) {
+    // Skip any that have been removed by an earlier recursive call.
+    if (Edge->Callee == nullptr && Edge->Caller == nullptr) {
+      assert(!std::count(Node->CallerEdges.begin(), Node->CallerEdges.end(),
+                         Edge));
+      continue;
+    }
+    recursivelyRemoveNoneTypeCalleeEdges(Edge->Caller, Visited);
+  }
+}
+
 template <typename DerivedCCG, typename FuncTy, typename CallTy>
 void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::identifyClones() {
   DenseSet<const ContextNode *> Visited;
   for (auto &Entry : AllocationCallToContextNodeMap)
     identifyClones(Entry.second, Visited);
+  Visited.clear();
+  for (auto &Entry : AllocationCallToContextNodeMap)
+    recursivelyRemoveNoneTypeCalleeEdges(Entry.second, Visited);
+  if (VerifyCCG)
+    check();
 }
 
 // helper function to check an AllocType is cold or notcold or both.
@@ -2450,7 +2489,7 @@ template <typename DerivedCCG, typename FuncTy, typename CallTy>
 void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::identifyClones(
     ContextNode *Node, DenseSet<const ContextNode *> &Visited) {
   if (VerifyNodes)
-    checkNode<DerivedCCG, FuncTy, CallTy>(Node);
+    checkNode<DerivedCCG, FuncTy, CallTy>(Node, /*CheckEdges=*/false);
   assert(!Node->CloneOf);
 
   // If Node as a null call, then either it wasn't found in the module (regular
@@ -2511,8 +2550,16 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::identifyClones(
   std::stable_sort(Node->CallerEdges.begin(), Node->CallerEdges.end(),
                    [&](const std::shared_ptr<ContextEdge> &A,
                        const std::shared_ptr<ContextEdge> &B) {
-                     assert(checkColdOrNotCold(A->AllocTypes) &&
-                            checkColdOrNotCold(B->AllocTypes));
+                     // Nodes with non-empty context ids should be sorted before
+                     // those with empty context ids.
+                     if (A->ContextIds.empty())
+                       // Either B ContextIds are non-empty (in which case we
+                       // should return false because B < A), or B ContextIds
+                       // are empty, in which case they are equal, and we should
+                       // maintain the original relative ordering.
+                       return false;
+                     if (B->ContextIds.empty())
+                       return true;
 
                      if (A->AllocTypes == B->AllocTypes)
                        // Use the first context id for each edge as a
@@ -2591,39 +2638,16 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::identifyClones(
            Node->AllocTypes != (uint8_t)AllocationType::None);
     // Sanity check that no alloc types on clone or its edges are None.
     assert(Clone->AllocTypes != (uint8_t)AllocationType::None);
-    assert(llvm::none_of(
-        Clone->CallerEdges, [&](const std::shared_ptr<ContextEdge> &E) {
-          return E->AllocTypes == (uint8_t)AllocationType::None;
-        }));
   }
 
-  // Cloning may have resulted in some cloned callee edges with type None,
-  // because they aren't carrying any contexts. Remove those edges.
-  for (auto *Clone : Node->Clones) {
-    removeNoneTypeCalleeEdges(Clone);
-    if (VerifyNodes)
-      checkNode<DerivedCCG, FuncTy, CallTy>(Clone);
-  }
   // We should still have some context ids on the original Node.
   assert(!Node->ContextIds.empty());
 
-  // Remove any callee edges that ended up with alloc type None after creating
-  // clones and updating callee edges.
-  removeNoneTypeCalleeEdges(Node);
-
   // Sanity check that no alloc types on node or edges are None.
   assert(Node->AllocTypes != (uint8_t)AllocationType::None);
-  assert(llvm::none_of(Node->CalleeEdges,
-                       [&](const std::shared_ptr<ContextEdge> &E) {
-                         return E->AllocTypes == (uint8_t)AllocationType::None;
-                       }));
-  assert(llvm::none_of(Node->CallerEdges,
-                       [&](const std::shared_ptr<ContextEdge> &E) {
-                         return E->AllocTypes == (uint8_t)AllocationType::None;
-                       }));
 
   if (VerifyNodes)
-    checkNode<DerivedCCG, FuncTy, CallTy>(Node);
+    checkNode<DerivedCCG, FuncTy, CallTy>(Node, /*CheckEdges=*/false);
 }
 
 void ModuleCallsiteContextGraph::updateAllocationCall(
-- 
GitLab


From 4d03a9ecc697a11f0edd3c31440a7cae3398e24a Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 26 Mar 2024 20:42:14 -0700
Subject: [PATCH 370/404] [RISCV] Preserve MMO when expanding
 PseudoRV32ZdinxSD/PseudoRV32ZdinxLD. (#85877)

This allows the asm printer to print the stack spill/reload messages.
---
 .../Target/RISCV/RISCVExpandPseudoInsts.cpp   | 35 +++++++++++++++----
 llvm/test/CodeGen/RISCV/zdinx-large-spill.mir | 32 ++++++++---------
 2 files changed, 44 insertions(+), 23 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
index 0a314fdd41cb..173995f05b51 100644
--- a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
@@ -312,10 +312,19 @@ bool RISCVExpandPseudo::expandRV32ZdinxStore(MachineBasicBlock &MBB,
       TRI->getSubReg(MBBI->getOperand(0).getReg(), RISCV::sub_gpr_even);
   Register Hi =
       TRI->getSubReg(MBBI->getOperand(0).getReg(), RISCV::sub_gpr_odd);
+
+  assert(MBBI->hasOneMemOperand() && "Expected mem operand");
+  MachineMemOperand *OldMMO = MBBI->memoperands().front();
+  MachineFunction *MF = MBB.getParent();
+  MachineMemOperand *MMOLo = MF->getMachineMemOperand(OldMMO, 0, 4);
+  MachineMemOperand *MMOHi = MF->getMachineMemOperand(OldMMO, 4, 4);
+
   BuildMI(MBB, MBBI, DL, TII->get(RISCV::SW))
       .addReg(Lo, getKillRegState(MBBI->getOperand(0).isKill()))
       .addReg(MBBI->getOperand(1).getReg())
-      .add(MBBI->getOperand(2));
+      .add(MBBI->getOperand(2))
+      .setMemRefs(MMOLo);
+
   if (MBBI->getOperand(2).isGlobal() || MBBI->getOperand(2).isCPI()) {
     // FIXME: Zdinx RV32 can not work on unaligned memory.
     assert(!STI->hasFastUnalignedAccess());
@@ -325,13 +334,15 @@ bool RISCVExpandPseudo::expandRV32ZdinxStore(MachineBasicBlock &MBB,
     BuildMI(MBB, MBBI, DL, TII->get(RISCV::SW))
         .addReg(Hi, getKillRegState(MBBI->getOperand(0).isKill()))
         .add(MBBI->getOperand(1))
-        .add(MBBI->getOperand(2));
+        .add(MBBI->getOperand(2))
+        .setMemRefs(MMOHi);
   } else {
     assert(isInt<12>(MBBI->getOperand(2).getImm() + 4));
     BuildMI(MBB, MBBI, DL, TII->get(RISCV::SW))
         .addReg(Hi, getKillRegState(MBBI->getOperand(0).isKill()))
         .add(MBBI->getOperand(1))
-        .addImm(MBBI->getOperand(2).getImm() + 4);
+        .addImm(MBBI->getOperand(2).getImm() + 4)
+        .setMemRefs(MMOHi);
   }
   MBBI->eraseFromParent();
   return true;
@@ -349,6 +360,12 @@ bool RISCVExpandPseudo::expandRV32ZdinxLoad(MachineBasicBlock &MBB,
   Register Hi =
       TRI->getSubReg(MBBI->getOperand(0).getReg(), RISCV::sub_gpr_odd);
 
+  assert(MBBI->hasOneMemOperand() && "Expected mem operand");
+  MachineMemOperand *OldMMO = MBBI->memoperands().front();
+  MachineFunction *MF = MBB.getParent();
+  MachineMemOperand *MMOLo = MF->getMachineMemOperand(OldMMO, 0, 4);
+  MachineMemOperand *MMOHi = MF->getMachineMemOperand(OldMMO, 4, 4);
+
   // If the register of operand 1 is equal to the Lo register, then swap the
   // order of loading the Lo and Hi statements.
   bool IsOp1EqualToLo = Lo == MBBI->getOperand(1).getReg();
@@ -356,7 +373,8 @@ bool RISCVExpandPseudo::expandRV32ZdinxLoad(MachineBasicBlock &MBB,
   if (!IsOp1EqualToLo) {
     BuildMI(MBB, MBBI, DL, TII->get(RISCV::LW), Lo)
         .addReg(MBBI->getOperand(1).getReg())
-        .add(MBBI->getOperand(2));
+        .add(MBBI->getOperand(2))
+        .setMemRefs(MMOLo);
   }
 
   if (MBBI->getOperand(2).isGlobal() || MBBI->getOperand(2).isCPI()) {
@@ -365,20 +383,23 @@ bool RISCVExpandPseudo::expandRV32ZdinxLoad(MachineBasicBlock &MBB,
     MBBI->getOperand(2).setOffset(Offset + 4);
     BuildMI(MBB, MBBI, DL, TII->get(RISCV::LW), Hi)
         .addReg(MBBI->getOperand(1).getReg())
-        .add(MBBI->getOperand(2));
+        .add(MBBI->getOperand(2))
+        .setMemRefs(MMOHi);
     MBBI->getOperand(2).setOffset(Offset);
   } else {
     assert(isInt<12>(MBBI->getOperand(2).getImm() + 4));
     BuildMI(MBB, MBBI, DL, TII->get(RISCV::LW), Hi)
         .addReg(MBBI->getOperand(1).getReg())
-        .addImm(MBBI->getOperand(2).getImm() + 4);
+        .addImm(MBBI->getOperand(2).getImm() + 4)
+        .setMemRefs(MMOHi);
   }
 
   // Order: Hi, Lo
   if (IsOp1EqualToLo) {
     BuildMI(MBB, MBBI, DL, TII->get(RISCV::LW), Lo)
         .addReg(MBBI->getOperand(1).getReg())
-        .add(MBBI->getOperand(2));
+        .add(MBBI->getOperand(2))
+        .setMemRefs(MMOLo);
   }
 
   MBBI->eraseFromParent();
diff --git a/llvm/test/CodeGen/RISCV/zdinx-large-spill.mir b/llvm/test/CodeGen/RISCV/zdinx-large-spill.mir
index a2a722a7f728..8596a65c378c 100644
--- a/llvm/test/CodeGen/RISCV/zdinx-large-spill.mir
+++ b/llvm/test/CodeGen/RISCV/zdinx-large-spill.mir
@@ -14,28 +14,28 @@
   ; CHECK-NEXT:    .cfi_def_cfa_offset 2064
   ; CHECK-NEXT:    lui t0, 1
   ; CHECK-NEXT:    add t0, sp, t0
-  ; CHECK-NEXT:    sw a0, -2040(t0)
-  ; CHECK-NEXT:    sw a1, -2036(t0)
+  ; CHECK-NEXT:    sw a0, -2040(t0) # 4-byte Folded Spill
+  ; CHECK-NEXT:    sw a1, -2036(t0) # 4-byte Folded Spill
   ; CHECK-NEXT:    lui a0, 1
   ; CHECK-NEXT:    add a0, sp, a0
-  ; CHECK-NEXT:    sw a2, -2048(a0)
-  ; CHECK-NEXT:    sw a3, -2044(a0)
-  ; CHECK-NEXT:    sw a4, 2040(sp)
-  ; CHECK-NEXT:    sw a5, 2044(sp)
-  ; CHECK-NEXT:    sw a6, 2032(sp)
-  ; CHECK-NEXT:    sw a7, 2036(sp)
+  ; CHECK-NEXT:    sw a2, -2048(a0) # 4-byte Folded Spill
+  ; CHECK-NEXT:    sw a3, -2044(a0) # 4-byte Folded Spill
+  ; CHECK-NEXT:    sw a4, 2040(sp) # 4-byte Folded Spill
+  ; CHECK-NEXT:    sw a5, 2044(sp) # 4-byte Folded Spill
+  ; CHECK-NEXT:    sw a6, 2032(sp) # 4-byte Folded Spill
+  ; CHECK-NEXT:    sw a7, 2036(sp) # 4-byte Folded Spill
   ; CHECK-NEXT:    lui a0, 1
   ; CHECK-NEXT:    add a0, sp, a0
-  ; CHECK-NEXT:    lw a1, -2036(a0)
-  ; CHECK-NEXT:    lw a0, -2040(a0)
+  ; CHECK-NEXT:    lw a1, -2036(a0) # 4-byte Folded Reload
+  ; CHECK-NEXT:    lw a0, -2040(a0) # 4-byte Folded Reload
   ; CHECK-NEXT:    lui a0, 1
   ; CHECK-NEXT:    add a0, sp, a0
-  ; CHECK-NEXT:    lw a2, -2048(a0)
-  ; CHECK-NEXT:    lw a3, -2044(a0)
-  ; CHECK-NEXT:    lw a4, 2040(sp)
-  ; CHECK-NEXT:    lw a5, 2044(sp)
-  ; CHECK-NEXT:    lw a6, 2032(sp)
-  ; CHECK-NEXT:    lw a7, 2036(sp)
+  ; CHECK-NEXT:    lw a2, -2048(a0) # 4-byte Folded Reload
+  ; CHECK-NEXT:    lw a3, -2044(a0) # 4-byte Folded Reload
+  ; CHECK-NEXT:    lw a4, 2040(sp) # 4-byte Folded Reload
+  ; CHECK-NEXT:    lw a5, 2044(sp) # 4-byte Folded Reload
+  ; CHECK-NEXT:    lw a6, 2032(sp) # 4-byte Folded Reload
+  ; CHECK-NEXT:    lw a7, 2036(sp) # 4-byte Folded Reload
   ; CHECK-NEXT:    addi sp, sp, 2032
   ; CHECK-NEXT:    addi sp, sp, 32
   ; CHECK-NEXT:    ret
-- 
GitLab


From 9961c03e9ec2fc47cb42fd16141b89dd8d8e2c01 Mon Sep 17 00:00:00 2001
From: Jeremy Day <jadaytime@gmail.com>
Date: Tue, 26 Mar 2024 21:03:29 -0700
Subject: [PATCH 371/404] Return `errc::no_such_file_or_directory` in
 `fs::access` if `GetFileAttributesW` fails (#83495)

Fixes https://github.com/llvm/llvm-project/issues/83046

There is a race condition when calling `GetFileAttributesW` that can
cause it to return `ERROR_ACCESS_DENIED` on a path which exists, which
is unexpected for callers using this function to check for file
existence by passing `AccessMode::Exist`. This was manifesting as a
compiler crash on Windows downstream in the Swift compiler when using
the `-index-store-path` flag (more information in
https://github.com/apple/llvm-project/issues/8224).

I looked for alternate APIs to avoid bringing in `shlwapi.h`, but didn't
see any good candidates. I'm not tied at all to this solution, any
feedback and alternative approaches are more than welcome.
---
 llvm/lib/Support/Windows/Path.inc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/llvm/lib/Support/Windows/Path.inc b/llvm/lib/Support/Windows/Path.inc
index 3e4c1f74161c..854d531ab371 100644
--- a/llvm/lib/Support/Windows/Path.inc
+++ b/llvm/lib/Support/Windows/Path.inc
@@ -623,6 +623,10 @@ std::error_code access(const Twine &Path, AccessMode Mode) {
   DWORD Attributes = ::GetFileAttributesW(PathUtf16.begin());
 
   if (Attributes == INVALID_FILE_ATTRIBUTES) {
+    // Avoid returning unexpected error codes when querying for existence.
+    if (Mode == AccessMode::Exist)
+      return errc::no_such_file_or_directory;
+
     // See if the file didn't actually exist.
     DWORD LastError = ::GetLastError();
     if (LastError != ERROR_FILE_NOT_FOUND && LastError != ERROR_PATH_NOT_FOUND)
-- 
GitLab


From fa9ee4a7f9f7fb9f586d40939269205fc3061c17 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Wed, 27 Mar 2024 00:09:40 -0400
Subject: [PATCH 372/404] [NFC][OpenMP] Silent unused variable in
 `kmp_collapse.cpp`

---
 openmp/runtime/src/kmp_collapse.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/openmp/runtime/src/kmp_collapse.cpp b/openmp/runtime/src/kmp_collapse.cpp
index e63a98081db9..f1bf04901dc7 100644
--- a/openmp/runtime/src/kmp_collapse.cpp
+++ b/openmp/runtime/src/kmp_collapse.cpp
@@ -1482,8 +1482,8 @@ void kmp_handle_upper_triangle_matrix(
                                     original_bounds_nest[0].ub0_u64);
   kmp_uint64 outer_lb0 = kmp_fix_iv(original_bounds_nest[0].loop_iv_type,
                                     original_bounds_nest[0].lb0_u64);
-  kmp_uint64 inner_ub0 = kmp_fix_iv(original_bounds_nest[1].loop_iv_type,
-                                    original_bounds_nest[1].ub0_u64);
+  [[maybe_unused]] kmp_uint64 inner_ub0 = kmp_fix_iv(
+      original_bounds_nest[1].loop_iv_type, original_bounds_nest[1].ub0_u64);
   // calculate the chunk's lower and upper bounds
   // the total number of iterations in the loop is the sum of the arithmetic
   // progression from the outer lower to outer upper bound (inclusive since the
-- 
GitLab


From a7ac0dd624962de1ccb55f1ed1357f548477f593 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Wed, 27 Mar 2024 00:23:32 -0400
Subject: [PATCH 373/404] [NFC][OpenMP] Use `SimpleVLA` to replace variable
 length arrays in C++

---
 openmp/runtime/src/kmp_csupport.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/openmp/runtime/src/kmp_csupport.cpp b/openmp/runtime/src/kmp_csupport.cpp
index 878e78b5c7ad..0268f692ff7f 100644
--- a/openmp/runtime/src/kmp_csupport.cpp
+++ b/openmp/runtime/src/kmp_csupport.cpp
@@ -18,6 +18,7 @@
 #include "kmp_itt.h"
 #include "kmp_lock.h"
 #include "kmp_stats.h"
+#include "kmp_utils.h"
 #include "ompt-specific.h"
 
 #define MAX_MESSAGE 512
@@ -4233,7 +4234,7 @@ void __kmpc_doacross_wait(ident_t *loc, int gtid, const kmp_int64 *vec) {
   up = pr_buf->th_doacross_info[3];
   st = pr_buf->th_doacross_info[4];
 #if OMPT_SUPPORT && OMPT_OPTIONAL
-  ompt_dependence_t deps[num_dims];
+  SimpleVLA<ompt_dependence_t> deps(num_dims);
 #endif
   if (st == 1) { // most common case
     if (vec[0] < lo || vec[0] > up) {
@@ -4345,7 +4346,7 @@ void __kmpc_doacross_post(ident_t *loc, int gtid, const kmp_int64 *vec) {
   lo = pr_buf->th_doacross_info[2];
   st = pr_buf->th_doacross_info[4];
 #if OMPT_SUPPORT && OMPT_OPTIONAL
-  ompt_dependence_t deps[num_dims];
+  SimpleVLA<ompt_dependence_t> deps(num_dims);
 #endif
   if (st == 1) { // most common case
     iter_number = vec[0] - lo;
-- 
GitLab


From ecf6bb2b4dbd26bbff2cee79db010369faa2fe90 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 26 Mar 2024 21:35:35 -0700
Subject: [PATCH 374/404] [RISCV] Add register overlap checks to the assembler
 for some Zvk* instructions. (#86745)

From the spec

| Instruction | Register | Cannot Overlap |
| ----------- | -------- | -------------- |
| vaes*.vs    | vs2      | vd             |
| vsm4r.vs    | vs2      | vd             |
| vsha2c[hl]  | vs1, vs2 | vd             |
| vsha2ms     | vs1, vs2 | vd             |
| sm3me       | vs2      | vd             |
| vsm3c       | vs2      | vd             |
---
 llvm/lib/Target/RISCV/RISCVInstrFormats.td |  2 ++
 llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td | 12 ++++++----
 llvm/test/MC/RISCV/rvv/zvkned-invalid.s    | 23 +++++++++++++++++++
 llvm/test/MC/RISCV/rvv/zvknh-invalid.s     | 26 ++++++++++++++++++++++
 llvm/test/MC/RISCV/rvv/zvksed-invalid.s    |  6 +++++
 llvm/test/MC/RISCV/rvv/zvksh-invalid.s     | 10 +++++++++
 llvm/test/MC/RISCV/rvv/zvksh.s             |  7 ++++++
 7 files changed, 82 insertions(+), 4 deletions(-)
 create mode 100644 llvm/test/MC/RISCV/rvv/zvkned-invalid.s
 create mode 100644 llvm/test/MC/RISCV/rvv/zvknh-invalid.s
 create mode 100644 llvm/test/MC/RISCV/rvv/zvksed-invalid.s
 create mode 100644 llvm/test/MC/RISCV/rvv/zvksh-invalid.s

diff --git a/llvm/lib/Target/RISCV/RISCVInstrFormats.td b/llvm/lib/Target/RISCV/RISCVInstrFormats.td
index f56f49ae2457..52c794446af0 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrFormats.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrFormats.td
@@ -109,6 +109,8 @@ def Vrgather     : RISCVVConstraint<!or(VS2Constraint.Value,
                                         VMConstraint.Value)>;
 def Vcompress    : RISCVVConstraint<!or(VS2Constraint.Value,
                                         VS1Constraint.Value)>;
+def Sha2Constraint : RISCVVConstraint<!or(VS2Constraint.Value,
+                                          VS1Constraint.Value)>;
 
 // The following opcode names match those given in Table 19.1 in the
 // RISC-V User-level ISA specification ("RISC-V base opcode map").
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
index 035ce63e91e9..b388bb00b0f2 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
@@ -82,7 +82,9 @@ class PALUVs2NoVm<bits<6> funct6, bits<5> vs1, RISCVVFormat opv, string opcodest
 
 multiclass VAES_MV_V_S<bits<6> funct6_vv, bits<6> funct6_vs, bits<5> vs1,
                          RISCVVFormat opv, string opcodestr> {
+  let RVVConstraint = NoConstraint in
   def NAME # _VV : PALUVs2NoVm<funct6_vv, vs1, opv, opcodestr # ".vv">;
+  let RVVConstraint = VS2Constraint in
   def NAME # _VS : PALUVs2NoVm<funct6_vs, vs1, opv, opcodestr # ".vs">;
 }
 } // hasSideEffects = 0, mayLoad = 0, mayStore = 0
@@ -118,28 +120,30 @@ let Predicates = [HasStdExtZvkg], RVVConstraint = NoConstraint in {
   def VGMUL_VV : PALUVs2NoVm<0b101000, 0b10001, OPMVV, "vgmul.vv">;
 } // Predicates = [HasStdExtZvkg]
 
-let Predicates = [HasStdExtZvknhaOrZvknhb], RVVConstraint = NoConstraint in {
+let Predicates = [HasStdExtZvknhaOrZvknhb], RVVConstraint = Sha2Constraint in {
   def VSHA2CH_VV : PALUVVNoVm<0b101110, OPMVV, "vsha2ch.vv">;
   def VSHA2CL_VV : PALUVVNoVm<0b101111, OPMVV, "vsha2cl.vv">;
   def VSHA2MS_VV : PALUVVNoVm<0b101101, OPMVV, "vsha2ms.vv">;
 } // Predicates = [HasStdExtZvknhaOrZvknhb]
 
-let Predicates = [HasStdExtZvkned], RVVConstraint = NoConstraint in {
+let Predicates = [HasStdExtZvkned]in {
   defm VAESDF     : VAES_MV_V_S<0b101000, 0b101001, 0b00001, OPMVV, "vaesdf">;
   defm VAESDM     : VAES_MV_V_S<0b101000, 0b101001, 0b00000, OPMVV, "vaesdm">;
   defm VAESEF     : VAES_MV_V_S<0b101000, 0b101001, 0b00011, OPMVV, "vaesef">;
   defm VAESEM     : VAES_MV_V_S<0b101000, 0b101001, 0b00010, OPMVV, "vaesem">;
   def  VAESKF1_VI : PALUVINoVm<0b100010, "vaeskf1.vi", uimm5>;
   def  VAESKF2_VI : PALUVINoVm<0b101010, "vaeskf2.vi", uimm5>;
+  let RVVConstraint = VS2Constraint in
   def  VAESZ_VS   : PALUVs2NoVm<0b101001, 0b00111, OPMVV, "vaesz.vs">;
 } // Predicates = [HasStdExtZvkned]
 
-let Predicates = [HasStdExtZvksed], RVVConstraint = NoConstraint in {
+let Predicates = [HasStdExtZvksed] in {
+  let RVVConstraint = NoConstraint in
   def  VSM4K_VI : PALUVINoVm<0b100001, "vsm4k.vi", uimm5>;
   defm VSM4R    : VAES_MV_V_S<0b101000, 0b101001, 0b10000, OPMVV, "vsm4r">;
 } // Predicates = [HasStdExtZvksed]
 
-let Predicates = [HasStdExtZvksh], RVVConstraint = NoConstraint in {
+let Predicates = [HasStdExtZvksh], RVVConstraint = VS2Constraint in {
   def VSM3C_VI  : PALUVINoVm<0b101011, "vsm3c.vi", uimm5>;
   def VSM3ME_VV : PALUVVNoVm<0b100000, OPMVV, "vsm3me.vv">;
 } // Predicates = [HasStdExtZvksh]
diff --git a/llvm/test/MC/RISCV/rvv/zvkned-invalid.s b/llvm/test/MC/RISCV/rvv/zvkned-invalid.s
new file mode 100644
index 000000000000..9230bc08e3fa
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/zvkned-invalid.s
@@ -0,0 +1,23 @@
+# RUN: not llvm-mc -triple=riscv64 --mattr=+zve64x --mattr=+zvkned %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+
+vaesdf.vs v10, v10
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vaesdf.vs v10, v10
+
+vaesef.vs v11, v11
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vaesef.vs v11, v11
+
+vaesdm.vs v12, v12
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vaesdm.vs v12, v12
+
+vaesem.vs v13, v13
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vaesem.vs v13, v13
+
+vaesz.vs v14, v14
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vaesz.vs v14, v14
+
diff --git a/llvm/test/MC/RISCV/rvv/zvknh-invalid.s b/llvm/test/MC/RISCV/rvv/zvknh-invalid.s
new file mode 100644
index 000000000000..d9902511c0e1
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/zvknh-invalid.s
@@ -0,0 +1,26 @@
+# RUN: not llvm-mc -triple=riscv64 --mattr=+zve64x --mattr=+zvknha %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+
+vsha2ms.vv v10, v10, v11
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vsha2ms.vv v10, v10, v11
+
+vsha2ms.vv v11, v10, v11
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vsha2ms.vv v11, v10, v11
+
+vsha2ch.vv v12, v12, v11
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vsha2ch.vv v12, v12, v11
+
+vsha2ch.vv v11, v12, v11
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vsha2ch.vv v11, v12, v11
+
+vsha2cl.vv v13, v13, v15
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vsha2cl.vv v13, v13, v15
+
+vsha2cl.vv v15, v13, v15
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vsha2cl.vv v15, v13, v15
diff --git a/llvm/test/MC/RISCV/rvv/zvksed-invalid.s b/llvm/test/MC/RISCV/rvv/zvksed-invalid.s
new file mode 100644
index 000000000000..41df8d3bc296
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/zvksed-invalid.s
@@ -0,0 +1,6 @@
+# RUN: not llvm-mc -triple=riscv64 --mattr=+zve64x --mattr=+zvksed %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+
+vsm4r.vs v10, v10
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vsm4r.vs v10, v10
diff --git a/llvm/test/MC/RISCV/rvv/zvksh-invalid.s b/llvm/test/MC/RISCV/rvv/zvksh-invalid.s
new file mode 100644
index 000000000000..cccec44b8191
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/zvksh-invalid.s
@@ -0,0 +1,10 @@
+# RUN: not llvm-mc -triple=riscv64 --mattr=+zve64x --mattr=+zvksh %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+
+vsm3me.vv v10, v10, v8
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vsm3me.vv v10, v10, v8
+
+vsm3c.vi v9, v9, 7
+# CHECK-ERROR: The destination vector register group cannot overlap the source vector register group.
+# CHECK-ERROR-LABEL: vsm3c.vi v9, v9, 7
diff --git a/llvm/test/MC/RISCV/rvv/zvksh.s b/llvm/test/MC/RISCV/rvv/zvksh.s
index ca6cb49d3079..06251ff6efe5 100644
--- a/llvm/test/MC/RISCV/rvv/zvksh.s
+++ b/llvm/test/MC/RISCV/rvv/zvksh.s
@@ -19,3 +19,10 @@ vsm3me.vv v10, v9, v8
 # CHECK-ENCODING: [0x77,0x25,0x94,0x82]
 # CHECK-ERROR: instruction requires the following: 'Zvksh' (SM3 Hash Function Instructions){{$}}
 # CHECK-UNKNOWN: 77 25 94 82   <unknown>
+
+# vs1 is allowed to overlap, but not vs2.
+vsm3me.vv v10, v9, v10
+# CHECK-INST: vsm3me.vv v10, v9, v10
+# CHECK-ENCODING: [0x77,0x25,0x95,0x82]
+# CHECK-ERROR: instruction requires the following: 'Zvksh' (SM3 Hash Function Instructions){{$}}
+# CHECK-UNKNOWN: 77 25 95 82   <unknown>
-- 
GitLab


From 8a9c1701704182d551bf9df1ff43363db56caab4 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 26 Mar 2024 21:37:19 -0700
Subject: [PATCH 375/404] [RISCV] Align stack size down to a multiple of 16
 before using cm.push/pop. (#86073)

This an alternative to #84935 to fix the miscompile, but not be optimal.

The immediate for cm.push/pop must be a multiple of 16. For RVE, it
might not be. It's not easy to increase the stack size without messing
up cfa directives and maybe other things.

This patch rounds the stack size down to a multiple of 16 before
clamping it to 48. This causes an extra addi to be emitted to handle the
remainder.

Once this commited, I can commit #84989 to add verification for these
instructions being generated with valid offsets.
---
 llvm/lib/Target/RISCV/RISCVFrameLowering.cpp     | 12 ++++++++----
 llvm/test/CodeGen/RISCV/zcmp-additional-stack.ll |  6 ++++--
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index 8bac41372b5a..39f2b3f62a9a 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -554,8 +554,10 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
   if (RVFI->isPushable(MF) && FirstFrameSetup != MBB.end() &&
       FirstFrameSetup->getOpcode() == RISCV::CM_PUSH) {
     // Use available stack adjustment in push instruction to allocate additional
-    // stack space.
-    uint64_t Spimm = std::min(StackSize, (uint64_t)48);
+    // stack space. Align the stack size down to a multiple of 16. This is
+    // needed for RVE.
+    // FIXME: Can we increase the stack size to a multiple of 16 instead?
+    uint64_t Spimm = std::min(alignDown(StackSize, 16), (uint64_t)48);
     FirstFrameSetup->getOperand(1).setImm(Spimm);
     StackSize -= Spimm;
   }
@@ -776,8 +778,10 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
   if (RVFI->isPushable(MF) && MBBI != MBB.end() &&
       MBBI->getOpcode() == RISCV::CM_POP) {
     // Use available stack adjustment in pop instruction to deallocate stack
-    // space.
-    uint64_t Spimm = std::min(StackSize, (uint64_t)48);
+    // space. Align the stack size down to a multiple of 16. This is needed for
+    // RVE.
+    // FIXME: Can we increase the stack size to a multiple of 16 instead?
+    uint64_t Spimm = std::min(alignDown(StackSize, 16), (uint64_t)48);
     MBBI->getOperand(1).setImm(Spimm);
     StackSize -= Spimm;
   }
diff --git a/llvm/test/CodeGen/RISCV/zcmp-additional-stack.ll b/llvm/test/CodeGen/RISCV/zcmp-additional-stack.ll
index e5c2e0180ee0..73ace2033985 100644
--- a/llvm/test/CodeGen/RISCV/zcmp-additional-stack.ll
+++ b/llvm/test/CodeGen/RISCV/zcmp-additional-stack.ll
@@ -3,7 +3,8 @@
 define ptr @func(ptr %s, i32 %_c, ptr %incdec.ptr, i1 %0, i8 %conv14) #0 {
 ; RV32-LABEL: func:
 ; RV32:       # %bb.0: # %entry
-; RV32-NEXT:    cm.push {ra, s0-s1}, -24
+; RV32-NEXT:    cm.push {ra, s0-s1}, -16
+; RV32-NEXT:    addi sp, sp, -8
 ; RV32-NEXT:    .cfi_def_cfa_offset 24
 ; RV32-NEXT:    .cfi_offset ra, -12
 ; RV32-NEXT:    .cfi_offset s0, -8
@@ -31,7 +32,8 @@ define ptr @func(ptr %s, i32 %_c, ptr %incdec.ptr, i1 %0, i8 %conv14) #0 {
 ; RV32-NEXT:    lw a0, 4(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    sb a0, 0(s0)
 ; RV32-NEXT:    mv a0, s1
-; RV32-NEXT:    cm.popret {ra, s0-s1}, 24
+; RV32-NEXT:    addi sp, sp, 8
+; RV32-NEXT:    cm.popret {ra, s0-s1}, 16
 entry:
   br label %while.body
 
-- 
GitLab


From da3e58e74ad160c3dfa89e92f7dd31efeee6b258 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung@sifive.com>
Date: Tue, 26 Mar 2024 21:46:37 -0700
Subject: [PATCH 376/404] Revert "[RISCV][TTI] Scale the cost of the sext/zext
 with LMUL (#86617)"

This reverts commit 7545c635729a2055a429c5decd26a619a8d6e74b as it's
failing on the Linux bots.
---
 .../Target/RISCV/RISCVTargetTransformInfo.cpp |  20 +-
 llvm/test/Analysis/CostModel/RISCV/cast.ll    | 920 +++++++++---------
 .../CostModel/RISCV/reduce-scalable-int.ll    |  12 +-
 .../CostModel/RISCV/rvv-extractelement.ll     |  84 +-
 .../CostModel/RISCV/rvv-insertelement.ll      |  84 +-
 .../CostModel/RISCV/shuffle-broadcast.ll      |   2 +-
 6 files changed, 556 insertions(+), 566 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index a0652d41e9f2..df289be18932 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -909,33 +909,23 @@ InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
   if (!IsTypeLegal)
     return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
 
-  std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(Dst);
-
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
   assert(ISD && "Invalid opcode");
 
+  // FIXME: Need to consider vsetvli and lmul.
   int PowDiff = (int)Log2_32(Dst->getScalarSizeInBits()) -
                 (int)Log2_32(Src->getScalarSizeInBits());
   switch (ISD) {
   case ISD::SIGN_EXTEND:
-  case ISD::ZERO_EXTEND: {
-    const unsigned SrcEltSize = Src->getScalarSizeInBits();
-    if (SrcEltSize == 1) {
+  case ISD::ZERO_EXTEND:
+    if (Src->getScalarSizeInBits() == 1) {
       // We do not use vsext/vzext to extend from mask vector.
       // Instead we use the following instructions to extend from mask vector:
       // vmv.v.i v8, 0
       // vmerge.vim v8, v8, -1, v0
-      return getRISCVInstructionCost({RISCV::VMV_V_I, RISCV::VMERGE_VIM},
-                                     DstLT.second, CostKind);
+      return 2;
     }
-    if (PowDiff > 3)
-      return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
-    unsigned SExtOp[] = {RISCV::VSEXT_VF2, RISCV::VSEXT_VF4, RISCV::VSEXT_VF8};
-    unsigned ZExtOp[] = {RISCV::VZEXT_VF2, RISCV::VZEXT_VF4, RISCV::VZEXT_VF8};
-    unsigned Op =
-        (ISD == ISD::SIGN_EXTEND) ? SExtOp[PowDiff - 1] : ZExtOp[PowDiff - 1];
-    return getRISCVInstructionCost(Op, DstLT.second, CostKind);
-  }
+    return 1;
   case ISD::TRUNCATE:
     if (Dst->getScalarSizeInBits() == 1) {
       // We do not use several vncvt to truncate to mask vector. So we could
diff --git a/llvm/test/Analysis/CostModel/RISCV/cast.ll b/llvm/test/Analysis/CostModel/RISCV/cast.ll
index 14da9a3f79d7..bd26c19c2f2c 100644
--- a/llvm/test/Analysis/CostModel/RISCV/cast.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/cast.ll
@@ -16,74 +16,74 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i1_v2i64 = sext <2 x i1> undef to <2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i16 = sext <4 x i8> undef to <4 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i32 = sext <4 x i8> undef to <4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_v4i64 = sext <4 x i8> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i64 = sext <4 x i8> undef to <4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i32 = sext <4 x i16> undef to <4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16_v4i64 = sext <4 x i16> undef to <4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32_v4i64 = sext <4 x i32> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i64 = sext <4 x i16> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_v4i64 = sext <4 x i32> undef to <4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i8 = sext <4 x i1> undef to <4 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i16 = sext <4 x i1> undef to <4 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i32 = sext <4 x i1> undef to <4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_v4i64 = sext <4 x i1> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i64 = sext <4 x i1> undef to <4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i16 = sext <8 x i8> undef to <8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_v8i32 = sext <8 x i8> undef to <8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_v8i64 = sext <8 x i8> undef to <8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16_v8i32 = sext <8 x i16> undef to <8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i16_v8i64 = sext <8 x i16> undef to <8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32_v8i64 = sext <8 x i32> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i32 = sext <8 x i8> undef to <8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i64 = sext <8 x i8> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i32 = sext <8 x i16> undef to <8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i64 = sext <8 x i16> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_v8i64 = sext <8 x i32> undef to <8 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i8 = sext <8 x i1> undef to <8 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i16 = sext <8 x i1> undef to <8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_v8i32 = sext <8 x i1> undef to <8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i1_v8i64 = sext <8 x i1> undef to <8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8_v16i16 = sext <16 x i8> undef to <16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i8_v16i32 = sext <16 x i8> undef to <16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_v16i64 = sext <16 x i8> undef to <16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16_v16i32 = sext <16 x i16> undef to <16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i16_v16i64 = sext <16 x i16> undef to <16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32_v16i64 = sext <16 x i32> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i32 = sext <8 x i1> undef to <8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i64 = sext <8 x i1> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i16 = sext <16 x i8> undef to <16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i32 = sext <16 x i8> undef to <16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i64 = sext <16 x i8> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i32 = sext <16 x i16> undef to <16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i64 = sext <16 x i16> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_v16i64 = sext <16 x i32> undef to <16 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i8 = sext <16 x i1> undef to <16 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_v16i16 = sext <16 x i1> undef to <16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_v16i32 = sext <16 x i1> undef to <16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i1_v16i64 = sext <16 x i1> undef to <16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i8_v32i16 = sext <32 x i8> undef to <32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i8_v32i32 = sext <32 x i8> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i8_v32i64 = sext <32 x i8> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i16_v32i32 = sext <32 x i16> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i16_v32i64 = sext <32 x i16> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i32_v32i64 = sext <32 x i32> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_v32i8 = sext <32 x i1> undef to <32 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_v32i16 = sext <32 x i1> undef to <32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32i32 = sext <32 x i1> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32i64 = sext <32 x i1> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i16 = sext <64 x i8> undef to <64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i8_v64i32 = sext <64 x i8> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i8_v64i64 = sext <64 x i8> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i16_v64i32 = sext <64 x i16> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i16_v64i64 = sext <64 x i16> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64i64 = sext <64 x i32> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i1_v64i8 = sext <64 x i1> undef to <64 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i1_v64i16 = sext <64 x i1> undef to <64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64i32 = sext <64 x i1> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64i64 = sext <64 x i1> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v128i8_v128i16 = sext <128 x i8> undef to <128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v128i8_v128i32 = sext <128 x i8> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %v128i8_v128i64 = sext <128 x i8> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v128i16_v128i32 = sext <128 x i16> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v128i16_v128i64 = sext <128 x i16> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128i64 = sext <128 x i32> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v128i1_v128i8 = sext <128 x i1> undef to <128 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v128i1_v128i16 = sext <128 x i1> undef to <128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128i32 = sext <128 x i1> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128i64 = sext <128 x i1> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v256i8_v256i16 = sext <256 x i8> undef to <256 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v256i8_v256i32 = sext <256 x i8> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %v256i8_v256i64 = sext <256 x i8> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v256i16_v256i32 = sext <256 x i16> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %v256i16_v256i64 = sext <256 x i16> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %v256i32_v256i64 = sext <256 x i32> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v256i1_v256i8 = sext <256 x i1> undef to <256 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v256i1_v256i16 = sext <256 x i1> undef to <256 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %v256i1_v256i32 = sext <256 x i1> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %v256i1_v256i64 = sext <256 x i1> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i16 = sext <16 x i1> undef to <16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i32 = sext <16 x i1> undef to <16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i64 = sext <16 x i1> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i16 = sext <32 x i8> undef to <32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i32 = sext <32 x i8> undef to <32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i8_v32i64 = sext <32 x i8> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_v32i32 = sext <32 x i16> undef to <32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i16_v32i64 = sext <32 x i16> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i32_v32i64 = sext <32 x i32> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i8 = sext <32 x i1> undef to <32 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i16 = sext <32 x i1> undef to <32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i32 = sext <32 x i1> undef to <32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_v32i64 = sext <32 x i1> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_v64i16 = sext <64 x i8> undef to <64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i8_v64i32 = sext <64 x i8> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i8_v64i64 = sext <64 x i8> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i16_v64i32 = sext <64 x i16> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i16_v64i64 = sext <64 x i16> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v64i32_v64i64 = sext <64 x i32> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i8 = sext <64 x i1> undef to <64 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i16 = sext <64 x i1> undef to <64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v64i1_v64i32 = sext <64 x i1> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v64i1_v64i64 = sext <64 x i1> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v128i8_v128i16 = sext <128 x i8> undef to <128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v128i8_v128i32 = sext <128 x i8> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v128i8_v128i64 = sext <128 x i8> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v128i16_v128i32 = sext <128 x i16> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v128i16_v128i64 = sext <128 x i16> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v128i32_v128i64 = sext <128 x i32> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v128i1_v128i8 = sext <128 x i1> undef to <128 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v128i1_v128i16 = sext <128 x i1> undef to <128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v128i1_v128i32 = sext <128 x i1> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v128i1_v128i64 = sext <128 x i1> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v256i8_v256i16 = sext <256 x i8> undef to <256 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v256i8_v256i32 = sext <256 x i8> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %v256i8_v256i64 = sext <256 x i8> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v256i16_v256i32 = sext <256 x i16> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v256i16_v256i64 = sext <256 x i16> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v256i32_v256i64 = sext <256 x i32> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v256i1_v256i8 = sext <256 x i1> undef to <256 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v256i1_v256i16 = sext <256 x i1> undef to <256 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %v256i1_v256i32 = sext <256 x i1> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %v256i1_v256i64 = sext <256 x i1> undef to <256 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i16 = sext <vscale x 1 x i8> undef to <vscale x 1 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i32 = sext <vscale x 1 x i8> undef to <vscale x 1 x i32>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i64 = sext <vscale x 1 x i8> undef to <vscale x 1 x i64>
@@ -96,73 +96,73 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i1_nxv1i64 = sext <vscale x 1 x i1> undef to <vscale x 1 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i16 = sext <vscale x 2 x i8> undef to <vscale x 2 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i32 = sext <vscale x 2 x i8> undef to <vscale x 2 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i8_nxv2i64 = sext <vscale x 2 x i8> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i64 = sext <vscale x 2 x i8> undef to <vscale x 2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i32 = sext <vscale x 2 x i16> undef to <vscale x 2 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i16_nxv2i64 = sext <vscale x 2 x i16> undef to <vscale x 2 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i32_nxv2i64 = sext <vscale x 2 x i32> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i64 = sext <vscale x 2 x i16> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32_nxv2i64 = sext <vscale x 2 x i32> undef to <vscale x 2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i8 = sext <vscale x 2 x i1> undef to <vscale x 2 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i16 = sext <vscale x 2 x i1> undef to <vscale x 2 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i32 = sext <vscale x 2 x i1> undef to <vscale x 2 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_nxv2i64 = sext <vscale x 2 x i1> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i64 = sext <vscale x 2 x i1> undef to <vscale x 2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i16 = sext <vscale x 4 x i8> undef to <vscale x 4 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i8_nxv4i32 = sext <vscale x 4 x i8> undef to <vscale x 4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i8_nxv4i64 = sext <vscale x 4 x i8> undef to <vscale x 4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i16_nxv4i32 = sext <vscale x 4 x i16> undef to <vscale x 4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i16_nxv4i64 = sext <vscale x 4 x i16> undef to <vscale x 4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i32_nxv4i64 = sext <vscale x 4 x i32> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i32 = sext <vscale x 4 x i8> undef to <vscale x 4 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i64 = sext <vscale x 4 x i8> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i32 = sext <vscale x 4 x i16> undef to <vscale x 4 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i64 = sext <vscale x 4 x i16> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32_nxv4i64 = sext <vscale x 4 x i32> undef to <vscale x 4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i8 = sext <vscale x 4 x i1> undef to <vscale x 4 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i16 = sext <vscale x 4 x i1> undef to <vscale x 4 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_nxv4i32 = sext <vscale x 4 x i1> undef to <vscale x 4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv4i1_nxv4i64 = sext <vscale x 4 x i1> undef to <vscale x 4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i8_nxv8i16 = sext <vscale x 8 x i8> undef to <vscale x 8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i8_nxv8i32 = sext <vscale x 8 x i8> undef to <vscale x 8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i8_nxv8i64 = sext <vscale x 8 x i8> undef to <vscale x 8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i16_nxv8i32 = sext <vscale x 8 x i16> undef to <vscale x 8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i16_nxv8i64 = sext <vscale x 8 x i16> undef to <vscale x 8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i32_nxv8i64 = sext <vscale x 8 x i32> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i32 = sext <vscale x 4 x i1> undef to <vscale x 4 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i64 = sext <vscale x 4 x i1> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i16 = sext <vscale x 8 x i8> undef to <vscale x 8 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i32 = sext <vscale x 8 x i8> undef to <vscale x 8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i64 = sext <vscale x 8 x i8> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i32 = sext <vscale x 8 x i16> undef to <vscale x 8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i64 = sext <vscale x 8 x i16> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32_nxv8i64 = sext <vscale x 8 x i32> undef to <vscale x 8 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i8 = sext <vscale x 8 x i1> undef to <vscale x 8 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_nxv8i16 = sext <vscale x 8 x i1> undef to <vscale x 8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i1_nxv8i32 = sext <vscale x 8 x i1> undef to <vscale x 8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv8i1_nxv8i64 = sext <vscale x 8 x i1> undef to <vscale x 8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i8_nxv16i16 = sext <vscale x 16 x i8> undef to <vscale x 16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i8_nxv16i32 = sext <vscale x 16 x i8> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i8_nxv16i64 = sext <vscale x 16 x i8> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i16_nxv16i32 = sext <vscale x 16 x i16> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i16_nxv16i64 = sext <vscale x 16 x i16> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i32_nxv16i64 = sext <vscale x 16 x i32> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_nxv16i8 = sext <vscale x 16 x i1> undef to <vscale x 16 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_nxv16i16 = sext <vscale x 16 x i1> undef to <vscale x 16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16i32 = sext <vscale x 16 x i1> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16i64 = sext <vscale x 16 x i1> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i8_nxv32i16 = sext <vscale x 32 x i8> undef to <vscale x 32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i8_nxv32i32 = sext <vscale x 32 x i8> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i8_nxv32i64 = sext <vscale x 32 x i8> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i16_nxv32i32 = sext <vscale x 32 x i16> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i16_nxv32i64 = sext <vscale x 32 x i16> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32i64 = sext <vscale x 32 x i32> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i1_nxv32i8 = sext <vscale x 32 x i1> undef to <vscale x 32 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv32i1_nxv32i16 = sext <vscale x 32 x i1> undef to <vscale x 32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32i32 = sext <vscale x 32 x i1> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32i64 = sext <vscale x 32 x i1> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv64i8_nxv64i16 = sext <vscale x 64 x i8> undef to <vscale x 64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv64i8_nxv64i32 = sext <vscale x 64 x i8> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i16 = sext <vscale x 8 x i1> undef to <vscale x 8 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i32 = sext <vscale x 8 x i1> undef to <vscale x 8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i64 = sext <vscale x 8 x i1> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i16 = sext <vscale x 16 x i8> undef to <vscale x 16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i32 = sext <vscale x 16 x i8> undef to <vscale x 16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i8_nxv16i64 = sext <vscale x 16 x i8> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16_nxv16i32 = sext <vscale x 16 x i16> undef to <vscale x 16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i16_nxv16i64 = sext <vscale x 16 x i16> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i32_nxv16i64 = sext <vscale x 16 x i32> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i8 = sext <vscale x 16 x i1> undef to <vscale x 16 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i16 = sext <vscale x 16 x i1> undef to <vscale x 16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i32 = sext <vscale x 16 x i1> undef to <vscale x 16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_nxv16i64 = sext <vscale x 16 x i1> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8_nxv32i16 = sext <vscale x 32 x i8> undef to <vscale x 32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i8_nxv32i32 = sext <vscale x 32 x i8> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i8_nxv32i64 = sext <vscale x 32 x i8> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i16_nxv32i32 = sext <vscale x 32 x i16> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i16_nxv32i64 = sext <vscale x 32 x i16> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i32_nxv32i64 = sext <vscale x 32 x i32> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i8 = sext <vscale x 32 x i1> undef to <vscale x 32 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i16 = sext <vscale x 32 x i1> undef to <vscale x 32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_nxv32i32 = sext <vscale x 32 x i1> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_nxv32i64 = sext <vscale x 32 x i1> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv64i8_nxv64i16 = sext <vscale x 64 x i8> undef to <vscale x 64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv64i8_nxv64i32 = sext <vscale x 64 x i8> undef to <vscale x 64 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i8_nxv64i64 = sext <vscale x 64 x i8> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv64i16_nxv64i32 = sext <vscale x 64 x i16> undef to <vscale x 64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %nxv64i16_nxv64i64 = sext <vscale x 64 x i16> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %nxv64i32_nxv64i64 = sext <vscale x 64 x i32> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv64i1_nxv64i8 = sext <vscale x 64 x i1> undef to <vscale x 64 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv64i1_nxv64i16 = sext <vscale x 64 x i1> undef to <vscale x 64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64i32 = sext <vscale x 64 x i1> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv64i16_nxv64i32 = sext <vscale x 64 x i16> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %nxv64i16_nxv64i64 = sext <vscale x 64 x i16> undef to <vscale x 64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv64i32_nxv64i64 = sext <vscale x 64 x i32> undef to <vscale x 64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv64i1_nxv64i8 = sext <vscale x 64 x i1> undef to <vscale x 64 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv64i1_nxv64i16 = sext <vscale x 64 x i1> undef to <vscale x 64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv64i1_nxv64i32 = sext <vscale x 64 x i1> undef to <vscale x 64 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i1_nxv64i64 = sext <vscale x 64 x i1> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv128i8_nxv128i16 = sext <vscale x 128 x i8> undef to <vscale x 128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv128i8_nxv128i32 = sext <vscale x 128 x i8> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv128i8_nxv128i16 = sext <vscale x 128 x i8> undef to <vscale x 128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %nxv128i8_nxv128i32 = sext <vscale x 128 x i8> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i8_nxv128i128 = sext <vscale x 128 x i8> undef to <vscale x 128 x i128>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv128i16_nxv128i32 = sext <vscale x 128 x i16> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv128i16_nxv128i32 = sext <vscale x 128 x i16> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i16_nxv128i128 = sext <vscale x 128 x i16> undef to <vscale x 128 x i128>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i32_nxv128i128 = sext <vscale x 128 x i32> undef to <vscale x 128 x i128>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %nxv128i1_nxv128i8 = sext <vscale x 128 x i1> undef to <vscale x 128 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %nxv128i1_nxv128i16 = sext <vscale x 128 x i1> undef to <vscale x 128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %nxv128i1_nxv128i32 = sext <vscale x 128 x i1> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv128i1_nxv128i8 = sext <vscale x 128 x i1> undef to <vscale x 128 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv128i1_nxv128i16 = sext <vscale x 128 x i1> undef to <vscale x 128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %nxv128i1_nxv128i32 = sext <vscale x 128 x i1> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i1_nxv128i128 = sext <vscale x 128 x i1> undef to <vscale x 128 x i128>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -179,74 +179,74 @@ define void @sext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i1_v2i64 = sext <2 x i1> undef to <2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i16 = sext <4 x i8> undef to <4 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i32 = sext <4 x i8> undef to <4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_v4i64 = sext <4 x i8> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i64 = sext <4 x i8> undef to <4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i32 = sext <4 x i16> undef to <4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16_v4i64 = sext <4 x i16> undef to <4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32_v4i64 = sext <4 x i32> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i64 = sext <4 x i16> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_v4i64 = sext <4 x i32> undef to <4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i8 = sext <4 x i1> undef to <4 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i16 = sext <4 x i1> undef to <4 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i32 = sext <4 x i1> undef to <4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_v4i64 = sext <4 x i1> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i64 = sext <4 x i1> undef to <4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i16 = sext <8 x i8> undef to <8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_v8i32 = sext <8 x i8> undef to <8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_v8i64 = sext <8 x i8> undef to <8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16_v8i32 = sext <8 x i16> undef to <8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i16_v8i64 = sext <8 x i16> undef to <8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32_v8i64 = sext <8 x i32> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i32 = sext <8 x i8> undef to <8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i64 = sext <8 x i8> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i32 = sext <8 x i16> undef to <8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i64 = sext <8 x i16> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_v8i64 = sext <8 x i32> undef to <8 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i8 = sext <8 x i1> undef to <8 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i16 = sext <8 x i1> undef to <8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_v8i32 = sext <8 x i1> undef to <8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i1_v8i64 = sext <8 x i1> undef to <8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8_v16i16 = sext <16 x i8> undef to <16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i8_v16i32 = sext <16 x i8> undef to <16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_v16i64 = sext <16 x i8> undef to <16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16_v16i32 = sext <16 x i16> undef to <16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i16_v16i64 = sext <16 x i16> undef to <16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32_v16i64 = sext <16 x i32> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i32 = sext <8 x i1> undef to <8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i64 = sext <8 x i1> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i16 = sext <16 x i8> undef to <16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i32 = sext <16 x i8> undef to <16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i64 = sext <16 x i8> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i32 = sext <16 x i16> undef to <16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i64 = sext <16 x i16> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_v16i64 = sext <16 x i32> undef to <16 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i8 = sext <16 x i1> undef to <16 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_v16i16 = sext <16 x i1> undef to <16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_v16i32 = sext <16 x i1> undef to <16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i1_v16i64 = sext <16 x i1> undef to <16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i8_v32i16 = sext <32 x i8> undef to <32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i8_v32i32 = sext <32 x i8> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i8_v32i64 = sext <32 x i8> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i16_v32i32 = sext <32 x i16> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i16_v32i64 = sext <32 x i16> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i32_v32i64 = sext <32 x i32> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_v32i8 = sext <32 x i1> undef to <32 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_v32i16 = sext <32 x i1> undef to <32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32i32 = sext <32 x i1> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32i64 = sext <32 x i1> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i16 = sext <64 x i8> undef to <64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i8_v64i32 = sext <64 x i8> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i8_v64i64 = sext <64 x i8> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i16_v64i32 = sext <64 x i16> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i16_v64i64 = sext <64 x i16> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64i64 = sext <64 x i32> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i1_v64i8 = sext <64 x i1> undef to <64 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i1_v64i16 = sext <64 x i1> undef to <64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64i32 = sext <64 x i1> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64i64 = sext <64 x i1> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v128i8_v128i16 = sext <128 x i8> undef to <128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v128i8_v128i32 = sext <128 x i8> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %v128i8_v128i64 = sext <128 x i8> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v128i16_v128i32 = sext <128 x i16> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v128i16_v128i64 = sext <128 x i16> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128i64 = sext <128 x i32> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v128i1_v128i8 = sext <128 x i1> undef to <128 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v128i1_v128i16 = sext <128 x i1> undef to <128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128i32 = sext <128 x i1> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128i64 = sext <128 x i1> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v256i8_v256i16 = sext <256 x i8> undef to <256 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v256i8_v256i32 = sext <256 x i8> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %v256i8_v256i64 = sext <256 x i8> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v256i16_v256i32 = sext <256 x i16> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %v256i16_v256i64 = sext <256 x i16> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %v256i32_v256i64 = sext <256 x i32> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v256i1_v256i8 = sext <256 x i1> undef to <256 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v256i1_v256i16 = sext <256 x i1> undef to <256 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %v256i1_v256i32 = sext <256 x i1> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %v256i1_v256i64 = sext <256 x i1> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i16 = sext <16 x i1> undef to <16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i32 = sext <16 x i1> undef to <16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i64 = sext <16 x i1> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i16 = sext <32 x i8> undef to <32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i32 = sext <32 x i8> undef to <32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i8_v32i64 = sext <32 x i8> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_v32i32 = sext <32 x i16> undef to <32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i16_v32i64 = sext <32 x i16> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i32_v32i64 = sext <32 x i32> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i8 = sext <32 x i1> undef to <32 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i16 = sext <32 x i1> undef to <32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i32 = sext <32 x i1> undef to <32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_v32i64 = sext <32 x i1> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_v64i16 = sext <64 x i8> undef to <64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i8_v64i32 = sext <64 x i8> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i8_v64i64 = sext <64 x i8> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i16_v64i32 = sext <64 x i16> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i16_v64i64 = sext <64 x i16> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v64i32_v64i64 = sext <64 x i32> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i8 = sext <64 x i1> undef to <64 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i16 = sext <64 x i1> undef to <64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v64i1_v64i32 = sext <64 x i1> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v64i1_v64i64 = sext <64 x i1> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v128i8_v128i16 = sext <128 x i8> undef to <128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v128i8_v128i32 = sext <128 x i8> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v128i8_v128i64 = sext <128 x i8> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v128i16_v128i32 = sext <128 x i16> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v128i16_v128i64 = sext <128 x i16> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v128i32_v128i64 = sext <128 x i32> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v128i1_v128i8 = sext <128 x i1> undef to <128 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v128i1_v128i16 = sext <128 x i1> undef to <128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v128i1_v128i32 = sext <128 x i1> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v128i1_v128i64 = sext <128 x i1> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v256i8_v256i16 = sext <256 x i8> undef to <256 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v256i8_v256i32 = sext <256 x i8> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %v256i8_v256i64 = sext <256 x i8> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v256i16_v256i32 = sext <256 x i16> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v256i16_v256i64 = sext <256 x i16> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v256i32_v256i64 = sext <256 x i32> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v256i1_v256i8 = sext <256 x i1> undef to <256 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v256i1_v256i16 = sext <256 x i1> undef to <256 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %v256i1_v256i32 = sext <256 x i1> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %v256i1_v256i64 = sext <256 x i1> undef to <256 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i16 = sext <vscale x 1 x i8> undef to <vscale x 1 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i32 = sext <vscale x 1 x i8> undef to <vscale x 1 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i64 = sext <vscale x 1 x i8> undef to <vscale x 1 x i64>
@@ -259,73 +259,73 @@ define void @sext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i1_nxv1i64 = sext <vscale x 1 x i1> undef to <vscale x 1 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i16 = sext <vscale x 2 x i8> undef to <vscale x 2 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i32 = sext <vscale x 2 x i8> undef to <vscale x 2 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i8_nxv2i64 = sext <vscale x 2 x i8> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i64 = sext <vscale x 2 x i8> undef to <vscale x 2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i32 = sext <vscale x 2 x i16> undef to <vscale x 2 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i16_nxv2i64 = sext <vscale x 2 x i16> undef to <vscale x 2 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i32_nxv2i64 = sext <vscale x 2 x i32> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i64 = sext <vscale x 2 x i16> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32_nxv2i64 = sext <vscale x 2 x i32> undef to <vscale x 2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i8 = sext <vscale x 2 x i1> undef to <vscale x 2 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i16 = sext <vscale x 2 x i1> undef to <vscale x 2 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i32 = sext <vscale x 2 x i1> undef to <vscale x 2 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_nxv2i64 = sext <vscale x 2 x i1> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i64 = sext <vscale x 2 x i1> undef to <vscale x 2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i16 = sext <vscale x 4 x i8> undef to <vscale x 4 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i8_nxv4i32 = sext <vscale x 4 x i8> undef to <vscale x 4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i8_nxv4i64 = sext <vscale x 4 x i8> undef to <vscale x 4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i16_nxv4i32 = sext <vscale x 4 x i16> undef to <vscale x 4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i16_nxv4i64 = sext <vscale x 4 x i16> undef to <vscale x 4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i32_nxv4i64 = sext <vscale x 4 x i32> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i32 = sext <vscale x 4 x i8> undef to <vscale x 4 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i64 = sext <vscale x 4 x i8> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i32 = sext <vscale x 4 x i16> undef to <vscale x 4 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i64 = sext <vscale x 4 x i16> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32_nxv4i64 = sext <vscale x 4 x i32> undef to <vscale x 4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i8 = sext <vscale x 4 x i1> undef to <vscale x 4 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i16 = sext <vscale x 4 x i1> undef to <vscale x 4 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_nxv4i32 = sext <vscale x 4 x i1> undef to <vscale x 4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv4i1_nxv4i64 = sext <vscale x 4 x i1> undef to <vscale x 4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i8_nxv8i16 = sext <vscale x 8 x i8> undef to <vscale x 8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i8_nxv8i32 = sext <vscale x 8 x i8> undef to <vscale x 8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i8_nxv8i64 = sext <vscale x 8 x i8> undef to <vscale x 8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i16_nxv8i32 = sext <vscale x 8 x i16> undef to <vscale x 8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i16_nxv8i64 = sext <vscale x 8 x i16> undef to <vscale x 8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i32_nxv8i64 = sext <vscale x 8 x i32> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i32 = sext <vscale x 4 x i1> undef to <vscale x 4 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i64 = sext <vscale x 4 x i1> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i16 = sext <vscale x 8 x i8> undef to <vscale x 8 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i32 = sext <vscale x 8 x i8> undef to <vscale x 8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i64 = sext <vscale x 8 x i8> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i32 = sext <vscale x 8 x i16> undef to <vscale x 8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i64 = sext <vscale x 8 x i16> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32_nxv8i64 = sext <vscale x 8 x i32> undef to <vscale x 8 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i8 = sext <vscale x 8 x i1> undef to <vscale x 8 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_nxv8i16 = sext <vscale x 8 x i1> undef to <vscale x 8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i1_nxv8i32 = sext <vscale x 8 x i1> undef to <vscale x 8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv8i1_nxv8i64 = sext <vscale x 8 x i1> undef to <vscale x 8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i8_nxv16i16 = sext <vscale x 16 x i8> undef to <vscale x 16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i8_nxv16i32 = sext <vscale x 16 x i8> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i8_nxv16i64 = sext <vscale x 16 x i8> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i16_nxv16i32 = sext <vscale x 16 x i16> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i16_nxv16i64 = sext <vscale x 16 x i16> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i32_nxv16i64 = sext <vscale x 16 x i32> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_nxv16i8 = sext <vscale x 16 x i1> undef to <vscale x 16 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_nxv16i16 = sext <vscale x 16 x i1> undef to <vscale x 16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16i32 = sext <vscale x 16 x i1> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16i64 = sext <vscale x 16 x i1> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i8_nxv32i16 = sext <vscale x 32 x i8> undef to <vscale x 32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i8_nxv32i32 = sext <vscale x 32 x i8> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i8_nxv32i64 = sext <vscale x 32 x i8> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i16_nxv32i32 = sext <vscale x 32 x i16> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i16_nxv32i64 = sext <vscale x 32 x i16> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32i64 = sext <vscale x 32 x i32> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i1_nxv32i8 = sext <vscale x 32 x i1> undef to <vscale x 32 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv32i1_nxv32i16 = sext <vscale x 32 x i1> undef to <vscale x 32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32i32 = sext <vscale x 32 x i1> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32i64 = sext <vscale x 32 x i1> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv64i8_nxv64i16 = sext <vscale x 64 x i8> undef to <vscale x 64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv64i8_nxv64i32 = sext <vscale x 64 x i8> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %nxv64i8_nxv64i64 = sext <vscale x 64 x i8> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv64i16_nxv64i32 = sext <vscale x 64 x i16> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv64i16_nxv64i64 = sext <vscale x 64 x i16> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv64i32_nxv64i64 = sext <vscale x 64 x i32> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv64i1_nxv64i8 = sext <vscale x 64 x i1> undef to <vscale x 64 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv64i1_nxv64i16 = sext <vscale x 64 x i1> undef to <vscale x 64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64i32 = sext <vscale x 64 x i1> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %nxv64i1_nxv64i64 = sext <vscale x 64 x i1> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv128i8_nxv128i16 = sext <vscale x 128 x i8> undef to <vscale x 128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv128i8_nxv128i32 = sext <vscale x 128 x i8> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i16 = sext <vscale x 8 x i1> undef to <vscale x 8 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i32 = sext <vscale x 8 x i1> undef to <vscale x 8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i64 = sext <vscale x 8 x i1> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i16 = sext <vscale x 16 x i8> undef to <vscale x 16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i32 = sext <vscale x 16 x i8> undef to <vscale x 16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i8_nxv16i64 = sext <vscale x 16 x i8> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16_nxv16i32 = sext <vscale x 16 x i16> undef to <vscale x 16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i16_nxv16i64 = sext <vscale x 16 x i16> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i32_nxv16i64 = sext <vscale x 16 x i32> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i8 = sext <vscale x 16 x i1> undef to <vscale x 16 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i16 = sext <vscale x 16 x i1> undef to <vscale x 16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i32 = sext <vscale x 16 x i1> undef to <vscale x 16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_nxv16i64 = sext <vscale x 16 x i1> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8_nxv32i16 = sext <vscale x 32 x i8> undef to <vscale x 32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i8_nxv32i32 = sext <vscale x 32 x i8> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i8_nxv32i64 = sext <vscale x 32 x i8> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i16_nxv32i32 = sext <vscale x 32 x i16> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i16_nxv32i64 = sext <vscale x 32 x i16> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i32_nxv32i64 = sext <vscale x 32 x i32> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i8 = sext <vscale x 32 x i1> undef to <vscale x 32 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i16 = sext <vscale x 32 x i1> undef to <vscale x 32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_nxv32i32 = sext <vscale x 32 x i1> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_nxv32i64 = sext <vscale x 32 x i1> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv64i8_nxv64i16 = sext <vscale x 64 x i8> undef to <vscale x 64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv64i8_nxv64i32 = sext <vscale x 64 x i8> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %nxv64i8_nxv64i64 = sext <vscale x 64 x i8> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv64i16_nxv64i32 = sext <vscale x 64 x i16> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %nxv64i16_nxv64i64 = sext <vscale x 64 x i16> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv64i32_nxv64i64 = sext <vscale x 64 x i32> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv64i1_nxv64i8 = sext <vscale x 64 x i1> undef to <vscale x 64 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv64i1_nxv64i16 = sext <vscale x 64 x i1> undef to <vscale x 64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv64i1_nxv64i32 = sext <vscale x 64 x i1> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %nxv64i1_nxv64i64 = sext <vscale x 64 x i1> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv128i8_nxv128i16 = sext <vscale x 128 x i8> undef to <vscale x 128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %nxv128i8_nxv128i32 = sext <vscale x 128 x i8> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i8_nxv128i128 = sext <vscale x 128 x i8> undef to <vscale x 128 x i128>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv128i16_nxv128i32 = sext <vscale x 128 x i16> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv128i16_nxv128i32 = sext <vscale x 128 x i16> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i16_nxv128i128 = sext <vscale x 128 x i16> undef to <vscale x 128 x i128>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i32_nxv128i128 = sext <vscale x 128 x i32> undef to <vscale x 128 x i128>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %nxv128i1_nxv128i8 = sext <vscale x 128 x i1> undef to <vscale x 128 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %nxv128i1_nxv128i16 = sext <vscale x 128 x i1> undef to <vscale x 128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %nxv128i1_nxv128i32 = sext <vscale x 128 x i1> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv128i1_nxv128i8 = sext <vscale x 128 x i1> undef to <vscale x 128 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv128i1_nxv128i16 = sext <vscale x 128 x i1> undef to <vscale x 128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %nxv128i1_nxv128i32 = sext <vscale x 128 x i1> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i1_nxv128i128 = sext <vscale x 128 x i1> undef to <vscale x 128 x i128>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -522,74 +522,74 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i1_v2i64 = zext <2 x i1> undef to <2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i16 = zext <4 x i8> undef to <4 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i32 = zext <4 x i8> undef to <4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_v4i64 = zext <4 x i8> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i64 = zext <4 x i8> undef to <4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i32 = zext <4 x i16> undef to <4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16_v4i64 = zext <4 x i16> undef to <4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32_v4i64 = zext <4 x i32> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i64 = zext <4 x i16> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_v4i64 = zext <4 x i32> undef to <4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i8 = zext <4 x i1> undef to <4 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i16 = zext <4 x i1> undef to <4 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i32 = zext <4 x i1> undef to <4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_v4i64 = zext <4 x i1> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i64 = zext <4 x i1> undef to <4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i16 = zext <8 x i8> undef to <8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_v8i32 = zext <8 x i8> undef to <8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_v8i64 = zext <8 x i8> undef to <8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16_v8i32 = zext <8 x i16> undef to <8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i16_v8i64 = zext <8 x i16> undef to <8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32_v8i64 = zext <8 x i32> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i32 = zext <8 x i8> undef to <8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i64 = zext <8 x i8> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i32 = zext <8 x i16> undef to <8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i64 = zext <8 x i16> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_v8i64 = zext <8 x i32> undef to <8 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i8 = zext <8 x i1> undef to <8 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i16 = zext <8 x i1> undef to <8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_v8i32 = zext <8 x i1> undef to <8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i1_v8i64 = zext <8 x i1> undef to <8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8_v16i16 = zext <16 x i8> undef to <16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i8_v16i32 = zext <16 x i8> undef to <16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_v16i64 = zext <16 x i8> undef to <16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16_v16i32 = zext <16 x i16> undef to <16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i16_v16i64 = zext <16 x i16> undef to <16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32_v16i64 = zext <16 x i32> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i32 = zext <8 x i1> undef to <8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i64 = zext <8 x i1> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i16 = zext <16 x i8> undef to <16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i32 = zext <16 x i8> undef to <16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i64 = zext <16 x i8> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i32 = zext <16 x i16> undef to <16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i64 = zext <16 x i16> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_v16i64 = zext <16 x i32> undef to <16 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i8 = zext <16 x i1> undef to <16 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_v16i16 = zext <16 x i1> undef to <16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_v16i32 = zext <16 x i1> undef to <16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i1_v16i64 = zext <16 x i1> undef to <16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i8_v32i16 = zext <32 x i8> undef to <32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i8_v32i32 = zext <32 x i8> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i8_v32i64 = zext <32 x i8> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i16_v32i32 = zext <32 x i16> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i16_v32i64 = zext <32 x i16> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i32_v32i64 = zext <32 x i32> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_v32i8 = zext <32 x i1> undef to <32 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_v32i16 = zext <32 x i1> undef to <32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32i32 = zext <32 x i1> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32i64 = zext <32 x i1> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i16 = zext <64 x i8> undef to <64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i8_v64i32 = zext <64 x i8> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i8_v64i64 = zext <64 x i8> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i16_v64i32 = zext <64 x i16> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i16_v64i64 = zext <64 x i16> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64i64 = zext <64 x i32> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i1_v64i8 = zext <64 x i1> undef to <64 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i1_v64i16 = zext <64 x i1> undef to <64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64i32 = zext <64 x i1> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64i64 = zext <64 x i1> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v128i8_v128i16 = zext <128 x i8> undef to <128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v128i8_v128i32 = zext <128 x i8> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %v128i8_v128i64 = zext <128 x i8> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v128i16_v128i32 = zext <128 x i16> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v128i16_v128i64 = zext <128 x i16> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128i64 = zext <128 x i32> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v128i1_v128i8 = zext <128 x i1> undef to <128 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v128i1_v128i16 = zext <128 x i1> undef to <128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128i32 = zext <128 x i1> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128i64 = zext <128 x i1> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v256i8_v256i16 = zext <256 x i8> undef to <256 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v256i8_v256i32 = zext <256 x i8> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %v256i8_v256i64 = zext <256 x i8> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v256i16_v256i32 = zext <256 x i16> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %v256i16_v256i64 = zext <256 x i16> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %v256i32_v256i64 = zext <256 x i32> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v256i1_v256i8 = zext <256 x i1> undef to <256 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v256i1_v256i16 = zext <256 x i1> undef to <256 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %v256i1_v256i32 = zext <256 x i1> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %v256i1_v256i64 = zext <256 x i1> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i16 = zext <16 x i1> undef to <16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i32 = zext <16 x i1> undef to <16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i64 = zext <16 x i1> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i16 = zext <32 x i8> undef to <32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i32 = zext <32 x i8> undef to <32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i8_v32i64 = zext <32 x i8> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_v32i32 = zext <32 x i16> undef to <32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i16_v32i64 = zext <32 x i16> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i32_v32i64 = zext <32 x i32> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i8 = zext <32 x i1> undef to <32 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i16 = zext <32 x i1> undef to <32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i32 = zext <32 x i1> undef to <32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_v32i64 = zext <32 x i1> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_v64i16 = zext <64 x i8> undef to <64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i8_v64i32 = zext <64 x i8> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i8_v64i64 = zext <64 x i8> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i16_v64i32 = zext <64 x i16> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i16_v64i64 = zext <64 x i16> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v64i32_v64i64 = zext <64 x i32> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i8 = zext <64 x i1> undef to <64 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i16 = zext <64 x i1> undef to <64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v64i1_v64i32 = zext <64 x i1> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v64i1_v64i64 = zext <64 x i1> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v128i8_v128i16 = zext <128 x i8> undef to <128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v128i8_v128i32 = zext <128 x i8> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v128i8_v128i64 = zext <128 x i8> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v128i16_v128i32 = zext <128 x i16> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v128i16_v128i64 = zext <128 x i16> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v128i32_v128i64 = zext <128 x i32> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v128i1_v128i8 = zext <128 x i1> undef to <128 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v128i1_v128i16 = zext <128 x i1> undef to <128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v128i1_v128i32 = zext <128 x i1> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v128i1_v128i64 = zext <128 x i1> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v256i8_v256i16 = zext <256 x i8> undef to <256 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v256i8_v256i32 = zext <256 x i8> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %v256i8_v256i64 = zext <256 x i8> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v256i16_v256i32 = zext <256 x i16> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v256i16_v256i64 = zext <256 x i16> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v256i32_v256i64 = zext <256 x i32> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v256i1_v256i8 = zext <256 x i1> undef to <256 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v256i1_v256i16 = zext <256 x i1> undef to <256 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %v256i1_v256i32 = zext <256 x i1> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %v256i1_v256i64 = zext <256 x i1> undef to <256 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i16 = zext <vscale x 1 x i8> undef to <vscale x 1 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i32 = zext <vscale x 1 x i8> undef to <vscale x 1 x i32>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i64 = zext <vscale x 1 x i8> undef to <vscale x 1 x i64>
@@ -602,73 +602,73 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i1_nxv1i64 = zext <vscale x 1 x i1> undef to <vscale x 1 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i16 = zext <vscale x 2 x i8> undef to <vscale x 2 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i32 = zext <vscale x 2 x i8> undef to <vscale x 2 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i8_nxv2i64 = zext <vscale x 2 x i8> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i64 = zext <vscale x 2 x i8> undef to <vscale x 2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i32 = zext <vscale x 2 x i16> undef to <vscale x 2 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i16_nxv2i64 = zext <vscale x 2 x i16> undef to <vscale x 2 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i32_nxv2i64 = zext <vscale x 2 x i32> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i64 = zext <vscale x 2 x i16> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32_nxv2i64 = zext <vscale x 2 x i32> undef to <vscale x 2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i8 = zext <vscale x 2 x i1> undef to <vscale x 2 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i16 = zext <vscale x 2 x i1> undef to <vscale x 2 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i32 = zext <vscale x 2 x i1> undef to <vscale x 2 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_nxv2i64 = zext <vscale x 2 x i1> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i64 = zext <vscale x 2 x i1> undef to <vscale x 2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i16 = zext <vscale x 4 x i8> undef to <vscale x 4 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i8_nxv4i32 = zext <vscale x 4 x i8> undef to <vscale x 4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i8_nxv4i64 = zext <vscale x 4 x i8> undef to <vscale x 4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i16_nxv4i32 = zext <vscale x 4 x i16> undef to <vscale x 4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i16_nxv4i64 = zext <vscale x 4 x i16> undef to <vscale x 4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i32_nxv4i64 = zext <vscale x 4 x i32> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i32 = zext <vscale x 4 x i8> undef to <vscale x 4 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i64 = zext <vscale x 4 x i8> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i32 = zext <vscale x 4 x i16> undef to <vscale x 4 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i64 = zext <vscale x 4 x i16> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32_nxv4i64 = zext <vscale x 4 x i32> undef to <vscale x 4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i8 = zext <vscale x 4 x i1> undef to <vscale x 4 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i16 = zext <vscale x 4 x i1> undef to <vscale x 4 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_nxv4i32 = zext <vscale x 4 x i1> undef to <vscale x 4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv4i1_nxv4i64 = zext <vscale x 4 x i1> undef to <vscale x 4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i8_nxv8i16 = zext <vscale x 8 x i8> undef to <vscale x 8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i8_nxv8i32 = zext <vscale x 8 x i8> undef to <vscale x 8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i8_nxv8i64 = zext <vscale x 8 x i8> undef to <vscale x 8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i16_nxv8i32 = zext <vscale x 8 x i16> undef to <vscale x 8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i16_nxv8i64 = zext <vscale x 8 x i16> undef to <vscale x 8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i32_nxv8i64 = zext <vscale x 8 x i32> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i32 = zext <vscale x 4 x i1> undef to <vscale x 4 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i64 = zext <vscale x 4 x i1> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i16 = zext <vscale x 8 x i8> undef to <vscale x 8 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i32 = zext <vscale x 8 x i8> undef to <vscale x 8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i64 = zext <vscale x 8 x i8> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i32 = zext <vscale x 8 x i16> undef to <vscale x 8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i64 = zext <vscale x 8 x i16> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32_nxv8i64 = zext <vscale x 8 x i32> undef to <vscale x 8 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i8 = zext <vscale x 8 x i1> undef to <vscale x 8 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_nxv8i16 = zext <vscale x 8 x i1> undef to <vscale x 8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i1_nxv8i32 = zext <vscale x 8 x i1> undef to <vscale x 8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv8i1_nxv8i64 = zext <vscale x 8 x i1> undef to <vscale x 8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i8_nxv16i16 = zext <vscale x 16 x i8> undef to <vscale x 16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i8_nxv16i32 = zext <vscale x 16 x i8> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i8_nxv16i64 = zext <vscale x 16 x i8> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i16_nxv16i32 = zext <vscale x 16 x i16> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i16_nxv16i64 = zext <vscale x 16 x i16> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i32_nxv16i64 = zext <vscale x 16 x i32> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_nxv16i8 = zext <vscale x 16 x i1> undef to <vscale x 16 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_nxv16i16 = zext <vscale x 16 x i1> undef to <vscale x 16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16i32 = zext <vscale x 16 x i1> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16i64 = zext <vscale x 16 x i1> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i8_nxv32i16 = zext <vscale x 32 x i8> undef to <vscale x 32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i8_nxv32i32 = zext <vscale x 32 x i8> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i8_nxv32i64 = zext <vscale x 32 x i8> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i16_nxv32i32 = zext <vscale x 32 x i16> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i16_nxv32i64 = zext <vscale x 32 x i16> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32i64 = zext <vscale x 32 x i32> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i1_nxv32i8 = zext <vscale x 32 x i1> undef to <vscale x 32 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv32i1_nxv32i16 = zext <vscale x 32 x i1> undef to <vscale x 32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32i32 = zext <vscale x 32 x i1> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32i64 = zext <vscale x 32 x i1> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv64i8_nxv64i16 = zext <vscale x 64 x i8> undef to <vscale x 64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv64i8_nxv64i32 = zext <vscale x 64 x i8> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i16 = zext <vscale x 8 x i1> undef to <vscale x 8 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i32 = zext <vscale x 8 x i1> undef to <vscale x 8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i64 = zext <vscale x 8 x i1> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i16 = zext <vscale x 16 x i8> undef to <vscale x 16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i32 = zext <vscale x 16 x i8> undef to <vscale x 16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i8_nxv16i64 = zext <vscale x 16 x i8> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16_nxv16i32 = zext <vscale x 16 x i16> undef to <vscale x 16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i16_nxv16i64 = zext <vscale x 16 x i16> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i32_nxv16i64 = zext <vscale x 16 x i32> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i8 = zext <vscale x 16 x i1> undef to <vscale x 16 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i16 = zext <vscale x 16 x i1> undef to <vscale x 16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i32 = zext <vscale x 16 x i1> undef to <vscale x 16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_nxv16i64 = zext <vscale x 16 x i1> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8_nxv32i16 = zext <vscale x 32 x i8> undef to <vscale x 32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i8_nxv32i32 = zext <vscale x 32 x i8> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i8_nxv32i64 = zext <vscale x 32 x i8> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i16_nxv32i32 = zext <vscale x 32 x i16> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i16_nxv32i64 = zext <vscale x 32 x i16> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i32_nxv32i64 = zext <vscale x 32 x i32> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i8 = zext <vscale x 32 x i1> undef to <vscale x 32 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i16 = zext <vscale x 32 x i1> undef to <vscale x 32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_nxv32i32 = zext <vscale x 32 x i1> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_nxv32i64 = zext <vscale x 32 x i1> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv64i8_nxv64i16 = zext <vscale x 64 x i8> undef to <vscale x 64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv64i8_nxv64i32 = zext <vscale x 64 x i8> undef to <vscale x 64 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i8_nxv64i64 = zext <vscale x 64 x i8> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv64i16_nxv64i32 = zext <vscale x 64 x i16> undef to <vscale x 64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %nxv64i16_nxv64i64 = zext <vscale x 64 x i16> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %nxv64i32_nxv64i64 = zext <vscale x 64 x i32> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv64i1_nxv64i8 = zext <vscale x 64 x i1> undef to <vscale x 64 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv64i1_nxv64i16 = zext <vscale x 64 x i1> undef to <vscale x 64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64i32 = zext <vscale x 64 x i1> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv64i16_nxv64i32 = zext <vscale x 64 x i16> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %nxv64i16_nxv64i64 = zext <vscale x 64 x i16> undef to <vscale x 64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv64i32_nxv64i64 = zext <vscale x 64 x i32> undef to <vscale x 64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv64i1_nxv64i8 = zext <vscale x 64 x i1> undef to <vscale x 64 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv64i1_nxv64i16 = zext <vscale x 64 x i1> undef to <vscale x 64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv64i1_nxv64i32 = zext <vscale x 64 x i1> undef to <vscale x 64 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i1_nxv64i64 = zext <vscale x 64 x i1> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv128i8_nxv128i16 = zext <vscale x 128 x i8> undef to <vscale x 128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv128i8_nxv128i32 = zext <vscale x 128 x i8> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv128i8_nxv128i16 = zext <vscale x 128 x i8> undef to <vscale x 128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %nxv128i8_nxv128i32 = zext <vscale x 128 x i8> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i8_nxv128i128 = zext <vscale x 128 x i8> undef to <vscale x 128 x i128>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv128i16_nxv128i32 = zext <vscale x 128 x i16> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv128i16_nxv128i32 = zext <vscale x 128 x i16> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i16_nxv128i128 = zext <vscale x 128 x i16> undef to <vscale x 128 x i128>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i32_nxv128i128 = zext <vscale x 128 x i32> undef to <vscale x 128 x i128>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %nxv128i1_nxv128i8 = zext <vscale x 128 x i1> undef to <vscale x 128 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %nxv128i1_nxv128i16 = zext <vscale x 128 x i1> undef to <vscale x 128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %nxv128i1_nxv128i32 = zext <vscale x 128 x i1> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv128i1_nxv128i8 = zext <vscale x 128 x i1> undef to <vscale x 128 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv128i1_nxv128i16 = zext <vscale x 128 x i1> undef to <vscale x 128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %nxv128i1_nxv128i32 = zext <vscale x 128 x i1> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i1_nxv128i128 = zext <vscale x 128 x i1> undef to <vscale x 128 x i128>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -685,74 +685,74 @@ define void @zext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i1_v2i64 = zext <2 x i1> undef to <2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i16 = zext <4 x i8> undef to <4 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i32 = zext <4 x i8> undef to <4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_v4i64 = zext <4 x i8> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i64 = zext <4 x i8> undef to <4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i32 = zext <4 x i16> undef to <4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16_v4i64 = zext <4 x i16> undef to <4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32_v4i64 = zext <4 x i32> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i64 = zext <4 x i16> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_v4i64 = zext <4 x i32> undef to <4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i8 = zext <4 x i1> undef to <4 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i16 = zext <4 x i1> undef to <4 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i32 = zext <4 x i1> undef to <4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_v4i64 = zext <4 x i1> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i64 = zext <4 x i1> undef to <4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i16 = zext <8 x i8> undef to <8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_v8i32 = zext <8 x i8> undef to <8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_v8i64 = zext <8 x i8> undef to <8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16_v8i32 = zext <8 x i16> undef to <8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i16_v8i64 = zext <8 x i16> undef to <8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32_v8i64 = zext <8 x i32> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i32 = zext <8 x i8> undef to <8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i64 = zext <8 x i8> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i32 = zext <8 x i16> undef to <8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i64 = zext <8 x i16> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_v8i64 = zext <8 x i32> undef to <8 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i8 = zext <8 x i1> undef to <8 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i16 = zext <8 x i1> undef to <8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_v8i32 = zext <8 x i1> undef to <8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i1_v8i64 = zext <8 x i1> undef to <8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8_v16i16 = zext <16 x i8> undef to <16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i8_v16i32 = zext <16 x i8> undef to <16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_v16i64 = zext <16 x i8> undef to <16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16_v16i32 = zext <16 x i16> undef to <16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i16_v16i64 = zext <16 x i16> undef to <16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32_v16i64 = zext <16 x i32> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i32 = zext <8 x i1> undef to <8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i64 = zext <8 x i1> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i16 = zext <16 x i8> undef to <16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i32 = zext <16 x i8> undef to <16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i64 = zext <16 x i8> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i32 = zext <16 x i16> undef to <16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i64 = zext <16 x i16> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_v16i64 = zext <16 x i32> undef to <16 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i8 = zext <16 x i1> undef to <16 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_v16i16 = zext <16 x i1> undef to <16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_v16i32 = zext <16 x i1> undef to <16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i1_v16i64 = zext <16 x i1> undef to <16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i8_v32i16 = zext <32 x i8> undef to <32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i8_v32i32 = zext <32 x i8> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i8_v32i64 = zext <32 x i8> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i16_v32i32 = zext <32 x i16> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i16_v32i64 = zext <32 x i16> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i32_v32i64 = zext <32 x i32> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_v32i8 = zext <32 x i1> undef to <32 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_v32i16 = zext <32 x i1> undef to <32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32i32 = zext <32 x i1> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32i64 = zext <32 x i1> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i16 = zext <64 x i8> undef to <64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i8_v64i32 = zext <64 x i8> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i8_v64i64 = zext <64 x i8> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i16_v64i32 = zext <64 x i16> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i16_v64i64 = zext <64 x i16> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64i64 = zext <64 x i32> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i1_v64i8 = zext <64 x i1> undef to <64 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i1_v64i16 = zext <64 x i1> undef to <64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64i32 = zext <64 x i1> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64i64 = zext <64 x i1> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v128i8_v128i16 = zext <128 x i8> undef to <128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v128i8_v128i32 = zext <128 x i8> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %v128i8_v128i64 = zext <128 x i8> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v128i16_v128i32 = zext <128 x i16> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v128i16_v128i64 = zext <128 x i16> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128i64 = zext <128 x i32> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v128i1_v128i8 = zext <128 x i1> undef to <128 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v128i1_v128i16 = zext <128 x i1> undef to <128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128i32 = zext <128 x i1> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128i64 = zext <128 x i1> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v256i8_v256i16 = zext <256 x i8> undef to <256 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v256i8_v256i32 = zext <256 x i8> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %v256i8_v256i64 = zext <256 x i8> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v256i16_v256i32 = zext <256 x i16> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %v256i16_v256i64 = zext <256 x i16> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %v256i32_v256i64 = zext <256 x i32> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v256i1_v256i8 = zext <256 x i1> undef to <256 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v256i1_v256i16 = zext <256 x i1> undef to <256 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %v256i1_v256i32 = zext <256 x i1> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %v256i1_v256i64 = zext <256 x i1> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i16 = zext <16 x i1> undef to <16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i32 = zext <16 x i1> undef to <16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i64 = zext <16 x i1> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i16 = zext <32 x i8> undef to <32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i32 = zext <32 x i8> undef to <32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i8_v32i64 = zext <32 x i8> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_v32i32 = zext <32 x i16> undef to <32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i16_v32i64 = zext <32 x i16> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i32_v32i64 = zext <32 x i32> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i8 = zext <32 x i1> undef to <32 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i16 = zext <32 x i1> undef to <32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i32 = zext <32 x i1> undef to <32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_v32i64 = zext <32 x i1> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_v64i16 = zext <64 x i8> undef to <64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i8_v64i32 = zext <64 x i8> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i8_v64i64 = zext <64 x i8> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i16_v64i32 = zext <64 x i16> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i16_v64i64 = zext <64 x i16> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v64i32_v64i64 = zext <64 x i32> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i8 = zext <64 x i1> undef to <64 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i16 = zext <64 x i1> undef to <64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v64i1_v64i32 = zext <64 x i1> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v64i1_v64i64 = zext <64 x i1> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v128i8_v128i16 = zext <128 x i8> undef to <128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v128i8_v128i32 = zext <128 x i8> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v128i8_v128i64 = zext <128 x i8> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v128i16_v128i32 = zext <128 x i16> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v128i16_v128i64 = zext <128 x i16> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v128i32_v128i64 = zext <128 x i32> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v128i1_v128i8 = zext <128 x i1> undef to <128 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v128i1_v128i16 = zext <128 x i1> undef to <128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v128i1_v128i32 = zext <128 x i1> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v128i1_v128i64 = zext <128 x i1> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v256i8_v256i16 = zext <256 x i8> undef to <256 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v256i8_v256i32 = zext <256 x i8> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %v256i8_v256i64 = zext <256 x i8> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v256i16_v256i32 = zext <256 x i16> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v256i16_v256i64 = zext <256 x i16> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v256i32_v256i64 = zext <256 x i32> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v256i1_v256i8 = zext <256 x i1> undef to <256 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v256i1_v256i16 = zext <256 x i1> undef to <256 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %v256i1_v256i32 = zext <256 x i1> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %v256i1_v256i64 = zext <256 x i1> undef to <256 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i16 = zext <vscale x 1 x i8> undef to <vscale x 1 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i32 = zext <vscale x 1 x i8> undef to <vscale x 1 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i64 = zext <vscale x 1 x i8> undef to <vscale x 1 x i64>
@@ -765,73 +765,73 @@ define void @zext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i1_nxv1i64 = zext <vscale x 1 x i1> undef to <vscale x 1 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i16 = zext <vscale x 2 x i8> undef to <vscale x 2 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i32 = zext <vscale x 2 x i8> undef to <vscale x 2 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i8_nxv2i64 = zext <vscale x 2 x i8> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i64 = zext <vscale x 2 x i8> undef to <vscale x 2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i32 = zext <vscale x 2 x i16> undef to <vscale x 2 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i16_nxv2i64 = zext <vscale x 2 x i16> undef to <vscale x 2 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i32_nxv2i64 = zext <vscale x 2 x i32> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i64 = zext <vscale x 2 x i16> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32_nxv2i64 = zext <vscale x 2 x i32> undef to <vscale x 2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i8 = zext <vscale x 2 x i1> undef to <vscale x 2 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i16 = zext <vscale x 2 x i1> undef to <vscale x 2 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i32 = zext <vscale x 2 x i1> undef to <vscale x 2 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_nxv2i64 = zext <vscale x 2 x i1> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i64 = zext <vscale x 2 x i1> undef to <vscale x 2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i16 = zext <vscale x 4 x i8> undef to <vscale x 4 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i8_nxv4i32 = zext <vscale x 4 x i8> undef to <vscale x 4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i8_nxv4i64 = zext <vscale x 4 x i8> undef to <vscale x 4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i16_nxv4i32 = zext <vscale x 4 x i16> undef to <vscale x 4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i16_nxv4i64 = zext <vscale x 4 x i16> undef to <vscale x 4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i32_nxv4i64 = zext <vscale x 4 x i32> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i32 = zext <vscale x 4 x i8> undef to <vscale x 4 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i64 = zext <vscale x 4 x i8> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i32 = zext <vscale x 4 x i16> undef to <vscale x 4 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i64 = zext <vscale x 4 x i16> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32_nxv4i64 = zext <vscale x 4 x i32> undef to <vscale x 4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i8 = zext <vscale x 4 x i1> undef to <vscale x 4 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i16 = zext <vscale x 4 x i1> undef to <vscale x 4 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_nxv4i32 = zext <vscale x 4 x i1> undef to <vscale x 4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv4i1_nxv4i64 = zext <vscale x 4 x i1> undef to <vscale x 4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i8_nxv8i16 = zext <vscale x 8 x i8> undef to <vscale x 8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i8_nxv8i32 = zext <vscale x 8 x i8> undef to <vscale x 8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i8_nxv8i64 = zext <vscale x 8 x i8> undef to <vscale x 8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i16_nxv8i32 = zext <vscale x 8 x i16> undef to <vscale x 8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i16_nxv8i64 = zext <vscale x 8 x i16> undef to <vscale x 8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i32_nxv8i64 = zext <vscale x 8 x i32> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i32 = zext <vscale x 4 x i1> undef to <vscale x 4 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i64 = zext <vscale x 4 x i1> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i16 = zext <vscale x 8 x i8> undef to <vscale x 8 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i32 = zext <vscale x 8 x i8> undef to <vscale x 8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i64 = zext <vscale x 8 x i8> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i32 = zext <vscale x 8 x i16> undef to <vscale x 8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i64 = zext <vscale x 8 x i16> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32_nxv8i64 = zext <vscale x 8 x i32> undef to <vscale x 8 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i8 = zext <vscale x 8 x i1> undef to <vscale x 8 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_nxv8i16 = zext <vscale x 8 x i1> undef to <vscale x 8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i1_nxv8i32 = zext <vscale x 8 x i1> undef to <vscale x 8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv8i1_nxv8i64 = zext <vscale x 8 x i1> undef to <vscale x 8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i8_nxv16i16 = zext <vscale x 16 x i8> undef to <vscale x 16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i8_nxv16i32 = zext <vscale x 16 x i8> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i8_nxv16i64 = zext <vscale x 16 x i8> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i16_nxv16i32 = zext <vscale x 16 x i16> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i16_nxv16i64 = zext <vscale x 16 x i16> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i32_nxv16i64 = zext <vscale x 16 x i32> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_nxv16i8 = zext <vscale x 16 x i1> undef to <vscale x 16 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_nxv16i16 = zext <vscale x 16 x i1> undef to <vscale x 16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16i32 = zext <vscale x 16 x i1> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16i64 = zext <vscale x 16 x i1> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i8_nxv32i16 = zext <vscale x 32 x i8> undef to <vscale x 32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i8_nxv32i32 = zext <vscale x 32 x i8> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i8_nxv32i64 = zext <vscale x 32 x i8> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i16_nxv32i32 = zext <vscale x 32 x i16> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i16_nxv32i64 = zext <vscale x 32 x i16> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32i64 = zext <vscale x 32 x i32> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i1_nxv32i8 = zext <vscale x 32 x i1> undef to <vscale x 32 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv32i1_nxv32i16 = zext <vscale x 32 x i1> undef to <vscale x 32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32i32 = zext <vscale x 32 x i1> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32i64 = zext <vscale x 32 x i1> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv64i8_nxv64i16 = zext <vscale x 64 x i8> undef to <vscale x 64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv64i8_nxv64i32 = zext <vscale x 64 x i8> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %nxv64i8_nxv64i64 = zext <vscale x 64 x i8> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv64i16_nxv64i32 = zext <vscale x 64 x i16> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv64i16_nxv64i64 = zext <vscale x 64 x i16> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv64i32_nxv64i64 = zext <vscale x 64 x i32> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv64i1_nxv64i8 = zext <vscale x 64 x i1> undef to <vscale x 64 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv64i1_nxv64i16 = zext <vscale x 64 x i1> undef to <vscale x 64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64i32 = zext <vscale x 64 x i1> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %nxv64i1_nxv64i64 = zext <vscale x 64 x i1> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv128i8_nxv128i16 = zext <vscale x 128 x i8> undef to <vscale x 128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv128i8_nxv128i32 = zext <vscale x 128 x i8> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i16 = zext <vscale x 8 x i1> undef to <vscale x 8 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i32 = zext <vscale x 8 x i1> undef to <vscale x 8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i64 = zext <vscale x 8 x i1> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i16 = zext <vscale x 16 x i8> undef to <vscale x 16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i32 = zext <vscale x 16 x i8> undef to <vscale x 16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i8_nxv16i64 = zext <vscale x 16 x i8> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16_nxv16i32 = zext <vscale x 16 x i16> undef to <vscale x 16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i16_nxv16i64 = zext <vscale x 16 x i16> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i32_nxv16i64 = zext <vscale x 16 x i32> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i8 = zext <vscale x 16 x i1> undef to <vscale x 16 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i16 = zext <vscale x 16 x i1> undef to <vscale x 16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i32 = zext <vscale x 16 x i1> undef to <vscale x 16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_nxv16i64 = zext <vscale x 16 x i1> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8_nxv32i16 = zext <vscale x 32 x i8> undef to <vscale x 32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i8_nxv32i32 = zext <vscale x 32 x i8> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i8_nxv32i64 = zext <vscale x 32 x i8> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i16_nxv32i32 = zext <vscale x 32 x i16> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i16_nxv32i64 = zext <vscale x 32 x i16> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i32_nxv32i64 = zext <vscale x 32 x i32> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i8 = zext <vscale x 32 x i1> undef to <vscale x 32 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i16 = zext <vscale x 32 x i1> undef to <vscale x 32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_nxv32i32 = zext <vscale x 32 x i1> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_nxv32i64 = zext <vscale x 32 x i1> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv64i8_nxv64i16 = zext <vscale x 64 x i8> undef to <vscale x 64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv64i8_nxv64i32 = zext <vscale x 64 x i8> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %nxv64i8_nxv64i64 = zext <vscale x 64 x i8> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv64i16_nxv64i32 = zext <vscale x 64 x i16> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %nxv64i16_nxv64i64 = zext <vscale x 64 x i16> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv64i32_nxv64i64 = zext <vscale x 64 x i32> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv64i1_nxv64i8 = zext <vscale x 64 x i1> undef to <vscale x 64 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv64i1_nxv64i16 = zext <vscale x 64 x i1> undef to <vscale x 64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv64i1_nxv64i32 = zext <vscale x 64 x i1> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %nxv64i1_nxv64i64 = zext <vscale x 64 x i1> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv128i8_nxv128i16 = zext <vscale x 128 x i8> undef to <vscale x 128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %nxv128i8_nxv128i32 = zext <vscale x 128 x i8> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i8_nxv128i128 = zext <vscale x 128 x i8> undef to <vscale x 128 x i128>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv128i16_nxv128i32 = zext <vscale x 128 x i16> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv128i16_nxv128i32 = zext <vscale x 128 x i16> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i16_nxv128i128 = zext <vscale x 128 x i16> undef to <vscale x 128 x i128>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i32_nxv128i128 = zext <vscale x 128 x i32> undef to <vscale x 128 x i128>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %nxv128i1_nxv128i8 = zext <vscale x 128 x i1> undef to <vscale x 128 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %nxv128i1_nxv128i16 = zext <vscale x 128 x i1> undef to <vscale x 128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %nxv128i1_nxv128i32 = zext <vscale x 128 x i1> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv128i1_nxv128i8 = zext <vscale x 128 x i1> undef to <vscale x 128 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv128i1_nxv128i16 = zext <vscale x 128 x i1> undef to <vscale x 128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %nxv128i1_nxv128i32 = zext <vscale x 128 x i1> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i1_nxv128i128 = zext <vscale x 128 x i1> undef to <vscale x 128 x i128>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll
index 30cb32ce4eaf..80efe912c869 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll
@@ -1141,7 +1141,7 @@ define signext i32 @vreduce_add_nxv4i32(<vscale x 4 x i32> %v) {
 
 define signext i32 @vwreduce_add_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vwreduce_add_nxv4i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e = sext <vscale x 4 x i16> %v to <vscale x 4 x i32>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = sext <vscale x 4 x i16> %v to <vscale x 4 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %e)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
@@ -1157,7 +1157,7 @@ define signext i32 @vwreduce_add_nxv4i16(<vscale x 4 x i16> %v) {
 
 define signext i32 @vwreduce_uadd_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vwreduce_uadd_nxv4i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e = zext <vscale x 4 x i16> %v to <vscale x 4 x i32>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = zext <vscale x 4 x i16> %v to <vscale x 4 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %e)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
@@ -1445,7 +1445,7 @@ define i64 @vreduce_add_nxv2i64(<vscale x 2 x i64> %v) {
 
 define i64 @vwreduce_add_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vwreduce_add_nxv2i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e = sext <vscale x 2 x i32> %v to <vscale x 2 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = sext <vscale x 2 x i32> %v to <vscale x 2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %e)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
@@ -1461,7 +1461,7 @@ define i64 @vwreduce_add_nxv2i32(<vscale x 2 x i32> %v) {
 
 define i64 @vwreduce_uadd_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vwreduce_uadd_nxv2i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e = zext <vscale x 2 x i32> %v to <vscale x 2 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = zext <vscale x 2 x i32> %v to <vscale x 2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %e)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
@@ -1597,7 +1597,7 @@ define i64 @vreduce_add_nxv4i64(<vscale x 4 x i64> %v) {
 
 define i64 @vwreduce_add_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vwreduce_add_nxv4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e = sext <vscale x 4 x i32> %v to <vscale x 4 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = sext <vscale x 4 x i32> %v to <vscale x 4 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> %e)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
@@ -1613,7 +1613,7 @@ define i64 @vwreduce_add_nxv4i32(<vscale x 4 x i32> %v) {
 
 define i64 @vwreduce_uadd_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vwreduce_uadd_nxv4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e = zext <vscale x 4 x i32> %v to <vscale x 4 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = zext <vscale x 4 x i32> %v to <vscale x 4 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> %e)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-extractelement.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-extractelement.ll
index aa7a90bece33..225bad6da591 100644
--- a/llvm/test/Analysis/CostModel/RISCV/rvv-extractelement.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/rvv-extractelement.ll
@@ -12,12 +12,12 @@ define void @extractelement_int(i32 %x) {
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i1_0 = extractelement <4 x i1> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i1_0 = extractelement <8 x i1> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i1_0 = extractelement <16 x i1> undef, i32 0
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i1_0 = extractelement <vscale x 2 x i1> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4i1_0 = extractelement <vscale x 4 x i1> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv8i1_0 = extractelement <vscale x 8 x i1> undef, i32 0
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = extractelement <2 x i8> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = extractelement <4 x i8> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = extractelement <8 x i8> undef, i32 0
@@ -66,12 +66,12 @@ define void @extractelement_int(i32 %x) {
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_1 = extractelement <4 x i1> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_1 = extractelement <8 x i1> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_1 = extractelement <16 x i1> undef, i32 1
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_1 = extractelement <vscale x 2 x i1> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_1 = extractelement <vscale x 4 x i1> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_1 = extractelement <vscale x 8 x i1> undef, i32 1
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = extractelement <2 x i8> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = extractelement <4 x i8> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = extractelement <8 x i8> undef, i32 1
@@ -120,12 +120,12 @@ define void @extractelement_int(i32 %x) {
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_x = extractelement <4 x i1> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_x = extractelement <8 x i1> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_x = extractelement <16 x i1> undef, i32 %x
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_x = extractelement <vscale x 2 x i1> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_x = extractelement <vscale x 4 x i1> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_x = extractelement <vscale x 8 x i1> undef, i32 %x
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_x = extractelement <2 x i8> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_x = extractelement <4 x i8> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_x = extractelement <8 x i8> undef, i32 %x
@@ -177,12 +177,12 @@ define void @extractelement_int(i32 %x) {
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i1_0 = extractelement <4 x i1> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i1_0 = extractelement <8 x i1> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i1_0 = extractelement <16 x i1> undef, i32 0
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i1_0 = extractelement <vscale x 2 x i1> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4i1_0 = extractelement <vscale x 4 x i1> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv8i1_0 = extractelement <vscale x 8 x i1> undef, i32 0
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = extractelement <2 x i8> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = extractelement <4 x i8> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = extractelement <8 x i8> undef, i32 0
@@ -231,12 +231,12 @@ define void @extractelement_int(i32 %x) {
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_1 = extractelement <4 x i1> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_1 = extractelement <8 x i1> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_1 = extractelement <16 x i1> undef, i32 1
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_1 = extractelement <vscale x 2 x i1> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_1 = extractelement <vscale x 4 x i1> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_1 = extractelement <vscale x 8 x i1> undef, i32 1
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = extractelement <2 x i8> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = extractelement <4 x i8> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = extractelement <8 x i8> undef, i32 1
@@ -285,12 +285,12 @@ define void @extractelement_int(i32 %x) {
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_x = extractelement <4 x i1> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_x = extractelement <8 x i1> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_x = extractelement <16 x i1> undef, i32 %x
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_x = extractelement <vscale x 2 x i1> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_x = extractelement <vscale x 4 x i1> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_x = extractelement <vscale x 8 x i1> undef, i32 %x
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_x = extractelement <2 x i8> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_x = extractelement <4 x i8> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_x = extractelement <8 x i8> undef, i32 %x
@@ -341,13 +341,13 @@ define void @extractelement_int(i32 %x) {
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i1_0 = extractelement <2 x i1> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i1_0 = extractelement <4 x i1> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i1_0 = extractelement <8 x i1> undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = extractelement <16 x i1> undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i1_0 = extractelement <16 x i1> undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i1_0 = extractelement <vscale x 2 x i1> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4i1_0 = extractelement <vscale x 4 x i1> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv8i1_0 = extractelement <vscale x 8 x i1> undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = extractelement <2 x i8> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = extractelement <4 x i8> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = extractelement <8 x i8> undef, i32 0
@@ -395,13 +395,13 @@ define void @extractelement_int(i32 %x) {
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i1_1 = extractelement <2 x i1> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_1 = extractelement <4 x i1> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_1 = extractelement <8 x i1> undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = extractelement <16 x i1> undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_1 = extractelement <16 x i1> undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_1 = extractelement <vscale x 2 x i1> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_1 = extractelement <vscale x 4 x i1> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_1 = extractelement <vscale x 8 x i1> undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = extractelement <2 x i8> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = extractelement <4 x i8> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = extractelement <8 x i8> undef, i32 1
@@ -449,13 +449,13 @@ define void @extractelement_int(i32 %x) {
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i1_x = extractelement <2 x i1> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_x = extractelement <4 x i1> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_x = extractelement <8 x i1> undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_x = extractelement <16 x i1> undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_x = extractelement <16 x i1> undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_x = extractelement <vscale x 2 x i1> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_x = extractelement <vscale x 4 x i1> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_x = extractelement <vscale x 8 x i1> undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_x = extractelement <2 x i8> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_x = extractelement <4 x i8> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_x = extractelement <8 x i8> undef, i32 %x
@@ -506,13 +506,13 @@ define void @extractelement_int(i32 %x) {
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i1_0 = extractelement <2 x i1> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i1_0 = extractelement <4 x i1> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i1_0 = extractelement <8 x i1> undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = extractelement <16 x i1> undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i1_0 = extractelement <16 x i1> undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i1_0 = extractelement <vscale x 2 x i1> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4i1_0 = extractelement <vscale x 4 x i1> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv8i1_0 = extractelement <vscale x 8 x i1> undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = extractelement <2 x i8> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = extractelement <4 x i8> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = extractelement <8 x i8> undef, i32 0
@@ -560,13 +560,13 @@ define void @extractelement_int(i32 %x) {
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i1_1 = extractelement <2 x i1> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_1 = extractelement <4 x i1> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_1 = extractelement <8 x i1> undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = extractelement <16 x i1> undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_1 = extractelement <16 x i1> undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_1 = extractelement <vscale x 2 x i1> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_1 = extractelement <vscale x 4 x i1> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_1 = extractelement <vscale x 8 x i1> undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = extractelement <2 x i8> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = extractelement <4 x i8> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = extractelement <8 x i8> undef, i32 1
@@ -614,13 +614,13 @@ define void @extractelement_int(i32 %x) {
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i1_x = extractelement <2 x i1> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_x = extractelement <4 x i1> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_x = extractelement <8 x i1> undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_x = extractelement <16 x i1> undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_x = extractelement <16 x i1> undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_x = extractelement <vscale x 2 x i1> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_x = extractelement <vscale x 4 x i1> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_x = extractelement <vscale x 8 x i1> undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_x = extractelement <2 x i8> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_x = extractelement <4 x i8> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_x = extractelement <8 x i8> undef, i32 %x
diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll
index 6e1ae0216f76..5387c8dc3594 100644
--- a/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll
@@ -12,12 +12,12 @@ define void @insertelement_int(i32 %x) {
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v4i1_0 = insertelement <4 x i1> undef, i1 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v8i1_0 = insertelement <8 x i1> undef, i1 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv2i1_0 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv4i1_0 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv8i1_0 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 0
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = insertelement <2 x i8> undef, i8 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = insertelement <4 x i8> undef, i8 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = insertelement <8 x i8> undef, i8 undef, i32 0
@@ -66,12 +66,12 @@ define void @insertelement_int(i32 %x) {
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v4i1_1 = insertelement <4 x i1> undef, i1 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i1_1 = insertelement <8 x i1> undef, i1 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv2i1_1 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i1_1 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8i1_1 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 1
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = insertelement <2 x i8> undef, i8 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = insertelement <4 x i8> undef, i8 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = insertelement <8 x i8> undef, i8 undef, i32 1
@@ -120,12 +120,12 @@ define void @insertelement_int(i32 %x) {
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i1_x = insertelement <4 x i1> undef, i1 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8i1_x = insertelement <8 x i1> undef, i1 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv2i1_x = insertelement <vscale x 2 x i1> undef, i1 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv4i1_x = insertelement <vscale x 4 x i1> undef, i1 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv8i1_x = insertelement <vscale x 8 x i1> undef, i1 undef, i32 %x
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i8_x = insertelement <2 x i8> undef, i8 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i8_x = insertelement <4 x i8> undef, i8 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i8_x = insertelement <8 x i8> undef, i8 undef, i32 %x
@@ -177,12 +177,12 @@ define void @insertelement_int(i32 %x) {
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v4i1_0 = insertelement <4 x i1> undef, i1 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v8i1_0 = insertelement <8 x i1> undef, i1 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv2i1_0 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv4i1_0 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv8i1_0 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 0
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = insertelement <2 x i8> undef, i8 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = insertelement <4 x i8> undef, i8 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = insertelement <8 x i8> undef, i8 undef, i32 0
@@ -231,12 +231,12 @@ define void @insertelement_int(i32 %x) {
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v4i1_1 = insertelement <4 x i1> undef, i1 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i1_1 = insertelement <8 x i1> undef, i1 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv2i1_1 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i1_1 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8i1_1 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 1
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = insertelement <2 x i8> undef, i8 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = insertelement <4 x i8> undef, i8 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = insertelement <8 x i8> undef, i8 undef, i32 1
@@ -285,12 +285,12 @@ define void @insertelement_int(i32 %x) {
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i1_x = insertelement <4 x i1> undef, i1 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8i1_x = insertelement <8 x i1> undef, i1 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv2i1_x = insertelement <vscale x 2 x i1> undef, i1 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv4i1_x = insertelement <vscale x 4 x i1> undef, i1 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv8i1_x = insertelement <vscale x 8 x i1> undef, i1 undef, i32 %x
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i8_x = insertelement <2 x i8> undef, i8 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i8_x = insertelement <4 x i8> undef, i8 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i8_x = insertelement <8 x i8> undef, i8 undef, i32 %x
@@ -341,13 +341,13 @@ define void @insertelement_int(i32 %x) {
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v2i1_0 = insertelement <2 x i1> undef, i1 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v4i1_0 = insertelement <4 x i1> undef, i1 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v8i1_0 = insertelement <8 x i1> undef, i1 undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv2i1_0 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv4i1_0 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv8i1_0 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = insertelement <2 x i8> undef, i8 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = insertelement <4 x i8> undef, i8 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = insertelement <8 x i8> undef, i8 undef, i32 0
@@ -395,13 +395,13 @@ define void @insertelement_int(i32 %x) {
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v2i1_1 = insertelement <2 x i1> undef, i1 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v4i1_1 = insertelement <4 x i1> undef, i1 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i1_1 = insertelement <8 x i1> undef, i1 undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv2i1_1 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i1_1 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8i1_1 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = insertelement <2 x i8> undef, i8 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = insertelement <4 x i8> undef, i8 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = insertelement <8 x i8> undef, i8 undef, i32 1
@@ -449,13 +449,13 @@ define void @insertelement_int(i32 %x) {
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2i1_x = insertelement <2 x i1> undef, i1 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i1_x = insertelement <4 x i1> undef, i1 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8i1_x = insertelement <8 x i1> undef, i1 undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv2i1_x = insertelement <vscale x 2 x i1> undef, i1 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv4i1_x = insertelement <vscale x 4 x i1> undef, i1 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv8i1_x = insertelement <vscale x 8 x i1> undef, i1 undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i8_x = insertelement <2 x i8> undef, i8 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i8_x = insertelement <4 x i8> undef, i8 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i8_x = insertelement <8 x i8> undef, i8 undef, i32 %x
@@ -506,13 +506,13 @@ define void @insertelement_int(i32 %x) {
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v2i1_0 = insertelement <2 x i1> undef, i1 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v4i1_0 = insertelement <4 x i1> undef, i1 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v8i1_0 = insertelement <8 x i1> undef, i1 undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv2i1_0 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv4i1_0 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv8i1_0 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = insertelement <2 x i8> undef, i8 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = insertelement <4 x i8> undef, i8 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = insertelement <8 x i8> undef, i8 undef, i32 0
@@ -560,13 +560,13 @@ define void @insertelement_int(i32 %x) {
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v2i1_1 = insertelement <2 x i1> undef, i1 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v4i1_1 = insertelement <4 x i1> undef, i1 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i1_1 = insertelement <8 x i1> undef, i1 undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv2i1_1 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i1_1 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8i1_1 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = insertelement <2 x i8> undef, i8 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = insertelement <4 x i8> undef, i8 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = insertelement <8 x i8> undef, i8 undef, i32 1
@@ -614,13 +614,13 @@ define void @insertelement_int(i32 %x) {
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2i1_x = insertelement <2 x i1> undef, i1 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i1_x = insertelement <4 x i1> undef, i1 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8i1_x = insertelement <8 x i1> undef, i1 undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv2i1_x = insertelement <vscale x 2 x i1> undef, i1 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv4i1_x = insertelement <vscale x 4 x i1> undef, i1 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv8i1_x = insertelement <vscale x 8 x i1> undef, i1 undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i8_x = insertelement <2 x i8> undef, i8 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i8_x = insertelement <4 x i8> undef, i8 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i8_x = insertelement <8 x i8> undef, i8 undef, i32 %x
diff --git a/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll b/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll
index b763198e98ba..46bf3152ac5b 100644
--- a/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll
@@ -197,7 +197,7 @@ define void  @broadcast_fixed() #0{
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %41 = shufflevector <32 x i1> undef, <32 x i1> undef, <32 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %42 = shufflevector <64 x i1> undef, <64 x i1> undef, <64 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %43 = shufflevector <128 x i1> undef, <128 x i1> undef, <128 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %ins1 = insertelement <128 x i1> poison, i1 poison, i32 0
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %ins1 = insertelement <128 x i1> poison, i1 poison, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %44 = shufflevector <128 x i1> %ins1, <128 x i1> poison, <128 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ins2 = insertelement <2 x i8> poison, i8 3, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %45 = shufflevector <2 x i8> %ins2, <2 x i8> undef, <2 x i32> zeroinitializer
-- 
GitLab


From f1dff836593d4601e3ad78117df1d980d284bb9c Mon Sep 17 00:00:00 2001
From: Christian Sigg <chsigg@users.noreply.github.com>
Date: Wed, 27 Mar 2024 06:07:58 +0100
Subject: [PATCH 377/404] [mlir][bazel] Expose GPUCommonPass.h only by a single
 target. (#86730)

Move `GPUOpsLowering.cpp` from `//mlir:GPUCommonTransforms` to
`//mlir:GPUToGPURuntimeTransforms` to match the CMake setup.

Ideally, header files should be used by only one target, but this is
hard because CMake is less strict with headers (no layering check). But
even with bazel, headers should only be exported once in the `hdrs`
attribute. Other targets may use them in the `srcs` attribute to avoid
circular dependencies.
---
 utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 5dcaf0265c92..73556060983b 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -3567,6 +3567,7 @@ cc_library(
         ":GPUCommonTransforms",
         ":GPUCompilationAttrInterfacesIncGen",
         ":GPUDialect",
+        ":GPUToGPURuntimeTransforms",
         ":IR",
         ":LLVMCommonConversion",
         ":LinalgDialect",
@@ -5603,6 +5604,7 @@ cc_library(
         ":FuncToLLVM",
         ":GPUCommonTransforms",
         ":GPUDialect",
+        ":GPUToGPURuntimeTransforms",
         ":GPUToNVVMTransforms",
         ":GPUTransforms",
         ":IndexToLLVM",
@@ -5727,6 +5729,7 @@ cc_library(
         ":FuncDialect",
         ":GPUCommonTransforms",
         ":GPUDialect",
+        ":GPUToGPURuntimeTransforms",
         ":GPUToNVVMTransforms",
         ":GPUTransformOpsIncGen",
         ":GPUTransforms",
@@ -5774,11 +5777,7 @@ td_library(
 
 cc_library(
     name = "GPUCommonTransforms",
-    srcs = [
-        "lib/Conversion/GPUCommon/GPUOpsLowering.cpp",
-    ],
     hdrs = [
-        "include/mlir/Conversion/GPUCommon/GPUCommonPass.h",
         "lib/Conversion/GPUCommon/GPUOpsLowering.h",
         "lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h",
         "lib/Conversion/GPUCommon/OpToFuncCallLowering.h",
@@ -5832,6 +5831,7 @@ cc_library(
         ":FuncToLLVM",
         ":GPUCommonTransforms",
         ":GPUDialect",
+        ":GPUToGPURuntimeTransforms",
         ":GPUToNVVMGen",
         ":GPUTransforms",
         ":IR",
@@ -5958,6 +5958,7 @@ cc_library(
         ":FuncToLLVM",
         ":GPUCommonTransforms",
         ":GPUDialect",
+        ":GPUToGPURuntimeTransforms",
         ":GPUToROCDLTGen",
         ":GPUTransforms",
         ":IR",
@@ -6001,6 +6002,7 @@ cc_library(
 cc_library(
     name = "GPUToGPURuntimeTransforms",
     srcs = [
+        "lib/Conversion/GPUCommon/GPUOpsLowering.cpp",
         "lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp",
     ],
     hdrs = ["include/mlir/Conversion/GPUCommon/GPUCommonPass.h"],
@@ -6014,6 +6016,7 @@ cc_library(
         ":ConvertToLLVM",
         ":ConvertToLLVMInterface",
         ":FuncToLLVM",
+        ":GPUCommonTransforms",
         ":GPUDialect",
         ":GPUTransforms",
         ":IR",
-- 
GitLab


From 9c0c98ed376f8705d0856f29a7ec659120187612 Mon Sep 17 00:00:00 2001
From: Carlos Alberto Enciso <carlos.alberto.enciso@gmail.com>
Date: Wed, 27 Mar 2024 05:27:44 +0000
Subject: [PATCH 378/404] [llvm-debuginfo-analyzer][DOC] Convert 'README.txt'
 to markdown. (#86394)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

As part of the WebAssembly support work
https://github.com/llvm/llvm-project/pull/85566

The README.txt is a bit odd since it only lists issues and problems
without talking about what works. It’s also hard to read on the GitHub
web view.

- Convert to Markdown and linking to the command docs
https://llvm.org/docs/CommandGuide/llvm-debuginfo-analyzer
- Rename some left 'elf reader' to 'DWARF reader'.
---
 .../CommandGuide/llvm-debuginfo-analyzer.rst  |   4 +
 .../LogicalView/Readers/LVCodeViewReader.h    |   2 +-
 .../LogicalView/Readers/LVDWARFReader.h       |   2 +-
 .../LogicalView/Readers/LVDWARFReader.cpp     |   4 +-
 llvm/tools/llvm-debuginfo-analyzer/README.md  | 170 ++++++++++++++
 llvm/tools/llvm-debuginfo-analyzer/README.txt | 221 ------------------
 6 files changed, 178 insertions(+), 225 deletions(-)
 create mode 100644 llvm/tools/llvm-debuginfo-analyzer/README.md
 delete mode 100644 llvm/tools/llvm-debuginfo-analyzer/README.txt

diff --git a/llvm/docs/CommandGuide/llvm-debuginfo-analyzer.rst b/llvm/docs/CommandGuide/llvm-debuginfo-analyzer.rst
index 54622cc61fdf..5b3200a4b782 100644
--- a/llvm/docs/CommandGuide/llvm-debuginfo-analyzer.rst
+++ b/llvm/docs/CommandGuide/llvm-debuginfo-analyzer.rst
@@ -2267,6 +2267,10 @@ EXIT STATUS
 :program:`llvm-debuginfo-analyzer` returns 0 if the input files were
 parsed and printed successfully. Otherwise, it returns 1.
 
+LIMITATIONS AND KNOWN ISSUES
+----------------------------
+See :download:`Limitations <../../tools/llvm-debuginfo-analyzer/README.md>`.
+
 SEE ALSO
 --------
 :manpage:`llvm-dwarfdump`
diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVCodeViewReader.h b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVCodeViewReader.h
index 8a32210bac3c..4dd7c967ddc1 100644
--- a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVCodeViewReader.h
+++ b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVCodeViewReader.h
@@ -58,7 +58,7 @@ class LVSymbolVisitorDelegate;
 
 using LVNames = SmallVector<StringRef, 16>;
 
-// The ELF reader uses the DWARF constants to create the logical elements.
+// The DWARF reader uses the DWARF constants to create the logical elements.
 // The DW_TAG_* and DW_AT_* are used to select the logical object and to
 // set specific attributes, such as name, type, etc.
 // As the CodeView constants are different to the DWARF constants, the
diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h
index 22e804a459f8..fdc97249d8e5 100644
--- a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h
+++ b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h
@@ -49,7 +49,7 @@ class LVDWARFReader final : public LVBinaryReader {
 
   // In DWARF v4, the files are 1-indexed.
   // In DWARF v5, the files are 0-indexed.
-  // The ELF reader expects the indexes as 1-indexed.
+  // The DWARF reader expects the indexes as 1-indexed.
   bool IncrementFileIndex = false;
 
   // Address ranges collected for current DIE.
diff --git a/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp b/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp
index 91e5a037054d..6a97bed9e3a8 100644
--- a/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp
@@ -878,7 +878,7 @@ Error LVDWARFReader::createScopes() {
     // Additional discussions here:
     // https://www.mail-archive.com/dwarf-discuss@lists.dwarfstd.org/msg00883.html
 
-    // The ELF Reader is expecting the files are 1-indexed, so using
+    // The DWARF reader is expecting the files are 1-indexed, so using
     // the .debug_line header information decide if the indexed require
     // an internal adjustment.
 
@@ -918,7 +918,7 @@ Error LVDWARFReader::createScopes() {
       // DWARF-5 -> Increment index.
       return true;
     };
-    // The ELF reader expects the indexes as 1-indexed.
+    // The DWARF reader expects the indexes as 1-indexed.
     IncrementFileIndex = DeduceIncrementFileIndex();
 
     DWARFDie UnitDie = CU->getUnitDIE();
diff --git a/llvm/tools/llvm-debuginfo-analyzer/README.md b/llvm/tools/llvm-debuginfo-analyzer/README.md
new file mode 100644
index 000000000000..28c4be6f7322
--- /dev/null
+++ b/llvm/tools/llvm-debuginfo-analyzer/README.md
@@ -0,0 +1,170 @@
+# `llvm-debuginfo-analyzer`
+
+These are the notes collected during the development, review and test.
+They describe limitations, known issues and future work.
+
+### Remove the use of macros in ``LVReader.h`` that describe the ``bumpallocators``.
+**[D137933](https://reviews.llvm.org/D137933#inline-1389904)**
+
+Use a standard (or LLVM) ``map`` with ``typeinfo`` (would need a specialization
+to expose equality and hasher) for the allocators and the creation
+functions could be a function template.
+
+### Use a **lit test** instead of a **unit test** for the **logical readers**.
+**[D125783](https://reviews.llvm.org/D125783#inline-1324376)**
+
+As the ``DebugInfoLogicalView`` library is sufficiently exposed via the
+``llvm-debuginfo-analyzer`` tool, follow the LLVM general approach and
+use ``lit`` tests to validate the **logical readers**.
+
+Convert the ``unitests``:
+```
+llvm-project/llvm/unittests/DebugInfo/LogicalView/CodeViewReaderTest.cpp
+llvm-project/llvm/unittests/DebugInfo/LogicalView/DWARFReaderTest.cpp
+```
+into ``lit`` tests:
+```
+llvm-project/llvm/test/DebugInfo/LogicalView/CodeViewReader.test
+llvm-project/llvm/test/DebugInfo/LogicalView/DWARFReader.test
+```
+
+### Eliminate calls to ``getInputFileDirectory()`` in the ``unittests``.
+**[D125783](https://reviews.llvm.org/D125783#inline-1324359)**
+
+Rewrite the unittests ``ReaderTest`` and ``CodeViewReaderTest`` to eliminate
+the call:
+```
+  getInputFileDirectory()
+```
+as use of that call is discouraged.
+
+### Fix mismatch between ``%d/%x`` format strings and ``uint64_t`` type.
+**[D137400](https://reviews.llvm.org/D137400) / [58758](https://github.com/llvm/llvm-project/issues/58758)**
+
+Incorrect printing of ``uint64_t`` on ``32-bit`` platforms.
+Add the ``PRIx64`` specifier to the printing code (``format()``).
+
+### Remove ``LVScope::Children`` container.
+**[D137933](https://reviews.llvm.org/D137933#inline-1373902)**
+
+Use a **chaining iterator** over the other containers rather than keep a
+separate container ``Children`` that mirrors their contents.
+
+### Use ``TableGen`` for command line options.
+**[D125777](https://reviews.llvm.org/D125777#inline-1291801)**
+
+The current trend is to use ``TableGen`` for command-line options in tools.
+Change command line options to use ``tablegen`` as many other LLVM tools.
+
+### ``LVDoubleMap`` to return ``optional<ValueType>`` instead of ``null pointer``.
+**[D125783](https://reviews.llvm.org/D125783#inline-1294164)**
+
+The more idiomatic LLVM way to handle this would be to have ``find``
+return ``Optional<ValueType>``.
+
+### Pass references instead of pointers (**Comparison functions**).
+**[D125782](https://reviews.llvm.org/D125782#inline-1293920)**
+
+In the **comparison functions**, pass references instead of pointers (when
+pointers cannot be null).
+
+### Use ``StringMap`` where possible.
+**[D125783](https://reviews.llvm.org/D125783#inline-1294211)**
+
+LLVM has a ``StringMap`` class that is advertised as more efficient than
+``std::map<std::string, ValueType>``. Mainly it does fewer allocations
+because the key is not a ``std::string``.
+
+Replace the use of ``std::map<std::string, ValueType>`` with ``StringMap``.
+One specific case is the ``LVSymbolNames`` definitions.
+
+### Calculate unique offset for CodeView elements.
+In order to have the same logical functionality as the DWARF reader, such
+as:
+
+* find scopes contribution to debug info
+* sort by its physical location
+
+The logical elements must have an unique offset (similar like the DWARF
+``DIE`` offset).
+
+### Move ``initializeFileAndStringTables`` to the CodeView Library.
+There is some code in the CodeView reader that was extracted/adapted
+from ``tools/llvm-readobj/COFFDumper.cpp`` that can be moved to the CodeView
+library.
+
+We had a similar case with code shared with ``llvm-pdbutil`` that was moved
+to the PDB library: **[D122226](https://reviews.llvm.org/D122226)**
+
+### Move ``getSymbolKindName`` and ``formatRegisterId`` to the CodeView Library.
+There is some code in the CodeView reader that was extracted/adapted
+from ``lib/DebugInfo/CodeView/SymbolDumper.cpp`` that can be used.
+
+### Use of ``std::unordered_set`` instead of ``std::set``.
+**[D125784](https://reviews.llvm.org/D125784#inline-1221421)**
+
+Replace the ``std::set`` usage for ``DeducedScopes``, ``UnresolvedScopes`` and
+``IdentifiedNamespaces`` with ``std::unordered_set`` and get the benefit
+of the O(1) while inserting/searching, as the order is not important.
+
+### Optimize ``LVNamespaceDeduction::find`` funtion.
+**[D125784](https://reviews.llvm.org/D125784#inline-1296195)**
+
+Optimize the ``find`` method to use the proposed code:
+
+```
+  LVStringRefs::iterator Iter = std::find_if(Components.begin(), Components.end(),
+    [](StringRef Name) {
+        return IdentifiedNamespaces.find(Name) == IdentifiedNamespaces.end();
+    });
+  LVStringRefs::size_type FirstNonNamespace = std::distance(Components.begin(), Iter);
+```
+
+### Move all the printing support to a common module.
+Factor out printing functionality from the logical elements into a
+common module.
+
+### Refactor ``LVBinaryReader::processLines``.
+**[D125783](https://reviews.llvm.org/D125783#inline-1246155) /
+[D137156](https://reviews.llvm.org/D137156)**
+
+During the traversal of the debug information sections, we created the
+logical lines representing the **disassembled instructions** from the **text
+section** and the logical lines representing the **line records** from the
+**debug line** section. Using the ranges associated with the logical scopes,
+we will allocate those logical lines to their logical scopes.
+
+Consider the case when any of those lines become orphans, causing
+incorrect scope parent for disassembly or line records.
+
+### Add support for ``-ffunction-sections``.
+**[D125783](https://reviews.llvm.org/D125783#inline-1295012)**
+
+Only linked executables are handled. It does not support relocatable
+files compiled with ``-ffunction-sections``.
+
+### Add support for DWARF v5 `.debug_names` section / CodeView public symbols stream.
+**[D125783](https://reviews.llvm.org/D125783#inline-1294142)**
+
+The DWARF and CodeView readers use the public names information to create
+the instructions (``LVLineAssembler``). Instead of relying on DWARF section
+names (``.debug_pubnames``, ``.debug_names``) and CodeView public symbol stream
+(``S_PUB32``), the readers should collect the needed information while processing
+the debug information.
+
+If the object file supports the above section names and stream, use them
+to create the public names.
+
+### Add support for some extra DWARF locations.
+The following DWARF debug location operands are not supported:
+
+* `DW_OP_const_type`
+* `DW_OP_entry_value`
+* `DW_OP_implicit_value`
+
+### Add support for additional binary formats.
+* Extended COFF (`XCOFF`)
+
+### Add support for ``JSON`` or ``YAML``
+The logical view uses its own and non-standard free form text when
+displaying information on logical elements.
diff --git a/llvm/tools/llvm-debuginfo-analyzer/README.txt b/llvm/tools/llvm-debuginfo-analyzer/README.txt
deleted file mode 100644
index ce7569d27224..000000000000
--- a/llvm/tools/llvm-debuginfo-analyzer/README.txt
+++ /dev/null
@@ -1,221 +0,0 @@
-//===- llvm/tools/llvm-debuginfo-analyzer/README.txt ----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains notes collected during the development, review and test.
-// It describes limitations, know issues and future work.
-//
-//===----------------------------------------------------------------------===//
-
-//===----------------------------------------------------------------------===//
-// Remove the use of macros in 'LVReader.h' that describe the bumpallocators.
-//===----------------------------------------------------------------------===//
-https://reviews.llvm.org/D137933#inline-1389904
-
-Use a standard (or LLVM) map with typeinfo (would need a specialization
-to expose equality and hasher) for the allocators and the creation
-functions could be a function template.
-
-//===----------------------------------------------------------------------===//
-// Use a lit test instead of a unit test for the logical readers.
-//===----------------------------------------------------------------------===//
-https://reviews.llvm.org/D125783#inline-1324376
-
-As the DebugInfoLogicalView library is sufficiently exposed via the
-llvm-debuginfo-analyzer tool, follow the LLVM general approach and
-use LIT tests to validate the logical readers.
-
-Convert the unitests:
-  llvm-project/llvm/unittests/DebugInfo/LogicalView/CodeViewReaderTest.cpp
-  llvm-project/llvm/unittests/DebugInfo/LogicalView/DWARFReaderTest.cpp
-
-into LIT tests:
-  llvm-project/llvm/test/DebugInfo/LogicalView/CodeViewReader.test
-  llvm-project/llvm/test/DebugInfo/LogicalView/DWARFReader.test
-
-//===----------------------------------------------------------------------===//
-// Eliminate calls to 'getInputFileDirectory()' in the unit tests.
-//===----------------------------------------------------------------------===//
-https://reviews.llvm.org/D125783#inline-1324359
-
-Rewrite the unittests 'LFReaderTest' and 'CodeViewReaderTest'to eliminate
-the call:
-
-  getInputFileDirectory()
-
-as use of that call is discouraged.
-
-See: Use a lit test instead of a unit test for the logical readers.
-
-//===----------------------------------------------------------------------===//
-// Fix mismatch between %d/%x format strings and uint64_t type.
-//===----------------------------------------------------------------------===//
-https://reviews.llvm.org/D137400
-https://github.com/llvm/llvm-project/issues/58758
-
-Incorrect printing of uint64_t on 32-bit platforms.
-Add the PRIx64 specifier to the printing code (format()).
-
-//===----------------------------------------------------------------------===//
-// Remove 'LVScope::Children' container.
-//===----------------------------------------------------------------------===//
-https://reviews.llvm.org/D137933#inline-1373902
-
-Use a chaining iterator over the other containers rather than keep a
-separate container 'Children' that mirrors their contents.
-
-//===----------------------------------------------------------------------===//
-// Use TableGen for command line options.
-//===----------------------------------------------------------------------===//
-https://reviews.llvm.org/D125777#inline-1291801
-
-The current trend is to use TableGen for command-line options in tools.
-Change command line options to use tablegen as many other LLVM tools.
-
-//===----------------------------------------------------------------------===//
-// LVDoubleMap to return optional<ValueType> instead of null pointer.
-//===----------------------------------------------------------------------===//
-https://reviews.llvm.org/D125783#inline-1294164
-
-The more idiomatic LLVM way to handle this would be to have 'find '
-return Optional<ValueType>.
-
-//===----------------------------------------------------------------------===//
-// Pass references instead of pointers (Comparison functions).
-//===----------------------------------------------------------------------===//
-https://reviews.llvm.org/D125782#inline-1293920
-
-In the comparison functions, pass references instead of pointers (when
-pointers cannot be null).
-
-//===----------------------------------------------------------------------===//
-// Use StringMap where possible.
-//===----------------------------------------------------------------------===//
-https://reviews.llvm.org/D125783#inline-1294211
-
-LLVM has a StringMap class that is advertised as more efficient than
-std::map<std::string, ValueType>. Mainly it does fewer allocations
-because the key is not a std::string.
-
-Replace the use of std::map<std::string, ValueType> with String Map.
-One specific case is the LVSymbolNames definitions.
-
-//===----------------------------------------------------------------------===//
-// Calculate unique offset for CodeView elements.
-//===----------------------------------------------------------------------===//
-In order to have the same logical functionality as the ELF Reader, such
-as:
-
-- find scopes contribution to debug info
-- sort by its physical location
-
-The logical elements must have an unique offset (similar like the DWARF
-DIE offset).
-
-//===----------------------------------------------------------------------===//
-// Move 'initializeFileAndStringTables' to the COFF Library.
-//===----------------------------------------------------------------------===//
-There is some code in the CodeView reader that was extracted/adapted
-from 'tools/llvm-readobj/COFFDumper.cpp' that can be moved to the COFF
-library.
-
-We had a similar case with code shared with llvm-pdbutil that was moved
-to the PDB library: https://reviews.llvm.org/D122226
-
-//===----------------------------------------------------------------------===//
-// Move 'getSymbolKindName'/'formatRegisterId' to the CodeView Library.
-//===----------------------------------------------------------------------===//
-There is some code in the CodeView reader that was extracted/adapted
-from 'lib/DebugInfo/CodeView/SymbolDumper.cpp' that can be used.
-
-//===----------------------------------------------------------------------===//
-// Use of std::unordered_set instead of std::set.
-//===----------------------------------------------------------------------===//
-https://reviews.llvm.org/D125784#inline-1221421
-
-Replace the std::set usage for DeducedScopes, UnresolvedScopes and
-IdentifiedNamespaces with std::unordered_set and get the benefit
-of the O(1) while inserting/searching, as the order is not important.
-
-//===----------------------------------------------------------------------===//
-// Optimize 'LVNamespaceDeduction::find' funtion.
-//===----------------------------------------------------------------------===//
-https://reviews.llvm.org/D125784#inline-1296195
-
-Optimize the 'find' method to use the proposed code:
-
-  LVStringRefs::iterator Iter = std::find_if(Components.begin(), Components.end(),
-    [](StringRef Name) {
-        return IdentifiedNamespaces.find(Name) == IdentifiedNamespaces.end();
-    });
-  LVStringRefs::size_type FirstNonNamespace = std::distance(Components.begin(), Iter);
-
-//===----------------------------------------------------------------------===//
-// Move all the printing support to a common module.
-//===----------------------------------------------------------------------===//
-Factor out printing functionality from the logical elements into a
-common module.
-
-//===----------------------------------------------------------------------===//
-// Refactor 'LVBinaryReader::processLines'.
-//===----------------------------------------------------------------------===//
-https://reviews.llvm.org/D125783#inline-1246155
-https://reviews.llvm.org/D137156
-
-During the traversal of the debug information sections, we created the
-logical lines representing the disassembled instructions from the text
-section and the logical lines representing the line records from the
-debug line section. Using the ranges associated with the logical scopes,
-we will allocate those logical lines to their logical scopes.
-
-Consider the case when any of those lines become orphans, causing
-incorrect scope parent for disassembly or line records.
-
-//===----------------------------------------------------------------------===//
-// Add support for '-ffunction-sections'.
-//===----------------------------------------------------------------------===//
-https://reviews.llvm.org/D125783#inline-1295012
-
-Only linked executables are handled. It does not support relocatable
-files compiled with -ffunction-sections.
-
-//===----------------------------------------------------------------------===//
-// Add support for DWARF v5 .debug_names section.
-// Add support for CodeView public symbols stream.
-//===----------------------------------------------------------------------===//
-https://reviews.llvm.org/D125783#inline-1294142
-
-The ELF and CodeView readers use the public names information to create
-the instructions (LVLineAssembler). Instead of relying on DWARF section
-names (.debug_pubnames, .debug_names) and CodeView public symbol stream
-(S_PUB32), the readers collects the needed information while processing
-the debug information.
-
-If the object file supports the above section names and stream, use them
-to create the public names.
-
-//===----------------------------------------------------------------------===//
-// Add support for some extra DWARF locations.
-//===----------------------------------------------------------------------===//
-The following DWARF debug location operands are not supported:
-
-- DW_OP_const_type
-- DW_OP_entry_value
-- DW_OP_implicit_value
-
-//===----------------------------------------------------------------------===//
-// Add support for additional binary formats.
-//===----------------------------------------------------------------------===//
-- Extended COFF (XCOFF)
-
-//===----------------------------------------------------------------------===//
-// Add support for JSON or YAML.
-//===----------------------------------------------------------------------===//
-The logical view uses its own and non-standard free form text when
-displaying information on logical elements.
-
-//===----------------------------------------------------------------------===//
-- 
GitLab


From 22bfc58cd08cd58c8866ef79f3529d314dcb77e5 Mon Sep 17 00:00:00 2001
From: Yeting Kuo <46629943+yetingk@users.noreply.github.com>
Date: Wed, 27 Mar 2024 13:40:38 +0800
Subject: [PATCH 379/404] [RISCV] Teach RISCVMakeCompressible handle byte/half
 load/store for Zcb. (#83375)

For targets with Zcb, this patch makes llvm generate more compress
c.lb/lbu/lh/lhu/sb/sh instructions.
---
 .../Target/RISCV/RISCVMakeCompressible.cpp    |  50 +-
 .../CodeGen/RISCV/make-compressible-zbc.mir   | 585 ++++++++++++++++++
 2 files changed, 632 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/CodeGen/RISCV/make-compressible-zbc.mir

diff --git a/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp b/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp
index af864ba0fbc4..3f423450618d 100644
--- a/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp
+++ b/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp
@@ -99,6 +99,13 @@ static unsigned log2LdstWidth(unsigned Opcode) {
   switch (Opcode) {
   default:
     llvm_unreachable("Unexpected opcode");
+  case RISCV::LBU:
+  case RISCV::SB:
+    return 0;
+  case RISCV::LH:
+  case RISCV::LHU:
+  case RISCV::SH:
+    return 1;
   case RISCV::LW:
   case RISCV::SW:
   case RISCV::FLW:
@@ -112,17 +119,47 @@ static unsigned log2LdstWidth(unsigned Opcode) {
   }
 }
 
+// Return bit field size of immediate operand of Opcode.
+static unsigned offsetMask(unsigned Opcode) {
+  switch (Opcode) {
+  default:
+    llvm_unreachable("Unexpected opcode");
+  case RISCV::LBU:
+  case RISCV::SB:
+    return maskTrailingOnes<unsigned>(2U);
+  case RISCV::LH:
+  case RISCV::LHU:
+  case RISCV::SH:
+    return maskTrailingOnes<unsigned>(1U);
+  case RISCV::LW:
+  case RISCV::SW:
+  case RISCV::FLW:
+  case RISCV::FSW:
+  case RISCV::LD:
+  case RISCV::SD:
+  case RISCV::FLD:
+  case RISCV::FSD:
+    return maskTrailingOnes<unsigned>(5U);
+  }
+}
+
 // Return a mask for the offset bits of a non-stack-pointer based compressed
 // load/store.
 static uint8_t compressedLDSTOffsetMask(unsigned Opcode) {
-  return 0x1f << log2LdstWidth(Opcode);
+  return offsetMask(Opcode) << log2LdstWidth(Opcode);
 }
 
 // Return true if Offset fits within a compressed stack-pointer based
 // load/store.
 static bool compressibleSPOffset(int64_t Offset, unsigned Opcode) {
-  return log2LdstWidth(Opcode) == 2 ? isShiftedUInt<6, 2>(Offset)
-                                    : isShiftedUInt<6, 3>(Offset);
+  // Compressed sp-based loads and stores only work for 32/64 bits.
+  switch (log2LdstWidth(Opcode)) {
+  case 2:
+    return isShiftedUInt<6, 2>(Offset);
+  case 3:
+    return isShiftedUInt<6, 3>(Offset);
+  }
+  return false;
 }
 
 // Given an offset for a load/store, return the adjustment required to the base
@@ -147,6 +184,10 @@ static bool isCompressibleLoad(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   default:
     return false;
+  case RISCV::LBU:
+  case RISCV::LH:
+  case RISCV::LHU:
+    return STI.hasStdExtZcb();
   case RISCV::LW:
   case RISCV::LD:
     return STI.hasStdExtCOrZca();
@@ -164,6 +205,9 @@ static bool isCompressibleStore(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   default:
     return false;
+  case RISCV::SB:
+  case RISCV::SH:
+    return STI.hasStdExtZcb();
   case RISCV::SW:
   case RISCV::SD:
     return STI.hasStdExtCOrZca();
diff --git a/llvm/test/CodeGen/RISCV/make-compressible-zbc.mir b/llvm/test/CodeGen/RISCV/make-compressible-zbc.mir
new file mode 100644
index 000000000000..89a6ca7af9be
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/make-compressible-zbc.mir
@@ -0,0 +1,585 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -o - %s -mtriple=riscv32 -mattr=+zcb -simplify-mir \
+# RUN:   -run-pass=riscv-make-compressible | FileCheck --check-prefixes=CHECK %s
+# RUN: llc -o - %s -mtriple=riscv64 -mattr=+zcb -simplify-mir \
+# RUN:   -run-pass=riscv-make-compressible | FileCheck --check-prefixes=CHECK %s
+
+--- |
+  define void @store_common_value_i8(ptr %a, ptr %b, ptr %c) #0 {
+  entry:
+    store i8 0, ptr %a, align 1
+    store i8 0, ptr %b, align 1
+    store i8 0, ptr %c, align 1
+    ret void
+  }
+
+  define void @store_common_value_i16(ptr %a, ptr %b, ptr %c) #0 {
+  entry:
+    store i16 0, ptr %a, align 2
+    store i16 0, ptr %b, align 2
+    store i16 0, ptr %c, align 2
+    ret void
+  }
+
+  define void @store_common_ptr_i8(ptr %p) #0 {
+  entry:
+    store volatile i8 1, ptr %p, align 1
+    store volatile i8 3, ptr %p, align 1
+    store volatile i8 5, ptr %p, align 1
+    ret void
+  }
+
+  define void @store_common_ptr_i16(ptr %p) #0 {
+  entry:
+    store volatile i16 1, ptr %p, align 2
+    store volatile i16 3, ptr %p, align 2
+    store volatile i16 5, ptr %p, align 2
+    ret void
+  }
+
+  define void @load_common_ptr_i8(ptr %p) #0 {
+  entry:
+    %0 = load volatile i8, ptr %p, align 1
+    %a = sext i8 %0 to i32
+    %1 = load volatile i8, ptr %p, align 1
+    %2 = load volatile i8, ptr %p, align 1
+    ret void
+  }
+
+  define void @load_common_ptr_s16(ptr %p) #0 {
+  entry:
+    %0 = load volatile i16, ptr %p, align 2
+    %1 = load volatile i16, ptr %p, align 2
+    %2 = load volatile i16, ptr %p, align 2
+    ret void
+  }
+
+  define void @load_common_ptr_u16(ptr %p) #0 {
+  entry:
+    %0 = load volatile i16, ptr %p, align 2
+    %1 = load volatile i16, ptr %p, align 2
+    %2 = load volatile i16, ptr %p, align 2
+    ret void
+  }
+
+  define void @store_large_offset_i8(ptr %p) #0 {
+  entry:
+    %0 = getelementptr inbounds i8, ptr %p, i8 100
+    store volatile i8 1, ptr %0, align 1
+    %1 = getelementptr inbounds i8, ptr %p, i8 101
+    store volatile i8 3, ptr %1, align 1
+    %2 = getelementptr inbounds i8, ptr %p, i8 102
+    store volatile i8 5, ptr %2, align 1
+    %3 = getelementptr inbounds i8, ptr %p, i8 103
+    store volatile i8 7, ptr %3, align 1
+    ret void
+  }
+
+  define void @store_large_offset_i16(ptr %p) #0 {
+  entry:
+    %0 = getelementptr inbounds i16, ptr %p, i16 100
+    store volatile i16 1, ptr %0, align 2
+    %1 = getelementptr inbounds i16, ptr %p, i16 100
+    store volatile i16 3, ptr %1, align 2
+    %2 = getelementptr inbounds i16, ptr %p, i16 101
+    store volatile i16 3, ptr %1, align 2
+    %3 = getelementptr inbounds i16, ptr %p, i16 101
+    store volatile i16 7, ptr %3, align 2
+    ret void
+  }
+
+  define void @load_large_offset_i8(ptr %p) #0 {
+  entry:
+    %0 = getelementptr inbounds i8, ptr %p, i8 100
+    %a = load volatile i8, ptr %0
+    %1 = getelementptr inbounds i8, ptr %p, i8 101
+    %b = load volatile i8, ptr %1
+    %2 = getelementptr inbounds i8, ptr %p, i8 102
+    %c = load volatile i8, ptr %2
+    %3 = getelementptr inbounds i8, ptr %p, i8 103
+    %d = load volatile i8, ptr %3
+    ret void
+  }
+
+  define void @load_large_offset_s16(ptr %p) #0 {
+  entry:
+    %0 = getelementptr inbounds i16, ptr %p, i16 100
+    %a = load volatile i16, ptr %0, align 2
+    %1 = getelementptr inbounds i16, ptr %p, i16 100
+    %b = load volatile i16, ptr %1, align 2
+    %2 = getelementptr inbounds i16, ptr %p, i16 101
+    %c = load volatile i16, ptr %2, align 2
+    %3 = getelementptr inbounds i16, ptr %p, i16 101
+    %d = load volatile i16, ptr %3, align 2
+    ret void
+  }
+
+  define void @load_large_offset_u16(ptr %p) #0 {
+  entry:
+    %0 = getelementptr inbounds i16, ptr %p, i16 100
+    %a = load volatile i16, ptr %0, align 2
+    %1 = getelementptr inbounds i16, ptr %p, i16 100
+    %b = load volatile i16, ptr %1, align 2
+    %2 = getelementptr inbounds i16, ptr %p, i16 101
+    %c = load volatile i16, ptr %2, align 2
+    %3 = getelementptr inbounds i16, ptr %p, i16 101
+    %d = load volatile i16, ptr %3, align 2
+    ret void
+  }
+  define void @store_large_offset_no_opt_i8(ptr %p) #0 {
+  entry:
+    %0 = getelementptr inbounds i8, ptr %p, i8 100
+    store volatile i8 1, ptr %0, align 1
+    %1 = getelementptr inbounds i8, ptr %p, i8 101
+    store volatile i8 3, ptr %1, align 1
+    %2 = getelementptr inbounds i8, ptr %p, i8 104
+    store volatile i8 5, ptr %2, align 1
+    ret void
+  }
+
+  define void @store_large_offset_no_opt_i16(ptr %p) #0 {
+  entry:
+    %0 = getelementptr inbounds i16, ptr %p, i16 100
+    %a = load volatile i16, ptr %0, align 2
+    %1 = getelementptr inbounds i16, ptr %p, i16 100
+    %b = load volatile i16, ptr %1, align 2
+    %2 = getelementptr inbounds i16, ptr %p, i16 101
+    %c = load volatile i16, ptr %2, align 2
+    %3 = getelementptr inbounds i16, ptr %p, i16 102
+    %d = load volatile i16, ptr %3, align 2
+    ret void
+  }
+
+  define void @load_large_offset_no_opt_i8(ptr %p) #0 {
+  entry:
+    %0 = getelementptr inbounds i8, ptr %p, i8 100
+    %a = load volatile i8, ptr %0
+    %1 = getelementptr inbounds i8, ptr %p, i8 101
+    %b = load volatile i8, ptr %1
+    %2 = getelementptr inbounds i8, ptr %p, i8 103
+    %c = load volatile i8, ptr %2
+    ret void
+  }
+
+  define void @load_large_offset_no_opt_s16(ptr %p) #0 {
+  entry:
+    %0 = getelementptr inbounds i16, ptr %p, i16 100
+    %a = load volatile i16, ptr %0, align 2
+    %1 = getelementptr inbounds i16, ptr %p, i16 101
+    %c = load volatile i16, ptr %1, align 2
+    %2 = getelementptr inbounds i16, ptr %p, i16 102
+    %d = load volatile i16, ptr %2, align 2
+    ret void
+  }
+
+  define void @load_large_offset_no_opt_u16(ptr %p) #0 {
+  entry:
+    %0 = getelementptr inbounds i16, ptr %p, i16 100
+    %a = load volatile i16, ptr %0, align 2
+    %1 = getelementptr inbounds i16, ptr %p, i16 101
+    %c = load volatile i16, ptr %1, align 2
+    %2 = getelementptr inbounds i16, ptr %p, i16 102
+    %d = load volatile i16, ptr %2, align 2
+    ret void
+  }
+  attributes #0 = { minsize }
+
+...
+---
+name:            store_common_value_i8
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11, $x12
+
+    ; CHECK-LABEL: name: store_common_value_i8
+    ; CHECK: liveins: $x10, $x11, $x12
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x13 = ADDI $x0, 0
+    ; CHECK-NEXT: SB $x13, killed renamable $x10, 0 :: (store (s8) into %ir.a)
+    ; CHECK-NEXT: SB $x13, killed renamable $x11, 0 :: (store (s8) into %ir.b)
+    ; CHECK-NEXT: SB $x13, killed renamable $x12, 0 :: (store (s8) into %ir.c)
+    ; CHECK-NEXT: PseudoRET
+    SB $x0, killed renamable $x10, 0 :: (store (s8) into %ir.a)
+    SB $x0, killed renamable $x11, 0 :: (store (s8) into %ir.b)
+    SB $x0, killed renamable $x12, 0 :: (store (s8) into %ir.c)
+    PseudoRET
+
+...
+---
+name:            store_common_value_i16
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11, $x12
+
+    ; CHECK-LABEL: name: store_common_value_i16
+    ; CHECK: liveins: $x10, $x11, $x12
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x13 = ADDI $x0, 0
+    ; CHECK-NEXT: SH $x13, killed renamable $x10, 0 :: (store (s16) into %ir.a)
+    ; CHECK-NEXT: SH $x13, killed renamable $x11, 0 :: (store (s16) into %ir.b)
+    ; CHECK-NEXT: SH $x13, killed renamable $x12, 0 :: (store (s16) into %ir.c)
+    ; CHECK-NEXT: PseudoRET
+    SH $x0, killed renamable $x10, 0 :: (store (s16) into %ir.a)
+    SH $x0, killed renamable $x11, 0 :: (store (s16) into %ir.b)
+    SH $x0, killed renamable $x12, 0 :: (store (s16) into %ir.c)
+    PseudoRET
+
+...
+---
+name:            store_common_ptr_i8
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: store_common_ptr_i8
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $x10 = ADDI $x0, 1
+    ; CHECK-NEXT: $x11 = ADDI $x16, 0
+    ; CHECK-NEXT: SB killed renamable $x10, $x11, 0 :: (volatile store (s8) into %ir.p)
+    ; CHECK-NEXT: renamable $x10 = ADDI $x0, 3
+    ; CHECK-NEXT: SB killed renamable $x10, $x11, 0 :: (volatile store (s8) into %ir.p)
+    ; CHECK-NEXT: renamable $x10 = ADDI $x0, 5
+    ; CHECK-NEXT: SB killed renamable $x10, killed $x11, 0 :: (volatile store (s8) into %ir.p)
+    ; CHECK-NEXT: PseudoRET
+    renamable $x10 = ADDI $x0, 1
+    SB killed renamable $x10, renamable $x16, 0 :: (volatile store (s8) into %ir.p)
+    renamable $x10 = ADDI $x0, 3
+    SB killed renamable $x10, renamable $x16, 0 :: (volatile store (s8) into %ir.p)
+    renamable $x10 = ADDI $x0, 5
+    SB killed renamable $x10, killed renamable $x16, 0 :: (volatile store (s8) into %ir.p)
+    PseudoRET
+
+...
+---
+name:            store_common_ptr_i16
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: store_common_ptr_i16
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $x10 = ADDI $x0, 1
+    ; CHECK-NEXT: $x11 = ADDI $x16, 0
+    ; CHECK-NEXT: SH killed renamable $x10, $x11, 0 :: (volatile store (s16) into %ir.p)
+    ; CHECK-NEXT: renamable $x10 = ADDI $x0, 3
+    ; CHECK-NEXT: SH killed renamable $x10, $x11, 0 :: (volatile store (s16) into %ir.p)
+    ; CHECK-NEXT: renamable $x10 = ADDI $x0, 5
+    ; CHECK-NEXT: SH killed renamable $x10, killed $x11, 0 :: (volatile store (s16) into %ir.p)
+    ; CHECK-NEXT: PseudoRET
+    renamable $x10 = ADDI $x0, 1
+    SH killed renamable $x10, renamable $x16, 0 :: (volatile store (s16) into %ir.p)
+    renamable $x10 = ADDI $x0, 3
+    SH killed renamable $x10, renamable $x16, 0 :: (volatile store (s16) into %ir.p)
+    renamable $x10 = ADDI $x0, 5
+    SH killed renamable $x10, killed renamable $x16, 0 :: (volatile store (s16) into %ir.p)
+    PseudoRET
+
+...
+---
+name:            load_common_ptr_i8
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: load_common_ptr_i8
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x11 = ADDI $x16, 0
+    ; CHECK-NEXT: dead $x10 = LBU $x11, 0 :: (volatile load (s8) from %ir.p)
+    ; CHECK-NEXT: dead $x10 = LBU $x11, 0 :: (volatile load (s8) from %ir.p)
+    ; CHECK-NEXT: dead $x10 = LBU killed $x11, 0 :: (volatile load (s8) from %ir.p)
+    ; CHECK-NEXT: PseudoRET
+    dead $x10 = LBU renamable $x16, 0 :: (volatile load (s8) from %ir.p)
+    dead $x10 = LBU renamable $x16, 0 :: (volatile load (s8) from %ir.p)
+    dead $x10 = LBU killed renamable $x16, 0 :: (volatile load (s8) from %ir.p)
+    PseudoRET
+
+...
+---
+name:            load_common_ptr_s16
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: load_common_ptr_s16
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x11 = ADDI $x16, 0
+    ; CHECK-NEXT: dead $x10 = LH $x11, 0 :: (volatile load (s16) from %ir.p)
+    ; CHECK-NEXT: dead $x10 = LH $x11, 0 :: (volatile load (s16) from %ir.p)
+    ; CHECK-NEXT: dead $x10 = LH killed $x11, 0 :: (volatile load (s16) from %ir.p)
+    ; CHECK-NEXT: PseudoRET
+    dead $x10 = LH renamable $x16, 0 :: (volatile load (s16) from %ir.p)
+    dead $x10 = LH renamable $x16, 0 :: (volatile load (s16) from %ir.p)
+    dead $x10 = LH killed renamable $x16, 0 :: (volatile load (s16) from %ir.p)
+    PseudoRET
+
+...
+---
+name:            load_common_ptr_u16
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: load_common_ptr_u16
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x11 = ADDI $x16, 0
+    ; CHECK-NEXT: dead $x10 = LHU $x11, 0 :: (volatile load (s16) from %ir.p)
+    ; CHECK-NEXT: dead $x10 = LHU $x11, 0 :: (volatile load (s16) from %ir.p)
+    ; CHECK-NEXT: dead $x10 = LHU killed $x11, 0 :: (volatile load (s16) from %ir.p)
+    ; CHECK-NEXT: PseudoRET
+    dead $x10 = LHU renamable $x16, 0 :: (volatile load (s16) from %ir.p)
+    dead $x10 = LHU renamable $x16, 0 :: (volatile load (s16) from %ir.p)
+    dead $x10 = LHU killed renamable $x16, 0 :: (volatile load (s16) from %ir.p)
+    PseudoRET
+
+...
+---
+name:            store_large_offset_i8
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: store_large_offset_i8
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $x11 = ADDI $x0, 1
+    ; CHECK-NEXT: $x12 = ADDI $x10, 100
+    ; CHECK-NEXT: SB killed renamable $x11, $x12, 0 :: (volatile store (s8) into %ir.0)
+    ; CHECK-NEXT: renamable $x11 = ADDI $x0, 3
+    ; CHECK-NEXT: SB killed renamable $x11, $x12, 1 :: (volatile store (s8) into %ir.1)
+    ; CHECK-NEXT: renamable $x11 = ADDI $x0, 5
+    ; CHECK-NEXT: SB killed renamable $x11, $x12, 2 :: (volatile store (s8) into %ir.2)
+    ; CHECK-NEXT: renamable $x11 = ADDI $x0, 7
+    ; CHECK-NEXT: SB killed renamable $x11, killed $x12, 3 :: (volatile store (s8) into %ir.3)
+    ; CHECK-NEXT: PseudoRET
+    renamable $x11 = ADDI $x0, 1
+    SB killed renamable $x11, renamable $x10, 100 :: (volatile store (s8) into %ir.0)
+    renamable $x11 = ADDI $x0, 3
+    SB killed renamable $x11, renamable $x10, 101 :: (volatile store (s8) into %ir.1)
+    renamable $x11 = ADDI $x0, 5
+    SB killed renamable $x11, renamable $x10, 102 :: (volatile store (s8) into %ir.2)
+    renamable $x11 = ADDI $x0, 7
+    SB killed renamable $x11, killed renamable $x10, 103 :: (volatile store (s8) into %ir.3)
+    PseudoRET
+
+...
+---
+name:            store_large_offset_i16
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    ; CHECK-LABEL: name: store_large_offset_i16
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $x11 = ADDI $x0, 1
+    ; CHECK-NEXT: $x12 = ADDI $x10, 200
+    ; CHECK-NEXT: SH killed renamable $x11, $x12, 0 :: (volatile store (s16) into %ir.0)
+    ; CHECK-NEXT: renamable $x11 = ADDI $x0, 3
+    ; CHECK-NEXT: SH killed renamable $x11, $x12, 0 :: (volatile store (s16) into %ir.1)
+    ; CHECK-NEXT: renamable $x11 = ADDI $x0, 5
+    ; CHECK-NEXT: SH killed renamable $x11, $x12, 2 :: (volatile store (s16) into %ir.2)
+    ; CHECK-NEXT: renamable $x11 = ADDI $x0, 7
+    ; CHECK-NEXT: SH killed renamable $x11, killed $x12, 2 :: (volatile store (s16) into %ir.3)
+    ; CHECK-NEXT: PseudoRET
+    renamable $x11 = ADDI $x0, 1
+    SH killed renamable $x11, renamable $x10, 200 :: (volatile store (s16) into %ir.0)
+    renamable $x11 = ADDI $x0, 3
+    SH killed renamable $x11, renamable $x10, 200 :: (volatile store (s16) into %ir.1)
+    renamable $x11 = ADDI $x0, 5
+    SH killed renamable $x11, renamable $x10, 202 :: (volatile store (s16) into %ir.2)
+    renamable $x11 = ADDI $x0, 7
+    SH killed renamable $x11, killed renamable $x10, 202 :: (volatile store (s16) into %ir.3)
+    PseudoRET
+
+...
+---
+name:            load_large_offset_i8
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: load_large_offset_i8
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x11 = ADDI $x16, 100
+    ; CHECK-NEXT: dead $x10 = LBU $x11, 0 :: (volatile load (s8) from %ir.0)
+    ; CHECK-NEXT: dead $x10 = LBU $x11, 1 :: (volatile load (s8) from %ir.1)
+    ; CHECK-NEXT: dead $x10 = LBU $x11, 2 :: (volatile load (s8) from %ir.2)
+    ; CHECK-NEXT: dead $x10 = LBU killed $x11, 3 :: (volatile load (s8) from %ir.3)
+    ; CHECK-NEXT: PseudoRET
+    dead $x10 = LBU renamable $x16, 100 :: (volatile load (s8) from %ir.0)
+    dead $x10 = LBU renamable $x16, 101 :: (volatile load (s8) from %ir.1)
+    dead $x10 = LBU renamable $x16, 102 :: (volatile load (s8) from %ir.2)
+    dead $x10 = LBU killed renamable $x16, 103 :: (volatile load (s8) from %ir.3)
+    PseudoRET
+
+...
+---
+name:            load_large_offset_s16
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: load_large_offset_s16
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x11 = ADDI $x16, 100
+    ; CHECK-NEXT: dead $x10 = LH $x11, 0 :: (volatile load (s16) from %ir.0)
+    ; CHECK-NEXT: dead $x10 = LH $x11, 0 :: (volatile load (s16) from %ir.1)
+    ; CHECK-NEXT: dead $x10 = LH $x11, 2 :: (volatile load (s16) from %ir.2)
+    ; CHECK-NEXT: dead $x10 = LH killed $x11, 2 :: (volatile load (s16) from %ir.3)
+    ; CHECK-NEXT: PseudoRET
+    dead $x10 = LH renamable $x16, 100 :: (volatile load (s16) from %ir.0)
+    dead $x10 = LH renamable $x16, 100 :: (volatile load (s16) from %ir.1)
+    dead $x10 = LH renamable $x16, 102 :: (volatile load (s16) from %ir.2)
+    dead $x10 = LH killed renamable $x16, 102 :: (volatile load (s16) from %ir.3)
+    PseudoRET
+
+...
+---
+name:            load_large_offset_u16
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: load_large_offset_u16
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x11 = ADDI $x16, 100
+    ; CHECK-NEXT: dead $x10 = LHU $x11, 0 :: (volatile load (s16) from %ir.0)
+    ; CHECK-NEXT: dead $x10 = LHU $x11, 0 :: (volatile load (s16) from %ir.1)
+    ; CHECK-NEXT: dead $x10 = LHU $x11, 2 :: (volatile load (s16) from %ir.2)
+    ; CHECK-NEXT: dead $x10 = LHU killed $x11, 2 :: (volatile load (s16) from %ir.3)
+    ; CHECK-NEXT: PseudoRET
+    dead $x10 = LHU renamable $x16, 100 :: (volatile load (s16) from %ir.0)
+    dead $x10 = LHU renamable $x16, 100 :: (volatile load (s16) from %ir.1)
+    dead $x10 = LHU renamable $x16, 102 :: (volatile load (s16) from %ir.2)
+    dead $x10 = LHU killed renamable $x16, 102 :: (volatile load (s16) from %ir.3)
+    PseudoRET
+
+...
+---
+name:            store_large_offset_no_opt_i8
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: store_large_offset_no_opt_i8
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $x11 = ADDI $x0, 1
+    ; CHECK-NEXT: SB killed renamable $x11, renamable $x16, 100 :: (volatile store (s8) into %ir.0)
+    ; CHECK-NEXT: renamable $x11 = ADDI $x0, 3
+    ; CHECK-NEXT: SB killed renamable $x11, renamable $x16, 101 :: (volatile store (s8) into %ir.1)
+    ; CHECK-NEXT: renamable $x11 = ADDI $x0, 5
+    ; CHECK-NEXT: SB killed renamable $x11, renamable $x16, 104 :: (volatile store (s8) into %ir.2)
+    ; CHECK-NEXT: PseudoRET
+    renamable $x11 = ADDI $x0, 1
+    SB killed renamable $x11, renamable $x16, 100 :: (volatile store (s8) into %ir.0)
+    renamable $x11 = ADDI $x0, 3
+    SB killed renamable $x11, renamable $x16, 101 :: (volatile store (s8) into %ir.1)
+    renamable $x11 = ADDI $x0, 5
+    SB killed renamable $x11, renamable $x16, 104 :: (volatile store (s8) into %ir.2)
+    PseudoRET
+
+...
+---
+name:            store_large_offset_no_opt_i16
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: store_large_offset_no_opt_i16
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $x11 = ADDI $x0, 1
+    ; CHECK-NEXT: SH killed renamable $x11, renamable $x16, 200 :: (volatile store (s16) into %ir.0)
+    ; CHECK-NEXT: renamable $x11 = ADDI $x0, 3
+    ; CHECK-NEXT: SH killed renamable $x11, renamable $x16, 202 :: (volatile store (s16) into %ir.1)
+    ; CHECK-NEXT: renamable $x11 = ADDI $x0, 5
+    ; CHECK-NEXT: SH killed renamable $x11, renamable $x16, 204 :: (volatile store (s16) into %ir.2)
+    ; CHECK-NEXT: PseudoRET
+    renamable $x11 = ADDI $x0, 1
+    SH killed renamable $x11, renamable $x16, 200 :: (volatile store (s16) into %ir.0)
+    renamable $x11 = ADDI $x0, 3
+    SH killed renamable $x11, renamable $x16, 202 :: (volatile store (s16) into %ir.1)
+    renamable $x11 = ADDI $x0, 5
+    SH killed renamable $x11, renamable $x16, 204 :: (volatile store (s16) into %ir.2)
+    PseudoRET
+
+...
+---
+name:            load_large_offset_no_opt_i8
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: load_large_offset_no_opt_i8
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: dead $x10 = LBU renamable $x16, 100 :: (volatile load (s8) from %ir.0)
+    ; CHECK-NEXT: dead $x10 = LBU renamable $x16, 101 :: (volatile load (s8) from %ir.1)
+    ; CHECK-NEXT: dead $x10 = LBU killed renamable $x16, 104 :: (volatile load (s8) from %ir.2)
+    ; CHECK-NEXT: PseudoRET
+    dead $x10 = LBU renamable $x16, 100 :: (volatile load (s8) from %ir.0)
+    dead $x10 = LBU renamable $x16, 101 :: (volatile load (s8) from %ir.1)
+    dead $x10 = LBU killed renamable $x16, 104 :: (volatile load (s8) from %ir.2)
+    PseudoRET
+
+...
+---
+name:            load_large_offset_no_opt_s16
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: load_large_offset_no_opt_s16
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: dead $x10 = LH renamable $x16, 100 :: (volatile load (s8) from %ir.0)
+    ; CHECK-NEXT: dead $x10 = LH renamable $x16, 102 :: (volatile load (s8) from %ir.1)
+    ; CHECK-NEXT: dead $x10 = LH killed renamable $x16, 104 :: (volatile load (s8) from %ir.2)
+    ; CHECK-NEXT: PseudoRET
+    dead $x10 = LH renamable $x16, 100 :: (volatile load (s8) from %ir.0)
+    dead $x10 = LH renamable $x16, 102 :: (volatile load (s8) from %ir.1)
+    dead $x10 = LH killed renamable $x16, 104 :: (volatile load (s8) from %ir.2)
+    PseudoRET
+
+...
+---
+name:            load_large_offset_no_opt_u16
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: load_large_offset_no_opt_u16
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: dead $x10 = LHU renamable $x16, 100 :: (volatile load (s8) from %ir.0)
+    ; CHECK-NEXT: dead $x10 = LHU renamable $x16, 102 :: (volatile load (s8) from %ir.1)
+    ; CHECK-NEXT: dead $x10 = LHU killed renamable $x16, 104 :: (volatile load (s8) from %ir.2)
+    ; CHECK-NEXT: PseudoRET
+    dead $x10 = LHU renamable $x16, 100 :: (volatile load (s8) from %ir.0)
+    dead $x10 = LHU renamable $x16, 102 :: (volatile load (s8) from %ir.1)
+    dead $x10 = LHU killed renamable $x16, 104 :: (volatile load (s8) from %ir.2)
+    PseudoRET
+
+...
-- 
GitLab


From 16993c793a7d81771ea17a2991f76e87b4b0a6c0 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Tue, 26 Mar 2024 22:39:00 -0700
Subject: [PATCH 380/404] [NFC][HWASAN] Regenerate test

---
 .../HWAddressSanitizer/use-after-scope-setjmp.ll              | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll b/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll
index 4bb846bff274..079d72241283 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll
@@ -48,8 +48,8 @@ define dso_local noundef i1 @_Z6targetv() sanitize_hwaddress {
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP26]], i8 [[TMP22]], i64 256, i1 false)
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 @setjmp(ptr noundef @jbuf)
 ; CHECK-NEXT:    switch i32 [[CALL]], label [[WHILE_BODY:%.*]] [
-; CHECK-NEXT:    i32 1, label [[RETURN:%.*]]
-; CHECK-NEXT:    i32 2, label [[SW_BB1:%.*]]
+; CHECK-NEXT:      i32 1, label [[RETURN:%.*]]
+; CHECK-NEXT:      i32 2, label [[SW_BB1:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       sw.bb1:
 ; CHECK-NEXT:    br label [[RETURN]]
-- 
GitLab


From 05a7b22a0132bebe99aabee591d3acc99d793ae1 Mon Sep 17 00:00:00 2001
From: Jianjian Guan <jacquesguan@me.com>
Date: Wed, 27 Mar 2024 14:16:03 +0800
Subject: [PATCH 381/404] [RISCV] Add areInlineCompatible for riscv target
 (#86639)

Inline a callee if its target-features are a subset of the callers
target-features.
---
 .../Target/RISCV/RISCVTargetTransformInfo.cpp | 14 ++++++++
 .../Target/RISCV/RISCVTargetTransformInfo.h   |  3 ++
 .../Inline/RISCV/inline-target-features.ll    | 34 +++++++++++++++++++
 .../Transforms/Inline/RISCV/lit.local.cfg     |  2 ++
 4 files changed, 53 insertions(+)
 create mode 100644 llvm/test/Transforms/Inline/RISCV/inline-target-features.ll
 create mode 100644 llvm/test/Transforms/Inline/RISCV/lit.local.cfg

diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index df289be18932..000d01b8366c 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1676,3 +1676,17 @@ bool RISCVTTIImpl::isLegalMaskedCompressStore(Type *DataTy, Align Alignment) {
     return false;
   return true;
 }
+
+bool RISCVTTIImpl::areInlineCompatible(const Function *Caller,
+                                       const Function *Callee) const {
+  const TargetMachine &TM = getTLI()->getTargetMachine();
+
+  const FeatureBitset &CallerBits =
+      TM.getSubtargetImpl(*Caller)->getFeatureBits();
+  const FeatureBitset &CalleeBits =
+      TM.getSubtargetImpl(*Callee)->getFeatureBits();
+
+  // Inline a callee if its target-features are a subset of the callers
+  // target-features.
+  return (CallerBits & CalleeBits) == CalleeBits;
+}
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 8daf6845dc8b..ac32aea4ce2b 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -60,6 +60,9 @@ public:
       : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)),
         TLI(ST->getTargetLowering()) {}
 
+  bool areInlineCompatible(const Function *Caller,
+                           const Function *Callee) const;
+
   /// Return the cost of materializing an immediate for a value operand of
   /// a store instruction.
   InstructionCost getStoreImmCost(Type *VecTy, TTI::OperandValueInfo OpInfo,
diff --git a/llvm/test/Transforms/Inline/RISCV/inline-target-features.ll b/llvm/test/Transforms/Inline/RISCV/inline-target-features.ll
new file mode 100644
index 000000000000..b626a229a737
--- /dev/null
+++ b/llvm/test/Transforms/Inline/RISCV/inline-target-features.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -mtriple=riscv64-unknown-linux-gnu -S -passes=inline | FileCheck %s
+; RUN: opt < %s -mtriple=riscv64-unknown-linux-gnu -S -passes='cgscc(inline)' | FileCheck %s
+; Check that we only inline when we have compatible target attributes.
+
+target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
+target triple = "riscv64-unknown-linux-gnu"
+
+define i32 @foo() #0 {
+entry:
+  %call = call i32 (...) @baz()
+  ret i32 %call
+; CHECK-LABEL: foo
+; CHECK: call i32 (...) @baz()
+}
+declare i32 @baz(...) #0
+
+define i32 @bar() #1 {
+entry:
+  %call = call i32 @foo()
+  ret i32 %call
+; CHECK-LABEL: bar
+; CHECK: call i32 (...) @baz()
+}
+
+define i32 @qux() #0 {
+entry:
+  %call = call i32 @bar()
+  ret i32 %call
+; CHECK-LABEL: qux
+; CHECK: call i32 @bar()
+}
+
+attributes #0 = { "target-cpu"="generic-rv64" "target-features"="+f,+d" }
+attributes #1 = { "target-cpu"="generic-rv64" "target-features"="+f,+d,+m,+v" }
diff --git a/llvm/test/Transforms/Inline/RISCV/lit.local.cfg b/llvm/test/Transforms/Inline/RISCV/lit.local.cfg
new file mode 100644
index 000000000000..17351748513d
--- /dev/null
+++ b/llvm/test/Transforms/Inline/RISCV/lit.local.cfg
@@ -0,0 +1,2 @@
+if not "RISCV" in config.root.targets:
+    config.unsupported = True
-- 
GitLab


From 577e0ef94fb0b4ba9f97a6f58a1961f7ba247d21 Mon Sep 17 00:00:00 2001
From: Alina Sbirlea <asbirlea@google.com>
Date: Tue, 26 Mar 2024 23:24:02 -0700
Subject: [PATCH 382/404] [clang][AST] Silence unused-value warnings in
 unittest DeclPrinterTest

---
 clang/unittests/AST/DeclPrinterTest.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/clang/unittests/AST/DeclPrinterTest.cpp b/clang/unittests/AST/DeclPrinterTest.cpp
index 07fa02bd96e2..8a29d0544a04 100644
--- a/clang/unittests/AST/DeclPrinterTest.cpp
+++ b/clang/unittests/AST/DeclPrinterTest.cpp
@@ -1391,7 +1391,7 @@ TEST(DeclPrinter, TestCXXRecordDecl17) {
                                       "struct X {};"
                                       "Z<X> A;",
                                       "A", "Z<X> A"));
-  [](PrintingPolicy &Policy) { Policy.SuppressTagKeyword = false; };
+  (void)[](PrintingPolicy &Policy) { Policy.SuppressTagKeyword = false; };
 }
 
 TEST(DeclPrinter, TestCXXRecordDecl18) {
@@ -1402,7 +1402,7 @@ TEST(DeclPrinter, TestCXXRecordDecl18) {
                                       "struct Y{};"
                                       "Y<Z<X>, 2> B;",
                                       "B", "Y<Z<X>, 2> B"));
-  [](PrintingPolicy &Policy) { Policy.SuppressTagKeyword = false; };
+  (void)[](PrintingPolicy &Policy) { Policy.SuppressTagKeyword = false; };
 }
 
 TEST(DeclPrinter, TestCXXRecordDecl19) {
@@ -1413,7 +1413,7 @@ TEST(DeclPrinter, TestCXXRecordDecl19) {
                                       "struct Y{};"
                                       "Y<Z<X>, 2> B;",
                                       "B", "Y<Z<X>, 2> B"));
-  [](PrintingPolicy &Policy) { Policy.SuppressTagKeyword = true; };
+  (void)[](PrintingPolicy &Policy) { Policy.SuppressTagKeyword = true; };
 }
 TEST(DeclPrinter, TestCXXRecordDecl20) {
   ASSERT_TRUE(PrintedDeclCXX98Matches(
@@ -1432,7 +1432,7 @@ TEST(DeclPrinter, TestCXXRecordDecl20) {
       "Outer<Inner<int, 10>, 5>::NestedStruct nestedInstance(100);",
       "nestedInstance",
       "Outer<Inner<int, 10>, 5>::NestedStruct nestedInstance(100)"));
-  [](PrintingPolicy &Policy) { Policy.SuppressTagKeyword = false; };
+  (void)[](PrintingPolicy &Policy) { Policy.SuppressTagKeyword = false; };
 }
 
 TEST(DeclPrinter, TestCXXRecordDecl21) {
@@ -1452,7 +1452,7 @@ TEST(DeclPrinter, TestCXXRecordDecl21) {
       "Outer<Inner<int, 10>, 5>::NestedStruct nestedInstance(100);",
       "nestedInstance",
       "Outer<Inner<int, 10>, 5>::NestedStruct nestedInstance(100)"));
-  [](PrintingPolicy &Policy) { Policy.SuppressTagKeyword = true; };
+  (void)[](PrintingPolicy &Policy) { Policy.SuppressTagKeyword = true; };
 }
 
 TEST(DeclPrinter, TestFunctionParamUglified) {
-- 
GitLab


From aa2d5d54130bd9c5e9efb9ae3eaec631f227f13b Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung@sifive.com>
Date: Tue, 26 Mar 2024 23:09:09 -0700
Subject: [PATCH 383/404] Recommit "[RISCV][TTI] Scale the cost of the
 sext/zext with LMUL (#86617)"

Changes in Recommit:
  Add an additional check on sign/zero extend to the same type.

Original message:
  Use the destination data type to measure the LMUL size for
  latency/throughput cost
---
 .../Target/RISCV/RISCVTargetTransformInfo.cpp |  20 +-
 llvm/test/Analysis/CostModel/RISCV/cast.ll    | 920 +++++++++---------
 .../CostModel/RISCV/reduce-scalable-int.ll    |  12 +-
 .../CostModel/RISCV/rvv-extractelement.ll     |  84 +-
 .../CostModel/RISCV/rvv-insertelement.ll      |  84 +-
 .../CostModel/RISCV/shuffle-broadcast.ll      |   2 +-
 6 files changed, 566 insertions(+), 556 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 000d01b8366c..38cdf3c47c64 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -909,23 +909,33 @@ InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
   if (!IsTypeLegal)
     return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
 
+  std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(Dst);
+
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
   assert(ISD && "Invalid opcode");
 
-  // FIXME: Need to consider vsetvli and lmul.
   int PowDiff = (int)Log2_32(Dst->getScalarSizeInBits()) -
                 (int)Log2_32(Src->getScalarSizeInBits());
   switch (ISD) {
   case ISD::SIGN_EXTEND:
-  case ISD::ZERO_EXTEND:
-    if (Src->getScalarSizeInBits() == 1) {
+  case ISD::ZERO_EXTEND: {
+    const unsigned SrcEltSize = Src->getScalarSizeInBits();
+    if (SrcEltSize == 1) {
       // We do not use vsext/vzext to extend from mask vector.
       // Instead we use the following instructions to extend from mask vector:
       // vmv.v.i v8, 0
       // vmerge.vim v8, v8, -1, v0
-      return 2;
+      return getRISCVInstructionCost({RISCV::VMV_V_I, RISCV::VMERGE_VIM},
+                                     DstLT.second, CostKind);
     }
-    return 1;
+    if ((PowDiff < 1) || (PowDiff > 3))
+      return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
+    unsigned SExtOp[] = {RISCV::VSEXT_VF2, RISCV::VSEXT_VF4, RISCV::VSEXT_VF8};
+    unsigned ZExtOp[] = {RISCV::VZEXT_VF2, RISCV::VZEXT_VF4, RISCV::VZEXT_VF8};
+    unsigned Op =
+        (ISD == ISD::SIGN_EXTEND) ? SExtOp[PowDiff - 1] : ZExtOp[PowDiff - 1];
+    return getRISCVInstructionCost(Op, DstLT.second, CostKind);
+  }
   case ISD::TRUNCATE:
     if (Dst->getScalarSizeInBits() == 1) {
       // We do not use several vncvt to truncate to mask vector. So we could
diff --git a/llvm/test/Analysis/CostModel/RISCV/cast.ll b/llvm/test/Analysis/CostModel/RISCV/cast.ll
index bd26c19c2f2c..14da9a3f79d7 100644
--- a/llvm/test/Analysis/CostModel/RISCV/cast.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/cast.ll
@@ -16,74 +16,74 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i1_v2i64 = sext <2 x i1> undef to <2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i16 = sext <4 x i8> undef to <4 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i32 = sext <4 x i8> undef to <4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i64 = sext <4 x i8> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_v4i64 = sext <4 x i8> undef to <4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i32 = sext <4 x i16> undef to <4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i64 = sext <4 x i16> undef to <4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_v4i64 = sext <4 x i32> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16_v4i64 = sext <4 x i16> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32_v4i64 = sext <4 x i32> undef to <4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i8 = sext <4 x i1> undef to <4 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i16 = sext <4 x i1> undef to <4 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i32 = sext <4 x i1> undef to <4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i64 = sext <4 x i1> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_v4i64 = sext <4 x i1> undef to <4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i16 = sext <8 x i8> undef to <8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i32 = sext <8 x i8> undef to <8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i64 = sext <8 x i8> undef to <8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i32 = sext <8 x i16> undef to <8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i64 = sext <8 x i16> undef to <8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_v8i64 = sext <8 x i32> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_v8i32 = sext <8 x i8> undef to <8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_v8i64 = sext <8 x i8> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16_v8i32 = sext <8 x i16> undef to <8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i16_v8i64 = sext <8 x i16> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32_v8i64 = sext <8 x i32> undef to <8 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i8 = sext <8 x i1> undef to <8 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i16 = sext <8 x i1> undef to <8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i32 = sext <8 x i1> undef to <8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i64 = sext <8 x i1> undef to <8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i16 = sext <16 x i8> undef to <16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i32 = sext <16 x i8> undef to <16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i64 = sext <16 x i8> undef to <16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i32 = sext <16 x i16> undef to <16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i64 = sext <16 x i16> undef to <16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_v16i64 = sext <16 x i32> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_v8i32 = sext <8 x i1> undef to <8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i1_v8i64 = sext <8 x i1> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8_v16i16 = sext <16 x i8> undef to <16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i8_v16i32 = sext <16 x i8> undef to <16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_v16i64 = sext <16 x i8> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16_v16i32 = sext <16 x i16> undef to <16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i16_v16i64 = sext <16 x i16> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32_v16i64 = sext <16 x i32> undef to <16 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i8 = sext <16 x i1> undef to <16 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i16 = sext <16 x i1> undef to <16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i32 = sext <16 x i1> undef to <16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i64 = sext <16 x i1> undef to <16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i16 = sext <32 x i8> undef to <32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i32 = sext <32 x i8> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i8_v32i64 = sext <32 x i8> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_v32i32 = sext <32 x i16> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i16_v32i64 = sext <32 x i16> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i32_v32i64 = sext <32 x i32> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i8 = sext <32 x i1> undef to <32 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i16 = sext <32 x i1> undef to <32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i32 = sext <32 x i1> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_v32i64 = sext <32 x i1> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_v64i16 = sext <64 x i8> undef to <64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i8_v64i32 = sext <64 x i8> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i8_v64i64 = sext <64 x i8> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i16_v64i32 = sext <64 x i16> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i16_v64i64 = sext <64 x i16> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v64i32_v64i64 = sext <64 x i32> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i8 = sext <64 x i1> undef to <64 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i16 = sext <64 x i1> undef to <64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v64i1_v64i32 = sext <64 x i1> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v64i1_v64i64 = sext <64 x i1> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v128i8_v128i16 = sext <128 x i8> undef to <128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v128i8_v128i32 = sext <128 x i8> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v128i8_v128i64 = sext <128 x i8> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v128i16_v128i32 = sext <128 x i16> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v128i16_v128i64 = sext <128 x i16> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v128i32_v128i64 = sext <128 x i32> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v128i1_v128i8 = sext <128 x i1> undef to <128 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v128i1_v128i16 = sext <128 x i1> undef to <128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v128i1_v128i32 = sext <128 x i1> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v128i1_v128i64 = sext <128 x i1> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v256i8_v256i16 = sext <256 x i8> undef to <256 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v256i8_v256i32 = sext <256 x i8> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %v256i8_v256i64 = sext <256 x i8> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v256i16_v256i32 = sext <256 x i16> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v256i16_v256i64 = sext <256 x i16> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v256i32_v256i64 = sext <256 x i32> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v256i1_v256i8 = sext <256 x i1> undef to <256 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v256i1_v256i16 = sext <256 x i1> undef to <256 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %v256i1_v256i32 = sext <256 x i1> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %v256i1_v256i64 = sext <256 x i1> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_v16i16 = sext <16 x i1> undef to <16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_v16i32 = sext <16 x i1> undef to <16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i1_v16i64 = sext <16 x i1> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i8_v32i16 = sext <32 x i8> undef to <32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i8_v32i32 = sext <32 x i8> undef to <32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i8_v32i64 = sext <32 x i8> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i16_v32i32 = sext <32 x i16> undef to <32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i16_v32i64 = sext <32 x i16> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i32_v32i64 = sext <32 x i32> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_v32i8 = sext <32 x i1> undef to <32 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_v32i16 = sext <32 x i1> undef to <32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32i32 = sext <32 x i1> undef to <32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32i64 = sext <32 x i1> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i16 = sext <64 x i8> undef to <64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i8_v64i32 = sext <64 x i8> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i8_v64i64 = sext <64 x i8> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i16_v64i32 = sext <64 x i16> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i16_v64i64 = sext <64 x i16> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64i64 = sext <64 x i32> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i1_v64i8 = sext <64 x i1> undef to <64 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i1_v64i16 = sext <64 x i1> undef to <64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64i32 = sext <64 x i1> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64i64 = sext <64 x i1> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v128i8_v128i16 = sext <128 x i8> undef to <128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v128i8_v128i32 = sext <128 x i8> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %v128i8_v128i64 = sext <128 x i8> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v128i16_v128i32 = sext <128 x i16> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v128i16_v128i64 = sext <128 x i16> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128i64 = sext <128 x i32> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v128i1_v128i8 = sext <128 x i1> undef to <128 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v128i1_v128i16 = sext <128 x i1> undef to <128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128i32 = sext <128 x i1> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128i64 = sext <128 x i1> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v256i8_v256i16 = sext <256 x i8> undef to <256 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v256i8_v256i32 = sext <256 x i8> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %v256i8_v256i64 = sext <256 x i8> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v256i16_v256i32 = sext <256 x i16> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %v256i16_v256i64 = sext <256 x i16> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %v256i32_v256i64 = sext <256 x i32> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v256i1_v256i8 = sext <256 x i1> undef to <256 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v256i1_v256i16 = sext <256 x i1> undef to <256 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %v256i1_v256i32 = sext <256 x i1> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %v256i1_v256i64 = sext <256 x i1> undef to <256 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i16 = sext <vscale x 1 x i8> undef to <vscale x 1 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i32 = sext <vscale x 1 x i8> undef to <vscale x 1 x i32>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i64 = sext <vscale x 1 x i8> undef to <vscale x 1 x i64>
@@ -96,73 +96,73 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i1_nxv1i64 = sext <vscale x 1 x i1> undef to <vscale x 1 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i16 = sext <vscale x 2 x i8> undef to <vscale x 2 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i32 = sext <vscale x 2 x i8> undef to <vscale x 2 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i64 = sext <vscale x 2 x i8> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i8_nxv2i64 = sext <vscale x 2 x i8> undef to <vscale x 2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i32 = sext <vscale x 2 x i16> undef to <vscale x 2 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i64 = sext <vscale x 2 x i16> undef to <vscale x 2 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32_nxv2i64 = sext <vscale x 2 x i32> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i16_nxv2i64 = sext <vscale x 2 x i16> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i32_nxv2i64 = sext <vscale x 2 x i32> undef to <vscale x 2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i8 = sext <vscale x 2 x i1> undef to <vscale x 2 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i16 = sext <vscale x 2 x i1> undef to <vscale x 2 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i32 = sext <vscale x 2 x i1> undef to <vscale x 2 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i64 = sext <vscale x 2 x i1> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_nxv2i64 = sext <vscale x 2 x i1> undef to <vscale x 2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i16 = sext <vscale x 4 x i8> undef to <vscale x 4 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i32 = sext <vscale x 4 x i8> undef to <vscale x 4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i64 = sext <vscale x 4 x i8> undef to <vscale x 4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i32 = sext <vscale x 4 x i16> undef to <vscale x 4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i64 = sext <vscale x 4 x i16> undef to <vscale x 4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32_nxv4i64 = sext <vscale x 4 x i32> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i8_nxv4i32 = sext <vscale x 4 x i8> undef to <vscale x 4 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i8_nxv4i64 = sext <vscale x 4 x i8> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i16_nxv4i32 = sext <vscale x 4 x i16> undef to <vscale x 4 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i16_nxv4i64 = sext <vscale x 4 x i16> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i32_nxv4i64 = sext <vscale x 4 x i32> undef to <vscale x 4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i8 = sext <vscale x 4 x i1> undef to <vscale x 4 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i16 = sext <vscale x 4 x i1> undef to <vscale x 4 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i32 = sext <vscale x 4 x i1> undef to <vscale x 4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i64 = sext <vscale x 4 x i1> undef to <vscale x 4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i16 = sext <vscale x 8 x i8> undef to <vscale x 8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i32 = sext <vscale x 8 x i8> undef to <vscale x 8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i64 = sext <vscale x 8 x i8> undef to <vscale x 8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i32 = sext <vscale x 8 x i16> undef to <vscale x 8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i64 = sext <vscale x 8 x i16> undef to <vscale x 8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32_nxv8i64 = sext <vscale x 8 x i32> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_nxv4i32 = sext <vscale x 4 x i1> undef to <vscale x 4 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv4i1_nxv4i64 = sext <vscale x 4 x i1> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i8_nxv8i16 = sext <vscale x 8 x i8> undef to <vscale x 8 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i8_nxv8i32 = sext <vscale x 8 x i8> undef to <vscale x 8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i8_nxv8i64 = sext <vscale x 8 x i8> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i16_nxv8i32 = sext <vscale x 8 x i16> undef to <vscale x 8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i16_nxv8i64 = sext <vscale x 8 x i16> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i32_nxv8i64 = sext <vscale x 8 x i32> undef to <vscale x 8 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i8 = sext <vscale x 8 x i1> undef to <vscale x 8 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i16 = sext <vscale x 8 x i1> undef to <vscale x 8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i32 = sext <vscale x 8 x i1> undef to <vscale x 8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i64 = sext <vscale x 8 x i1> undef to <vscale x 8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i16 = sext <vscale x 16 x i8> undef to <vscale x 16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i32 = sext <vscale x 16 x i8> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i8_nxv16i64 = sext <vscale x 16 x i8> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16_nxv16i32 = sext <vscale x 16 x i16> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i16_nxv16i64 = sext <vscale x 16 x i16> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i32_nxv16i64 = sext <vscale x 16 x i32> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i8 = sext <vscale x 16 x i1> undef to <vscale x 16 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i16 = sext <vscale x 16 x i1> undef to <vscale x 16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i32 = sext <vscale x 16 x i1> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_nxv16i64 = sext <vscale x 16 x i1> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8_nxv32i16 = sext <vscale x 32 x i8> undef to <vscale x 32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i8_nxv32i32 = sext <vscale x 32 x i8> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i8_nxv32i64 = sext <vscale x 32 x i8> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i16_nxv32i32 = sext <vscale x 32 x i16> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i16_nxv32i64 = sext <vscale x 32 x i16> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i32_nxv32i64 = sext <vscale x 32 x i32> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i8 = sext <vscale x 32 x i1> undef to <vscale x 32 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i16 = sext <vscale x 32 x i1> undef to <vscale x 32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_nxv32i32 = sext <vscale x 32 x i1> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_nxv32i64 = sext <vscale x 32 x i1> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv64i8_nxv64i16 = sext <vscale x 64 x i8> undef to <vscale x 64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv64i8_nxv64i32 = sext <vscale x 64 x i8> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_nxv8i16 = sext <vscale x 8 x i1> undef to <vscale x 8 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i1_nxv8i32 = sext <vscale x 8 x i1> undef to <vscale x 8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv8i1_nxv8i64 = sext <vscale x 8 x i1> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i8_nxv16i16 = sext <vscale x 16 x i8> undef to <vscale x 16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i8_nxv16i32 = sext <vscale x 16 x i8> undef to <vscale x 16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i8_nxv16i64 = sext <vscale x 16 x i8> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i16_nxv16i32 = sext <vscale x 16 x i16> undef to <vscale x 16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i16_nxv16i64 = sext <vscale x 16 x i16> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i32_nxv16i64 = sext <vscale x 16 x i32> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_nxv16i8 = sext <vscale x 16 x i1> undef to <vscale x 16 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_nxv16i16 = sext <vscale x 16 x i1> undef to <vscale x 16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16i32 = sext <vscale x 16 x i1> undef to <vscale x 16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16i64 = sext <vscale x 16 x i1> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i8_nxv32i16 = sext <vscale x 32 x i8> undef to <vscale x 32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i8_nxv32i32 = sext <vscale x 32 x i8> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i8_nxv32i64 = sext <vscale x 32 x i8> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i16_nxv32i32 = sext <vscale x 32 x i16> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i16_nxv32i64 = sext <vscale x 32 x i16> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32i64 = sext <vscale x 32 x i32> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i1_nxv32i8 = sext <vscale x 32 x i1> undef to <vscale x 32 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv32i1_nxv32i16 = sext <vscale x 32 x i1> undef to <vscale x 32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32i32 = sext <vscale x 32 x i1> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32i64 = sext <vscale x 32 x i1> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv64i8_nxv64i16 = sext <vscale x 64 x i8> undef to <vscale x 64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv64i8_nxv64i32 = sext <vscale x 64 x i8> undef to <vscale x 64 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i8_nxv64i64 = sext <vscale x 64 x i8> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv64i16_nxv64i32 = sext <vscale x 64 x i16> undef to <vscale x 64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %nxv64i16_nxv64i64 = sext <vscale x 64 x i16> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv64i32_nxv64i64 = sext <vscale x 64 x i32> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv64i1_nxv64i8 = sext <vscale x 64 x i1> undef to <vscale x 64 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv64i1_nxv64i16 = sext <vscale x 64 x i1> undef to <vscale x 64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv64i1_nxv64i32 = sext <vscale x 64 x i1> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv64i16_nxv64i32 = sext <vscale x 64 x i16> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %nxv64i16_nxv64i64 = sext <vscale x 64 x i16> undef to <vscale x 64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %nxv64i32_nxv64i64 = sext <vscale x 64 x i32> undef to <vscale x 64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv64i1_nxv64i8 = sext <vscale x 64 x i1> undef to <vscale x 64 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv64i1_nxv64i16 = sext <vscale x 64 x i1> undef to <vscale x 64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64i32 = sext <vscale x 64 x i1> undef to <vscale x 64 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i1_nxv64i64 = sext <vscale x 64 x i1> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv128i8_nxv128i16 = sext <vscale x 128 x i8> undef to <vscale x 128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %nxv128i8_nxv128i32 = sext <vscale x 128 x i8> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv128i8_nxv128i16 = sext <vscale x 128 x i8> undef to <vscale x 128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv128i8_nxv128i32 = sext <vscale x 128 x i8> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i8_nxv128i128 = sext <vscale x 128 x i8> undef to <vscale x 128 x i128>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv128i16_nxv128i32 = sext <vscale x 128 x i16> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv128i16_nxv128i32 = sext <vscale x 128 x i16> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i16_nxv128i128 = sext <vscale x 128 x i16> undef to <vscale x 128 x i128>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i32_nxv128i128 = sext <vscale x 128 x i32> undef to <vscale x 128 x i128>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv128i1_nxv128i8 = sext <vscale x 128 x i1> undef to <vscale x 128 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv128i1_nxv128i16 = sext <vscale x 128 x i1> undef to <vscale x 128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %nxv128i1_nxv128i32 = sext <vscale x 128 x i1> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %nxv128i1_nxv128i8 = sext <vscale x 128 x i1> undef to <vscale x 128 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %nxv128i1_nxv128i16 = sext <vscale x 128 x i1> undef to <vscale x 128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %nxv128i1_nxv128i32 = sext <vscale x 128 x i1> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i1_nxv128i128 = sext <vscale x 128 x i1> undef to <vscale x 128 x i128>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -179,74 +179,74 @@ define void @sext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i1_v2i64 = sext <2 x i1> undef to <2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i16 = sext <4 x i8> undef to <4 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i32 = sext <4 x i8> undef to <4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i64 = sext <4 x i8> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_v4i64 = sext <4 x i8> undef to <4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i32 = sext <4 x i16> undef to <4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i64 = sext <4 x i16> undef to <4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_v4i64 = sext <4 x i32> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16_v4i64 = sext <4 x i16> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32_v4i64 = sext <4 x i32> undef to <4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i8 = sext <4 x i1> undef to <4 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i16 = sext <4 x i1> undef to <4 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i32 = sext <4 x i1> undef to <4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i64 = sext <4 x i1> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_v4i64 = sext <4 x i1> undef to <4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i16 = sext <8 x i8> undef to <8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i32 = sext <8 x i8> undef to <8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i64 = sext <8 x i8> undef to <8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i32 = sext <8 x i16> undef to <8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i64 = sext <8 x i16> undef to <8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_v8i64 = sext <8 x i32> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_v8i32 = sext <8 x i8> undef to <8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_v8i64 = sext <8 x i8> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16_v8i32 = sext <8 x i16> undef to <8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i16_v8i64 = sext <8 x i16> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32_v8i64 = sext <8 x i32> undef to <8 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i8 = sext <8 x i1> undef to <8 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i16 = sext <8 x i1> undef to <8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i32 = sext <8 x i1> undef to <8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i64 = sext <8 x i1> undef to <8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i16 = sext <16 x i8> undef to <16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i32 = sext <16 x i8> undef to <16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i64 = sext <16 x i8> undef to <16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i32 = sext <16 x i16> undef to <16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i64 = sext <16 x i16> undef to <16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_v16i64 = sext <16 x i32> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_v8i32 = sext <8 x i1> undef to <8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i1_v8i64 = sext <8 x i1> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8_v16i16 = sext <16 x i8> undef to <16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i8_v16i32 = sext <16 x i8> undef to <16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_v16i64 = sext <16 x i8> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16_v16i32 = sext <16 x i16> undef to <16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i16_v16i64 = sext <16 x i16> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32_v16i64 = sext <16 x i32> undef to <16 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i8 = sext <16 x i1> undef to <16 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i16 = sext <16 x i1> undef to <16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i32 = sext <16 x i1> undef to <16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i64 = sext <16 x i1> undef to <16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i16 = sext <32 x i8> undef to <32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i32 = sext <32 x i8> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i8_v32i64 = sext <32 x i8> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_v32i32 = sext <32 x i16> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i16_v32i64 = sext <32 x i16> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i32_v32i64 = sext <32 x i32> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i8 = sext <32 x i1> undef to <32 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i16 = sext <32 x i1> undef to <32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i32 = sext <32 x i1> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_v32i64 = sext <32 x i1> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_v64i16 = sext <64 x i8> undef to <64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i8_v64i32 = sext <64 x i8> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i8_v64i64 = sext <64 x i8> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i16_v64i32 = sext <64 x i16> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i16_v64i64 = sext <64 x i16> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v64i32_v64i64 = sext <64 x i32> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i8 = sext <64 x i1> undef to <64 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i16 = sext <64 x i1> undef to <64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v64i1_v64i32 = sext <64 x i1> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v64i1_v64i64 = sext <64 x i1> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v128i8_v128i16 = sext <128 x i8> undef to <128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v128i8_v128i32 = sext <128 x i8> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v128i8_v128i64 = sext <128 x i8> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v128i16_v128i32 = sext <128 x i16> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v128i16_v128i64 = sext <128 x i16> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v128i32_v128i64 = sext <128 x i32> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v128i1_v128i8 = sext <128 x i1> undef to <128 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v128i1_v128i16 = sext <128 x i1> undef to <128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v128i1_v128i32 = sext <128 x i1> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v128i1_v128i64 = sext <128 x i1> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v256i8_v256i16 = sext <256 x i8> undef to <256 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v256i8_v256i32 = sext <256 x i8> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %v256i8_v256i64 = sext <256 x i8> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v256i16_v256i32 = sext <256 x i16> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v256i16_v256i64 = sext <256 x i16> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v256i32_v256i64 = sext <256 x i32> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v256i1_v256i8 = sext <256 x i1> undef to <256 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v256i1_v256i16 = sext <256 x i1> undef to <256 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %v256i1_v256i32 = sext <256 x i1> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %v256i1_v256i64 = sext <256 x i1> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_v16i16 = sext <16 x i1> undef to <16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_v16i32 = sext <16 x i1> undef to <16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i1_v16i64 = sext <16 x i1> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i8_v32i16 = sext <32 x i8> undef to <32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i8_v32i32 = sext <32 x i8> undef to <32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i8_v32i64 = sext <32 x i8> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i16_v32i32 = sext <32 x i16> undef to <32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i16_v32i64 = sext <32 x i16> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i32_v32i64 = sext <32 x i32> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_v32i8 = sext <32 x i1> undef to <32 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_v32i16 = sext <32 x i1> undef to <32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32i32 = sext <32 x i1> undef to <32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32i64 = sext <32 x i1> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i16 = sext <64 x i8> undef to <64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i8_v64i32 = sext <64 x i8> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i8_v64i64 = sext <64 x i8> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i16_v64i32 = sext <64 x i16> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i16_v64i64 = sext <64 x i16> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64i64 = sext <64 x i32> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i1_v64i8 = sext <64 x i1> undef to <64 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i1_v64i16 = sext <64 x i1> undef to <64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64i32 = sext <64 x i1> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64i64 = sext <64 x i1> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v128i8_v128i16 = sext <128 x i8> undef to <128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v128i8_v128i32 = sext <128 x i8> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %v128i8_v128i64 = sext <128 x i8> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v128i16_v128i32 = sext <128 x i16> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v128i16_v128i64 = sext <128 x i16> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128i64 = sext <128 x i32> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v128i1_v128i8 = sext <128 x i1> undef to <128 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v128i1_v128i16 = sext <128 x i1> undef to <128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128i32 = sext <128 x i1> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128i64 = sext <128 x i1> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v256i8_v256i16 = sext <256 x i8> undef to <256 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v256i8_v256i32 = sext <256 x i8> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %v256i8_v256i64 = sext <256 x i8> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v256i16_v256i32 = sext <256 x i16> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %v256i16_v256i64 = sext <256 x i16> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %v256i32_v256i64 = sext <256 x i32> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v256i1_v256i8 = sext <256 x i1> undef to <256 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v256i1_v256i16 = sext <256 x i1> undef to <256 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %v256i1_v256i32 = sext <256 x i1> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %v256i1_v256i64 = sext <256 x i1> undef to <256 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i16 = sext <vscale x 1 x i8> undef to <vscale x 1 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i32 = sext <vscale x 1 x i8> undef to <vscale x 1 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i64 = sext <vscale x 1 x i8> undef to <vscale x 1 x i64>
@@ -259,73 +259,73 @@ define void @sext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i1_nxv1i64 = sext <vscale x 1 x i1> undef to <vscale x 1 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i16 = sext <vscale x 2 x i8> undef to <vscale x 2 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i32 = sext <vscale x 2 x i8> undef to <vscale x 2 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i64 = sext <vscale x 2 x i8> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i8_nxv2i64 = sext <vscale x 2 x i8> undef to <vscale x 2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i32 = sext <vscale x 2 x i16> undef to <vscale x 2 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i64 = sext <vscale x 2 x i16> undef to <vscale x 2 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32_nxv2i64 = sext <vscale x 2 x i32> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i16_nxv2i64 = sext <vscale x 2 x i16> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i32_nxv2i64 = sext <vscale x 2 x i32> undef to <vscale x 2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i8 = sext <vscale x 2 x i1> undef to <vscale x 2 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i16 = sext <vscale x 2 x i1> undef to <vscale x 2 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i32 = sext <vscale x 2 x i1> undef to <vscale x 2 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i64 = sext <vscale x 2 x i1> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_nxv2i64 = sext <vscale x 2 x i1> undef to <vscale x 2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i16 = sext <vscale x 4 x i8> undef to <vscale x 4 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i32 = sext <vscale x 4 x i8> undef to <vscale x 4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i64 = sext <vscale x 4 x i8> undef to <vscale x 4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i32 = sext <vscale x 4 x i16> undef to <vscale x 4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i64 = sext <vscale x 4 x i16> undef to <vscale x 4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32_nxv4i64 = sext <vscale x 4 x i32> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i8_nxv4i32 = sext <vscale x 4 x i8> undef to <vscale x 4 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i8_nxv4i64 = sext <vscale x 4 x i8> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i16_nxv4i32 = sext <vscale x 4 x i16> undef to <vscale x 4 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i16_nxv4i64 = sext <vscale x 4 x i16> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i32_nxv4i64 = sext <vscale x 4 x i32> undef to <vscale x 4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i8 = sext <vscale x 4 x i1> undef to <vscale x 4 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i16 = sext <vscale x 4 x i1> undef to <vscale x 4 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i32 = sext <vscale x 4 x i1> undef to <vscale x 4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i64 = sext <vscale x 4 x i1> undef to <vscale x 4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i16 = sext <vscale x 8 x i8> undef to <vscale x 8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i32 = sext <vscale x 8 x i8> undef to <vscale x 8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i64 = sext <vscale x 8 x i8> undef to <vscale x 8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i32 = sext <vscale x 8 x i16> undef to <vscale x 8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i64 = sext <vscale x 8 x i16> undef to <vscale x 8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32_nxv8i64 = sext <vscale x 8 x i32> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_nxv4i32 = sext <vscale x 4 x i1> undef to <vscale x 4 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv4i1_nxv4i64 = sext <vscale x 4 x i1> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i8_nxv8i16 = sext <vscale x 8 x i8> undef to <vscale x 8 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i8_nxv8i32 = sext <vscale x 8 x i8> undef to <vscale x 8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i8_nxv8i64 = sext <vscale x 8 x i8> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i16_nxv8i32 = sext <vscale x 8 x i16> undef to <vscale x 8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i16_nxv8i64 = sext <vscale x 8 x i16> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i32_nxv8i64 = sext <vscale x 8 x i32> undef to <vscale x 8 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i8 = sext <vscale x 8 x i1> undef to <vscale x 8 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i16 = sext <vscale x 8 x i1> undef to <vscale x 8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i32 = sext <vscale x 8 x i1> undef to <vscale x 8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i64 = sext <vscale x 8 x i1> undef to <vscale x 8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i16 = sext <vscale x 16 x i8> undef to <vscale x 16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i32 = sext <vscale x 16 x i8> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i8_nxv16i64 = sext <vscale x 16 x i8> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16_nxv16i32 = sext <vscale x 16 x i16> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i16_nxv16i64 = sext <vscale x 16 x i16> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i32_nxv16i64 = sext <vscale x 16 x i32> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i8 = sext <vscale x 16 x i1> undef to <vscale x 16 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i16 = sext <vscale x 16 x i1> undef to <vscale x 16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i32 = sext <vscale x 16 x i1> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_nxv16i64 = sext <vscale x 16 x i1> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8_nxv32i16 = sext <vscale x 32 x i8> undef to <vscale x 32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i8_nxv32i32 = sext <vscale x 32 x i8> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i8_nxv32i64 = sext <vscale x 32 x i8> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i16_nxv32i32 = sext <vscale x 32 x i16> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i16_nxv32i64 = sext <vscale x 32 x i16> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i32_nxv32i64 = sext <vscale x 32 x i32> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i8 = sext <vscale x 32 x i1> undef to <vscale x 32 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i16 = sext <vscale x 32 x i1> undef to <vscale x 32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_nxv32i32 = sext <vscale x 32 x i1> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_nxv32i64 = sext <vscale x 32 x i1> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv64i8_nxv64i16 = sext <vscale x 64 x i8> undef to <vscale x 64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv64i8_nxv64i32 = sext <vscale x 64 x i8> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %nxv64i8_nxv64i64 = sext <vscale x 64 x i8> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv64i16_nxv64i32 = sext <vscale x 64 x i16> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %nxv64i16_nxv64i64 = sext <vscale x 64 x i16> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv64i32_nxv64i64 = sext <vscale x 64 x i32> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv64i1_nxv64i8 = sext <vscale x 64 x i1> undef to <vscale x 64 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv64i1_nxv64i16 = sext <vscale x 64 x i1> undef to <vscale x 64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv64i1_nxv64i32 = sext <vscale x 64 x i1> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %nxv64i1_nxv64i64 = sext <vscale x 64 x i1> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv128i8_nxv128i16 = sext <vscale x 128 x i8> undef to <vscale x 128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %nxv128i8_nxv128i32 = sext <vscale x 128 x i8> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_nxv8i16 = sext <vscale x 8 x i1> undef to <vscale x 8 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i1_nxv8i32 = sext <vscale x 8 x i1> undef to <vscale x 8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv8i1_nxv8i64 = sext <vscale x 8 x i1> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i8_nxv16i16 = sext <vscale x 16 x i8> undef to <vscale x 16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i8_nxv16i32 = sext <vscale x 16 x i8> undef to <vscale x 16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i8_nxv16i64 = sext <vscale x 16 x i8> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i16_nxv16i32 = sext <vscale x 16 x i16> undef to <vscale x 16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i16_nxv16i64 = sext <vscale x 16 x i16> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i32_nxv16i64 = sext <vscale x 16 x i32> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_nxv16i8 = sext <vscale x 16 x i1> undef to <vscale x 16 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_nxv16i16 = sext <vscale x 16 x i1> undef to <vscale x 16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16i32 = sext <vscale x 16 x i1> undef to <vscale x 16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16i64 = sext <vscale x 16 x i1> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i8_nxv32i16 = sext <vscale x 32 x i8> undef to <vscale x 32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i8_nxv32i32 = sext <vscale x 32 x i8> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i8_nxv32i64 = sext <vscale x 32 x i8> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i16_nxv32i32 = sext <vscale x 32 x i16> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i16_nxv32i64 = sext <vscale x 32 x i16> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32i64 = sext <vscale x 32 x i32> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i1_nxv32i8 = sext <vscale x 32 x i1> undef to <vscale x 32 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv32i1_nxv32i16 = sext <vscale x 32 x i1> undef to <vscale x 32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32i32 = sext <vscale x 32 x i1> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32i64 = sext <vscale x 32 x i1> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv64i8_nxv64i16 = sext <vscale x 64 x i8> undef to <vscale x 64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv64i8_nxv64i32 = sext <vscale x 64 x i8> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %nxv64i8_nxv64i64 = sext <vscale x 64 x i8> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv64i16_nxv64i32 = sext <vscale x 64 x i16> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv64i16_nxv64i64 = sext <vscale x 64 x i16> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv64i32_nxv64i64 = sext <vscale x 64 x i32> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv64i1_nxv64i8 = sext <vscale x 64 x i1> undef to <vscale x 64 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv64i1_nxv64i16 = sext <vscale x 64 x i1> undef to <vscale x 64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64i32 = sext <vscale x 64 x i1> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %nxv64i1_nxv64i64 = sext <vscale x 64 x i1> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv128i8_nxv128i16 = sext <vscale x 128 x i8> undef to <vscale x 128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv128i8_nxv128i32 = sext <vscale x 128 x i8> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i8_nxv128i128 = sext <vscale x 128 x i8> undef to <vscale x 128 x i128>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv128i16_nxv128i32 = sext <vscale x 128 x i16> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv128i16_nxv128i32 = sext <vscale x 128 x i16> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i16_nxv128i128 = sext <vscale x 128 x i16> undef to <vscale x 128 x i128>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i32_nxv128i128 = sext <vscale x 128 x i32> undef to <vscale x 128 x i128>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv128i1_nxv128i8 = sext <vscale x 128 x i1> undef to <vscale x 128 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv128i1_nxv128i16 = sext <vscale x 128 x i1> undef to <vscale x 128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %nxv128i1_nxv128i32 = sext <vscale x 128 x i1> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %nxv128i1_nxv128i8 = sext <vscale x 128 x i1> undef to <vscale x 128 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %nxv128i1_nxv128i16 = sext <vscale x 128 x i1> undef to <vscale x 128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %nxv128i1_nxv128i32 = sext <vscale x 128 x i1> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i1_nxv128i128 = sext <vscale x 128 x i1> undef to <vscale x 128 x i128>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -522,74 +522,74 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i1_v2i64 = zext <2 x i1> undef to <2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i16 = zext <4 x i8> undef to <4 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i32 = zext <4 x i8> undef to <4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i64 = zext <4 x i8> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_v4i64 = zext <4 x i8> undef to <4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i32 = zext <4 x i16> undef to <4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i64 = zext <4 x i16> undef to <4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_v4i64 = zext <4 x i32> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16_v4i64 = zext <4 x i16> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32_v4i64 = zext <4 x i32> undef to <4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i8 = zext <4 x i1> undef to <4 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i16 = zext <4 x i1> undef to <4 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i32 = zext <4 x i1> undef to <4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i64 = zext <4 x i1> undef to <4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_v4i64 = zext <4 x i1> undef to <4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i16 = zext <8 x i8> undef to <8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i32 = zext <8 x i8> undef to <8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i64 = zext <8 x i8> undef to <8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i32 = zext <8 x i16> undef to <8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i64 = zext <8 x i16> undef to <8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_v8i64 = zext <8 x i32> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_v8i32 = zext <8 x i8> undef to <8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_v8i64 = zext <8 x i8> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16_v8i32 = zext <8 x i16> undef to <8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i16_v8i64 = zext <8 x i16> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32_v8i64 = zext <8 x i32> undef to <8 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i8 = zext <8 x i1> undef to <8 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i16 = zext <8 x i1> undef to <8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i32 = zext <8 x i1> undef to <8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i64 = zext <8 x i1> undef to <8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i16 = zext <16 x i8> undef to <16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i32 = zext <16 x i8> undef to <16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i64 = zext <16 x i8> undef to <16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i32 = zext <16 x i16> undef to <16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i64 = zext <16 x i16> undef to <16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_v16i64 = zext <16 x i32> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_v8i32 = zext <8 x i1> undef to <8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i1_v8i64 = zext <8 x i1> undef to <8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8_v16i16 = zext <16 x i8> undef to <16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i8_v16i32 = zext <16 x i8> undef to <16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_v16i64 = zext <16 x i8> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16_v16i32 = zext <16 x i16> undef to <16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i16_v16i64 = zext <16 x i16> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32_v16i64 = zext <16 x i32> undef to <16 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i8 = zext <16 x i1> undef to <16 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i16 = zext <16 x i1> undef to <16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i32 = zext <16 x i1> undef to <16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i64 = zext <16 x i1> undef to <16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i16 = zext <32 x i8> undef to <32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i32 = zext <32 x i8> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i8_v32i64 = zext <32 x i8> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_v32i32 = zext <32 x i16> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i16_v32i64 = zext <32 x i16> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i32_v32i64 = zext <32 x i32> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i8 = zext <32 x i1> undef to <32 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i16 = zext <32 x i1> undef to <32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i32 = zext <32 x i1> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_v32i64 = zext <32 x i1> undef to <32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_v64i16 = zext <64 x i8> undef to <64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i8_v64i32 = zext <64 x i8> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i8_v64i64 = zext <64 x i8> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i16_v64i32 = zext <64 x i16> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i16_v64i64 = zext <64 x i16> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v64i32_v64i64 = zext <64 x i32> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i8 = zext <64 x i1> undef to <64 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i16 = zext <64 x i1> undef to <64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v64i1_v64i32 = zext <64 x i1> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v64i1_v64i64 = zext <64 x i1> undef to <64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v128i8_v128i16 = zext <128 x i8> undef to <128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v128i8_v128i32 = zext <128 x i8> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v128i8_v128i64 = zext <128 x i8> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v128i16_v128i32 = zext <128 x i16> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v128i16_v128i64 = zext <128 x i16> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v128i32_v128i64 = zext <128 x i32> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v128i1_v128i8 = zext <128 x i1> undef to <128 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v128i1_v128i16 = zext <128 x i1> undef to <128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v128i1_v128i32 = zext <128 x i1> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v128i1_v128i64 = zext <128 x i1> undef to <128 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v256i8_v256i16 = zext <256 x i8> undef to <256 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v256i8_v256i32 = zext <256 x i8> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %v256i8_v256i64 = zext <256 x i8> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v256i16_v256i32 = zext <256 x i16> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v256i16_v256i64 = zext <256 x i16> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v256i32_v256i64 = zext <256 x i32> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v256i1_v256i8 = zext <256 x i1> undef to <256 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v256i1_v256i16 = zext <256 x i1> undef to <256 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %v256i1_v256i32 = zext <256 x i1> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %v256i1_v256i64 = zext <256 x i1> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_v16i16 = zext <16 x i1> undef to <16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_v16i32 = zext <16 x i1> undef to <16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i1_v16i64 = zext <16 x i1> undef to <16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i8_v32i16 = zext <32 x i8> undef to <32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i8_v32i32 = zext <32 x i8> undef to <32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i8_v32i64 = zext <32 x i8> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i16_v32i32 = zext <32 x i16> undef to <32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i16_v32i64 = zext <32 x i16> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i32_v32i64 = zext <32 x i32> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_v32i8 = zext <32 x i1> undef to <32 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_v32i16 = zext <32 x i1> undef to <32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32i32 = zext <32 x i1> undef to <32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32i64 = zext <32 x i1> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i16 = zext <64 x i8> undef to <64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i8_v64i32 = zext <64 x i8> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i8_v64i64 = zext <64 x i8> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i16_v64i32 = zext <64 x i16> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i16_v64i64 = zext <64 x i16> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64i64 = zext <64 x i32> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i1_v64i8 = zext <64 x i1> undef to <64 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i1_v64i16 = zext <64 x i1> undef to <64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64i32 = zext <64 x i1> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64i64 = zext <64 x i1> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v128i8_v128i16 = zext <128 x i8> undef to <128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v128i8_v128i32 = zext <128 x i8> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %v128i8_v128i64 = zext <128 x i8> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v128i16_v128i32 = zext <128 x i16> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v128i16_v128i64 = zext <128 x i16> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128i64 = zext <128 x i32> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v128i1_v128i8 = zext <128 x i1> undef to <128 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v128i1_v128i16 = zext <128 x i1> undef to <128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128i32 = zext <128 x i1> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128i64 = zext <128 x i1> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v256i8_v256i16 = zext <256 x i8> undef to <256 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v256i8_v256i32 = zext <256 x i8> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %v256i8_v256i64 = zext <256 x i8> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v256i16_v256i32 = zext <256 x i16> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %v256i16_v256i64 = zext <256 x i16> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %v256i32_v256i64 = zext <256 x i32> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v256i1_v256i8 = zext <256 x i1> undef to <256 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v256i1_v256i16 = zext <256 x i1> undef to <256 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %v256i1_v256i32 = zext <256 x i1> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %v256i1_v256i64 = zext <256 x i1> undef to <256 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i16 = zext <vscale x 1 x i8> undef to <vscale x 1 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i32 = zext <vscale x 1 x i8> undef to <vscale x 1 x i32>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i64 = zext <vscale x 1 x i8> undef to <vscale x 1 x i64>
@@ -602,73 +602,73 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i1_nxv1i64 = zext <vscale x 1 x i1> undef to <vscale x 1 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i16 = zext <vscale x 2 x i8> undef to <vscale x 2 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i32 = zext <vscale x 2 x i8> undef to <vscale x 2 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i64 = zext <vscale x 2 x i8> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i8_nxv2i64 = zext <vscale x 2 x i8> undef to <vscale x 2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i32 = zext <vscale x 2 x i16> undef to <vscale x 2 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i64 = zext <vscale x 2 x i16> undef to <vscale x 2 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32_nxv2i64 = zext <vscale x 2 x i32> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i16_nxv2i64 = zext <vscale x 2 x i16> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i32_nxv2i64 = zext <vscale x 2 x i32> undef to <vscale x 2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i8 = zext <vscale x 2 x i1> undef to <vscale x 2 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i16 = zext <vscale x 2 x i1> undef to <vscale x 2 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i32 = zext <vscale x 2 x i1> undef to <vscale x 2 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i64 = zext <vscale x 2 x i1> undef to <vscale x 2 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_nxv2i64 = zext <vscale x 2 x i1> undef to <vscale x 2 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i16 = zext <vscale x 4 x i8> undef to <vscale x 4 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i32 = zext <vscale x 4 x i8> undef to <vscale x 4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i64 = zext <vscale x 4 x i8> undef to <vscale x 4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i32 = zext <vscale x 4 x i16> undef to <vscale x 4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i64 = zext <vscale x 4 x i16> undef to <vscale x 4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32_nxv4i64 = zext <vscale x 4 x i32> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i8_nxv4i32 = zext <vscale x 4 x i8> undef to <vscale x 4 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i8_nxv4i64 = zext <vscale x 4 x i8> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i16_nxv4i32 = zext <vscale x 4 x i16> undef to <vscale x 4 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i16_nxv4i64 = zext <vscale x 4 x i16> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i32_nxv4i64 = zext <vscale x 4 x i32> undef to <vscale x 4 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i8 = zext <vscale x 4 x i1> undef to <vscale x 4 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i16 = zext <vscale x 4 x i1> undef to <vscale x 4 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i32 = zext <vscale x 4 x i1> undef to <vscale x 4 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i64 = zext <vscale x 4 x i1> undef to <vscale x 4 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i16 = zext <vscale x 8 x i8> undef to <vscale x 8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i32 = zext <vscale x 8 x i8> undef to <vscale x 8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i64 = zext <vscale x 8 x i8> undef to <vscale x 8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i32 = zext <vscale x 8 x i16> undef to <vscale x 8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i64 = zext <vscale x 8 x i16> undef to <vscale x 8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32_nxv8i64 = zext <vscale x 8 x i32> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_nxv4i32 = zext <vscale x 4 x i1> undef to <vscale x 4 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv4i1_nxv4i64 = zext <vscale x 4 x i1> undef to <vscale x 4 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i8_nxv8i16 = zext <vscale x 8 x i8> undef to <vscale x 8 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i8_nxv8i32 = zext <vscale x 8 x i8> undef to <vscale x 8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i8_nxv8i64 = zext <vscale x 8 x i8> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i16_nxv8i32 = zext <vscale x 8 x i16> undef to <vscale x 8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i16_nxv8i64 = zext <vscale x 8 x i16> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i32_nxv8i64 = zext <vscale x 8 x i32> undef to <vscale x 8 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i8 = zext <vscale x 8 x i1> undef to <vscale x 8 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i16 = zext <vscale x 8 x i1> undef to <vscale x 8 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i32 = zext <vscale x 8 x i1> undef to <vscale x 8 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i64 = zext <vscale x 8 x i1> undef to <vscale x 8 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i16 = zext <vscale x 16 x i8> undef to <vscale x 16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i32 = zext <vscale x 16 x i8> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i8_nxv16i64 = zext <vscale x 16 x i8> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16_nxv16i32 = zext <vscale x 16 x i16> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i16_nxv16i64 = zext <vscale x 16 x i16> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i32_nxv16i64 = zext <vscale x 16 x i32> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i8 = zext <vscale x 16 x i1> undef to <vscale x 16 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i16 = zext <vscale x 16 x i1> undef to <vscale x 16 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i32 = zext <vscale x 16 x i1> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_nxv16i64 = zext <vscale x 16 x i1> undef to <vscale x 16 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8_nxv32i16 = zext <vscale x 32 x i8> undef to <vscale x 32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i8_nxv32i32 = zext <vscale x 32 x i8> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i8_nxv32i64 = zext <vscale x 32 x i8> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i16_nxv32i32 = zext <vscale x 32 x i16> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i16_nxv32i64 = zext <vscale x 32 x i16> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i32_nxv32i64 = zext <vscale x 32 x i32> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i8 = zext <vscale x 32 x i1> undef to <vscale x 32 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i16 = zext <vscale x 32 x i1> undef to <vscale x 32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_nxv32i32 = zext <vscale x 32 x i1> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_nxv32i64 = zext <vscale x 32 x i1> undef to <vscale x 32 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv64i8_nxv64i16 = zext <vscale x 64 x i8> undef to <vscale x 64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv64i8_nxv64i32 = zext <vscale x 64 x i8> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_nxv8i16 = zext <vscale x 8 x i1> undef to <vscale x 8 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i1_nxv8i32 = zext <vscale x 8 x i1> undef to <vscale x 8 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv8i1_nxv8i64 = zext <vscale x 8 x i1> undef to <vscale x 8 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i8_nxv16i16 = zext <vscale x 16 x i8> undef to <vscale x 16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i8_nxv16i32 = zext <vscale x 16 x i8> undef to <vscale x 16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i8_nxv16i64 = zext <vscale x 16 x i8> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i16_nxv16i32 = zext <vscale x 16 x i16> undef to <vscale x 16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i16_nxv16i64 = zext <vscale x 16 x i16> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i32_nxv16i64 = zext <vscale x 16 x i32> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_nxv16i8 = zext <vscale x 16 x i1> undef to <vscale x 16 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_nxv16i16 = zext <vscale x 16 x i1> undef to <vscale x 16 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16i32 = zext <vscale x 16 x i1> undef to <vscale x 16 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16i64 = zext <vscale x 16 x i1> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i8_nxv32i16 = zext <vscale x 32 x i8> undef to <vscale x 32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i8_nxv32i32 = zext <vscale x 32 x i8> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i8_nxv32i64 = zext <vscale x 32 x i8> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i16_nxv32i32 = zext <vscale x 32 x i16> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i16_nxv32i64 = zext <vscale x 32 x i16> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32i64 = zext <vscale x 32 x i32> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i1_nxv32i8 = zext <vscale x 32 x i1> undef to <vscale x 32 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv32i1_nxv32i16 = zext <vscale x 32 x i1> undef to <vscale x 32 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32i32 = zext <vscale x 32 x i1> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32i64 = zext <vscale x 32 x i1> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv64i8_nxv64i16 = zext <vscale x 64 x i8> undef to <vscale x 64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv64i8_nxv64i32 = zext <vscale x 64 x i8> undef to <vscale x 64 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i8_nxv64i64 = zext <vscale x 64 x i8> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv64i16_nxv64i32 = zext <vscale x 64 x i16> undef to <vscale x 64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %nxv64i16_nxv64i64 = zext <vscale x 64 x i16> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv64i32_nxv64i64 = zext <vscale x 64 x i32> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv64i1_nxv64i8 = zext <vscale x 64 x i1> undef to <vscale x 64 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv64i1_nxv64i16 = zext <vscale x 64 x i1> undef to <vscale x 64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv64i1_nxv64i32 = zext <vscale x 64 x i1> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv64i16_nxv64i32 = zext <vscale x 64 x i16> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %nxv64i16_nxv64i64 = zext <vscale x 64 x i16> undef to <vscale x 64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %nxv64i32_nxv64i64 = zext <vscale x 64 x i32> undef to <vscale x 64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv64i1_nxv64i8 = zext <vscale x 64 x i1> undef to <vscale x 64 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv64i1_nxv64i16 = zext <vscale x 64 x i1> undef to <vscale x 64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64i32 = zext <vscale x 64 x i1> undef to <vscale x 64 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i1_nxv64i64 = zext <vscale x 64 x i1> undef to <vscale x 64 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv128i8_nxv128i16 = zext <vscale x 128 x i8> undef to <vscale x 128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %nxv128i8_nxv128i32 = zext <vscale x 128 x i8> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv128i8_nxv128i16 = zext <vscale x 128 x i8> undef to <vscale x 128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv128i8_nxv128i32 = zext <vscale x 128 x i8> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i8_nxv128i128 = zext <vscale x 128 x i8> undef to <vscale x 128 x i128>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv128i16_nxv128i32 = zext <vscale x 128 x i16> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv128i16_nxv128i32 = zext <vscale x 128 x i16> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i16_nxv128i128 = zext <vscale x 128 x i16> undef to <vscale x 128 x i128>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i32_nxv128i128 = zext <vscale x 128 x i32> undef to <vscale x 128 x i128>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv128i1_nxv128i8 = zext <vscale x 128 x i1> undef to <vscale x 128 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv128i1_nxv128i16 = zext <vscale x 128 x i1> undef to <vscale x 128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %nxv128i1_nxv128i32 = zext <vscale x 128 x i1> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %nxv128i1_nxv128i8 = zext <vscale x 128 x i1> undef to <vscale x 128 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %nxv128i1_nxv128i16 = zext <vscale x 128 x i1> undef to <vscale x 128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %nxv128i1_nxv128i32 = zext <vscale x 128 x i1> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i1_nxv128i128 = zext <vscale x 128 x i1> undef to <vscale x 128 x i128>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -685,74 +685,74 @@ define void @zext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i1_v2i64 = zext <2 x i1> undef to <2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i16 = zext <4 x i8> undef to <4 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i32 = zext <4 x i8> undef to <4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_v4i64 = zext <4 x i8> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_v4i64 = zext <4 x i8> undef to <4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i32 = zext <4 x i16> undef to <4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4i64 = zext <4 x i16> undef to <4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_v4i64 = zext <4 x i32> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16_v4i64 = zext <4 x i16> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32_v4i64 = zext <4 x i32> undef to <4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i8 = zext <4 x i1> undef to <4 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i16 = zext <4 x i1> undef to <4 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i32 = zext <4 x i1> undef to <4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i1_v4i64 = zext <4 x i1> undef to <4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_v4i64 = zext <4 x i1> undef to <4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i16 = zext <8 x i8> undef to <8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i32 = zext <8 x i8> undef to <8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_v8i64 = zext <8 x i8> undef to <8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i32 = zext <8 x i16> undef to <8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_v8i64 = zext <8 x i16> undef to <8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_v8i64 = zext <8 x i32> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_v8i32 = zext <8 x i8> undef to <8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_v8i64 = zext <8 x i8> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16_v8i32 = zext <8 x i16> undef to <8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i16_v8i64 = zext <8 x i16> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32_v8i64 = zext <8 x i32> undef to <8 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i8 = zext <8 x i1> undef to <8 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i16 = zext <8 x i1> undef to <8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i32 = zext <8 x i1> undef to <8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i1_v8i64 = zext <8 x i1> undef to <8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i16 = zext <16 x i8> undef to <16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i32 = zext <16 x i8> undef to <16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_v16i64 = zext <16 x i8> undef to <16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i32 = zext <16 x i16> undef to <16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_v16i64 = zext <16 x i16> undef to <16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32_v16i64 = zext <16 x i32> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_v8i32 = zext <8 x i1> undef to <8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i1_v8i64 = zext <8 x i1> undef to <8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8_v16i16 = zext <16 x i8> undef to <16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i8_v16i32 = zext <16 x i8> undef to <16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_v16i64 = zext <16 x i8> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16_v16i32 = zext <16 x i16> undef to <16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i16_v16i64 = zext <16 x i16> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32_v16i64 = zext <16 x i32> undef to <16 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i8 = zext <16 x i1> undef to <16 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i16 = zext <16 x i1> undef to <16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i32 = zext <16 x i1> undef to <16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i1_v16i64 = zext <16 x i1> undef to <16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i16 = zext <32 x i8> undef to <32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8_v32i32 = zext <32 x i8> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i8_v32i64 = zext <32 x i8> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i16_v32i32 = zext <32 x i16> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i16_v32i64 = zext <32 x i16> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i32_v32i64 = zext <32 x i32> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i8 = zext <32 x i1> undef to <32 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i16 = zext <32 x i1> undef to <32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i1_v32i32 = zext <32 x i1> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_v32i64 = zext <32 x i1> undef to <32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i8_v64i16 = zext <64 x i8> undef to <64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i8_v64i32 = zext <64 x i8> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i8_v64i64 = zext <64 x i8> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v64i16_v64i32 = zext <64 x i16> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v64i16_v64i64 = zext <64 x i16> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v64i32_v64i64 = zext <64 x i32> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i8 = zext <64 x i1> undef to <64 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64i1_v64i16 = zext <64 x i1> undef to <64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v64i1_v64i32 = zext <64 x i1> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v64i1_v64i64 = zext <64 x i1> undef to <64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v128i8_v128i16 = zext <128 x i8> undef to <128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v128i8_v128i32 = zext <128 x i8> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v128i8_v128i64 = zext <128 x i8> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v128i16_v128i32 = zext <128 x i16> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v128i16_v128i64 = zext <128 x i16> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v128i32_v128i64 = zext <128 x i32> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v128i1_v128i8 = zext <128 x i1> undef to <128 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v128i1_v128i16 = zext <128 x i1> undef to <128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v128i1_v128i32 = zext <128 x i1> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v128i1_v128i64 = zext <128 x i1> undef to <128 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v256i8_v256i16 = zext <256 x i8> undef to <256 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v256i8_v256i32 = zext <256 x i8> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %v256i8_v256i64 = zext <256 x i8> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v256i16_v256i32 = zext <256 x i16> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %v256i16_v256i64 = zext <256 x i16> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v256i32_v256i64 = zext <256 x i32> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v256i1_v256i8 = zext <256 x i1> undef to <256 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v256i1_v256i16 = zext <256 x i1> undef to <256 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %v256i1_v256i32 = zext <256 x i1> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %v256i1_v256i64 = zext <256 x i1> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_v16i16 = zext <16 x i1> undef to <16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_v16i32 = zext <16 x i1> undef to <16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i1_v16i64 = zext <16 x i1> undef to <16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i8_v32i16 = zext <32 x i8> undef to <32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i8_v32i32 = zext <32 x i8> undef to <32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i8_v32i64 = zext <32 x i8> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i16_v32i32 = zext <32 x i16> undef to <32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i16_v32i64 = zext <32 x i16> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i32_v32i64 = zext <32 x i32> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_v32i8 = zext <32 x i1> undef to <32 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_v32i16 = zext <32 x i1> undef to <32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32i32 = zext <32 x i1> undef to <32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32i64 = zext <32 x i1> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i16 = zext <64 x i8> undef to <64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i8_v64i32 = zext <64 x i8> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i8_v64i64 = zext <64 x i8> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i16_v64i32 = zext <64 x i16> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i16_v64i64 = zext <64 x i16> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64i64 = zext <64 x i32> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i1_v64i8 = zext <64 x i1> undef to <64 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i1_v64i16 = zext <64 x i1> undef to <64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64i32 = zext <64 x i1> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64i64 = zext <64 x i1> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v128i8_v128i16 = zext <128 x i8> undef to <128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v128i8_v128i32 = zext <128 x i8> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %v128i8_v128i64 = zext <128 x i8> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v128i16_v128i32 = zext <128 x i16> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v128i16_v128i64 = zext <128 x i16> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128i64 = zext <128 x i32> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v128i1_v128i8 = zext <128 x i1> undef to <128 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v128i1_v128i16 = zext <128 x i1> undef to <128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128i32 = zext <128 x i1> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128i64 = zext <128 x i1> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v256i8_v256i16 = zext <256 x i8> undef to <256 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v256i8_v256i32 = zext <256 x i8> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %v256i8_v256i64 = zext <256 x i8> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v256i16_v256i32 = zext <256 x i16> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %v256i16_v256i64 = zext <256 x i16> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %v256i32_v256i64 = zext <256 x i32> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v256i1_v256i8 = zext <256 x i1> undef to <256 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v256i1_v256i16 = zext <256 x i1> undef to <256 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %v256i1_v256i32 = zext <256 x i1> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %v256i1_v256i64 = zext <256 x i1> undef to <256 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i16 = zext <vscale x 1 x i8> undef to <vscale x 1 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i32 = zext <vscale x 1 x i8> undef to <vscale x 1 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i64 = zext <vscale x 1 x i8> undef to <vscale x 1 x i64>
@@ -765,73 +765,73 @@ define void @zext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i1_nxv1i64 = zext <vscale x 1 x i1> undef to <vscale x 1 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i16 = zext <vscale x 2 x i8> undef to <vscale x 2 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i32 = zext <vscale x 2 x i8> undef to <vscale x 2 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8_nxv2i64 = zext <vscale x 2 x i8> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i8_nxv2i64 = zext <vscale x 2 x i8> undef to <vscale x 2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i32 = zext <vscale x 2 x i16> undef to <vscale x 2 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2i64 = zext <vscale x 2 x i16> undef to <vscale x 2 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32_nxv2i64 = zext <vscale x 2 x i32> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i16_nxv2i64 = zext <vscale x 2 x i16> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i32_nxv2i64 = zext <vscale x 2 x i32> undef to <vscale x 2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i8 = zext <vscale x 2 x i1> undef to <vscale x 2 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i16 = zext <vscale x 2 x i1> undef to <vscale x 2 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i32 = zext <vscale x 2 x i1> undef to <vscale x 2 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i1_nxv2i64 = zext <vscale x 2 x i1> undef to <vscale x 2 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_nxv2i64 = zext <vscale x 2 x i1> undef to <vscale x 2 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i16 = zext <vscale x 4 x i8> undef to <vscale x 4 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i32 = zext <vscale x 4 x i8> undef to <vscale x 4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8_nxv4i64 = zext <vscale x 4 x i8> undef to <vscale x 4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i32 = zext <vscale x 4 x i16> undef to <vscale x 4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16_nxv4i64 = zext <vscale x 4 x i16> undef to <vscale x 4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32_nxv4i64 = zext <vscale x 4 x i32> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i8_nxv4i32 = zext <vscale x 4 x i8> undef to <vscale x 4 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i8_nxv4i64 = zext <vscale x 4 x i8> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i16_nxv4i32 = zext <vscale x 4 x i16> undef to <vscale x 4 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i16_nxv4i64 = zext <vscale x 4 x i16> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i32_nxv4i64 = zext <vscale x 4 x i32> undef to <vscale x 4 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i8 = zext <vscale x 4 x i1> undef to <vscale x 4 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i16 = zext <vscale x 4 x i1> undef to <vscale x 4 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i32 = zext <vscale x 4 x i1> undef to <vscale x 4 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i1_nxv4i64 = zext <vscale x 4 x i1> undef to <vscale x 4 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i16 = zext <vscale x 8 x i8> undef to <vscale x 8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i32 = zext <vscale x 8 x i8> undef to <vscale x 8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8_nxv8i64 = zext <vscale x 8 x i8> undef to <vscale x 8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i32 = zext <vscale x 8 x i16> undef to <vscale x 8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16_nxv8i64 = zext <vscale x 8 x i16> undef to <vscale x 8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32_nxv8i64 = zext <vscale x 8 x i32> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_nxv4i32 = zext <vscale x 4 x i1> undef to <vscale x 4 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv4i1_nxv4i64 = zext <vscale x 4 x i1> undef to <vscale x 4 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i8_nxv8i16 = zext <vscale x 8 x i8> undef to <vscale x 8 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i8_nxv8i32 = zext <vscale x 8 x i8> undef to <vscale x 8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i8_nxv8i64 = zext <vscale x 8 x i8> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i16_nxv8i32 = zext <vscale x 8 x i16> undef to <vscale x 8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i16_nxv8i64 = zext <vscale x 8 x i16> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i32_nxv8i64 = zext <vscale x 8 x i32> undef to <vscale x 8 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i8 = zext <vscale x 8 x i1> undef to <vscale x 8 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i16 = zext <vscale x 8 x i1> undef to <vscale x 8 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i32 = zext <vscale x 8 x i1> undef to <vscale x 8 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i1_nxv8i64 = zext <vscale x 8 x i1> undef to <vscale x 8 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i16 = zext <vscale x 16 x i8> undef to <vscale x 16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8_nxv16i32 = zext <vscale x 16 x i8> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i8_nxv16i64 = zext <vscale x 16 x i8> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16_nxv16i32 = zext <vscale x 16 x i16> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i16_nxv16i64 = zext <vscale x 16 x i16> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i32_nxv16i64 = zext <vscale x 16 x i32> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i8 = zext <vscale x 16 x i1> undef to <vscale x 16 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i16 = zext <vscale x 16 x i1> undef to <vscale x 16 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i1_nxv16i32 = zext <vscale x 16 x i1> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_nxv16i64 = zext <vscale x 16 x i1> undef to <vscale x 16 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8_nxv32i16 = zext <vscale x 32 x i8> undef to <vscale x 32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i8_nxv32i32 = zext <vscale x 32 x i8> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i8_nxv32i64 = zext <vscale x 32 x i8> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i16_nxv32i32 = zext <vscale x 32 x i16> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i16_nxv32i64 = zext <vscale x 32 x i16> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i32_nxv32i64 = zext <vscale x 32 x i32> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i8 = zext <vscale x 32 x i1> undef to <vscale x 32 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i1_nxv32i16 = zext <vscale x 32 x i1> undef to <vscale x 32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_nxv32i32 = zext <vscale x 32 x i1> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_nxv32i64 = zext <vscale x 32 x i1> undef to <vscale x 32 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv64i8_nxv64i16 = zext <vscale x 64 x i8> undef to <vscale x 64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv64i8_nxv64i32 = zext <vscale x 64 x i8> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %nxv64i8_nxv64i64 = zext <vscale x 64 x i8> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv64i16_nxv64i32 = zext <vscale x 64 x i16> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %nxv64i16_nxv64i64 = zext <vscale x 64 x i16> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv64i32_nxv64i64 = zext <vscale x 64 x i32> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv64i1_nxv64i8 = zext <vscale x 64 x i1> undef to <vscale x 64 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv64i1_nxv64i16 = zext <vscale x 64 x i1> undef to <vscale x 64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv64i1_nxv64i32 = zext <vscale x 64 x i1> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %nxv64i1_nxv64i64 = zext <vscale x 64 x i1> undef to <vscale x 64 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv128i8_nxv128i16 = zext <vscale x 128 x i8> undef to <vscale x 128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %nxv128i8_nxv128i32 = zext <vscale x 128 x i8> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_nxv8i16 = zext <vscale x 8 x i1> undef to <vscale x 8 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i1_nxv8i32 = zext <vscale x 8 x i1> undef to <vscale x 8 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv8i1_nxv8i64 = zext <vscale x 8 x i1> undef to <vscale x 8 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i8_nxv16i16 = zext <vscale x 16 x i8> undef to <vscale x 16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i8_nxv16i32 = zext <vscale x 16 x i8> undef to <vscale x 16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i8_nxv16i64 = zext <vscale x 16 x i8> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i16_nxv16i32 = zext <vscale x 16 x i16> undef to <vscale x 16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i16_nxv16i64 = zext <vscale x 16 x i16> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i32_nxv16i64 = zext <vscale x 16 x i32> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_nxv16i8 = zext <vscale x 16 x i1> undef to <vscale x 16 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_nxv16i16 = zext <vscale x 16 x i1> undef to <vscale x 16 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16i32 = zext <vscale x 16 x i1> undef to <vscale x 16 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16i64 = zext <vscale x 16 x i1> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i8_nxv32i16 = zext <vscale x 32 x i8> undef to <vscale x 32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i8_nxv32i32 = zext <vscale x 32 x i8> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i8_nxv32i64 = zext <vscale x 32 x i8> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i16_nxv32i32 = zext <vscale x 32 x i16> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i16_nxv32i64 = zext <vscale x 32 x i16> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32i64 = zext <vscale x 32 x i32> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i1_nxv32i8 = zext <vscale x 32 x i1> undef to <vscale x 32 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv32i1_nxv32i16 = zext <vscale x 32 x i1> undef to <vscale x 32 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32i32 = zext <vscale x 32 x i1> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32i64 = zext <vscale x 32 x i1> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv64i8_nxv64i16 = zext <vscale x 64 x i8> undef to <vscale x 64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv64i8_nxv64i32 = zext <vscale x 64 x i8> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %nxv64i8_nxv64i64 = zext <vscale x 64 x i8> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv64i16_nxv64i32 = zext <vscale x 64 x i16> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv64i16_nxv64i64 = zext <vscale x 64 x i16> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv64i32_nxv64i64 = zext <vscale x 64 x i32> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv64i1_nxv64i8 = zext <vscale x 64 x i1> undef to <vscale x 64 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv64i1_nxv64i16 = zext <vscale x 64 x i1> undef to <vscale x 64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64i32 = zext <vscale x 64 x i1> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %nxv64i1_nxv64i64 = zext <vscale x 64 x i1> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv128i8_nxv128i16 = zext <vscale x 128 x i8> undef to <vscale x 128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv128i8_nxv128i32 = zext <vscale x 128 x i8> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i8_nxv128i128 = zext <vscale x 128 x i8> undef to <vscale x 128 x i128>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv128i16_nxv128i32 = zext <vscale x 128 x i16> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv128i16_nxv128i32 = zext <vscale x 128 x i16> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i16_nxv128i128 = zext <vscale x 128 x i16> undef to <vscale x 128 x i128>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i32_nxv128i128 = zext <vscale x 128 x i32> undef to <vscale x 128 x i128>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv128i1_nxv128i8 = zext <vscale x 128 x i1> undef to <vscale x 128 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv128i1_nxv128i16 = zext <vscale x 128 x i1> undef to <vscale x 128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %nxv128i1_nxv128i32 = zext <vscale x 128 x i1> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %nxv128i1_nxv128i8 = zext <vscale x 128 x i1> undef to <vscale x 128 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %nxv128i1_nxv128i16 = zext <vscale x 128 x i1> undef to <vscale x 128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %nxv128i1_nxv128i32 = zext <vscale x 128 x i1> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i1_nxv128i128 = zext <vscale x 128 x i1> undef to <vscale x 128 x i128>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
diff --git a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll
index 80efe912c869..30cb32ce4eaf 100644
--- a/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/reduce-scalable-int.ll
@@ -1141,7 +1141,7 @@ define signext i32 @vreduce_add_nxv4i32(<vscale x 4 x i32> %v) {
 
 define signext i32 @vwreduce_add_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vwreduce_add_nxv4i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = sext <vscale x 4 x i16> %v to <vscale x 4 x i32>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e = sext <vscale x 4 x i16> %v to <vscale x 4 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %e)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
@@ -1157,7 +1157,7 @@ define signext i32 @vwreduce_add_nxv4i16(<vscale x 4 x i16> %v) {
 
 define signext i32 @vwreduce_uadd_nxv4i16(<vscale x 4 x i16> %v) {
 ; CHECK-LABEL: 'vwreduce_uadd_nxv4i16'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = zext <vscale x 4 x i16> %v to <vscale x 4 x i32>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e = zext <vscale x 4 x i16> %v to <vscale x 4 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %e)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %red
 ;
@@ -1445,7 +1445,7 @@ define i64 @vreduce_add_nxv2i64(<vscale x 2 x i64> %v) {
 
 define i64 @vwreduce_add_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vwreduce_add_nxv2i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = sext <vscale x 2 x i32> %v to <vscale x 2 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e = sext <vscale x 2 x i32> %v to <vscale x 2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %e)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
@@ -1461,7 +1461,7 @@ define i64 @vwreduce_add_nxv2i32(<vscale x 2 x i32> %v) {
 
 define i64 @vwreduce_uadd_nxv2i32(<vscale x 2 x i32> %v) {
 ; CHECK-LABEL: 'vwreduce_uadd_nxv2i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = zext <vscale x 2 x i32> %v to <vscale x 2 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e = zext <vscale x 2 x i32> %v to <vscale x 2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %red = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %e)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
@@ -1597,7 +1597,7 @@ define i64 @vreduce_add_nxv4i64(<vscale x 4 x i64> %v) {
 
 define i64 @vwreduce_add_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vwreduce_add_nxv4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = sext <vscale x 4 x i32> %v to <vscale x 4 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e = sext <vscale x 4 x i32> %v to <vscale x 4 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> %e)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
@@ -1613,7 +1613,7 @@ define i64 @vwreduce_add_nxv4i32(<vscale x 4 x i32> %v) {
 
 define i64 @vwreduce_uadd_nxv4i32(<vscale x 4 x i32> %v) {
 ; CHECK-LABEL: 'vwreduce_uadd_nxv4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = zext <vscale x 4 x i32> %v to <vscale x 4 x i64>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %e = zext <vscale x 4 x i32> %v to <vscale x 4 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %red = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> %e)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i64 %red
 ;
diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-extractelement.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-extractelement.ll
index 225bad6da591..aa7a90bece33 100644
--- a/llvm/test/Analysis/CostModel/RISCV/rvv-extractelement.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/rvv-extractelement.ll
@@ -12,12 +12,12 @@ define void @extractelement_int(i32 %x) {
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i1_0 = extractelement <4 x i1> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i1_0 = extractelement <8 x i1> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i1_0 = extractelement <16 x i1> undef, i32 0
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i1_0 = extractelement <vscale x 2 x i1> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4i1_0 = extractelement <vscale x 4 x i1> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv8i1_0 = extractelement <vscale x 8 x i1> undef, i32 0
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = extractelement <2 x i8> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = extractelement <4 x i8> undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = extractelement <8 x i8> undef, i32 0
@@ -66,12 +66,12 @@ define void @extractelement_int(i32 %x) {
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_1 = extractelement <4 x i1> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_1 = extractelement <8 x i1> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_1 = extractelement <16 x i1> undef, i32 1
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_1 = extractelement <vscale x 2 x i1> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_1 = extractelement <vscale x 4 x i1> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_1 = extractelement <vscale x 8 x i1> undef, i32 1
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = extractelement <2 x i8> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = extractelement <4 x i8> undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = extractelement <8 x i8> undef, i32 1
@@ -120,12 +120,12 @@ define void @extractelement_int(i32 %x) {
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_x = extractelement <4 x i1> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_x = extractelement <8 x i1> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_x = extractelement <16 x i1> undef, i32 %x
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_x = extractelement <vscale x 2 x i1> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_x = extractelement <vscale x 4 x i1> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_x = extractelement <vscale x 8 x i1> undef, i32 %x
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_x = extractelement <2 x i8> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_x = extractelement <4 x i8> undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_x = extractelement <8 x i8> undef, i32 %x
@@ -177,12 +177,12 @@ define void @extractelement_int(i32 %x) {
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i1_0 = extractelement <4 x i1> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i1_0 = extractelement <8 x i1> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i1_0 = extractelement <16 x i1> undef, i32 0
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i1_0 = extractelement <vscale x 2 x i1> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4i1_0 = extractelement <vscale x 4 x i1> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv8i1_0 = extractelement <vscale x 8 x i1> undef, i32 0
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = extractelement <2 x i8> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = extractelement <4 x i8> undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = extractelement <8 x i8> undef, i32 0
@@ -231,12 +231,12 @@ define void @extractelement_int(i32 %x) {
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_1 = extractelement <4 x i1> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_1 = extractelement <8 x i1> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_1 = extractelement <16 x i1> undef, i32 1
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_1 = extractelement <vscale x 2 x i1> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_1 = extractelement <vscale x 4 x i1> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_1 = extractelement <vscale x 8 x i1> undef, i32 1
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = extractelement <2 x i8> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = extractelement <4 x i8> undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = extractelement <8 x i8> undef, i32 1
@@ -285,12 +285,12 @@ define void @extractelement_int(i32 %x) {
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_x = extractelement <4 x i1> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_x = extractelement <8 x i1> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_x = extractelement <16 x i1> undef, i32 %x
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_x = extractelement <vscale x 2 x i1> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_x = extractelement <vscale x 4 x i1> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_x = extractelement <vscale x 8 x i1> undef, i32 %x
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_x = extractelement <2 x i8> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_x = extractelement <4 x i8> undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_x = extractelement <8 x i8> undef, i32 %x
@@ -341,13 +341,13 @@ define void @extractelement_int(i32 %x) {
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i1_0 = extractelement <2 x i1> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i1_0 = extractelement <4 x i1> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i1_0 = extractelement <8 x i1> undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i1_0 = extractelement <16 x i1> undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = extractelement <16 x i1> undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i1_0 = extractelement <vscale x 2 x i1> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4i1_0 = extractelement <vscale x 4 x i1> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv8i1_0 = extractelement <vscale x 8 x i1> undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = extractelement <2 x i8> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = extractelement <4 x i8> undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = extractelement <8 x i8> undef, i32 0
@@ -395,13 +395,13 @@ define void @extractelement_int(i32 %x) {
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i1_1 = extractelement <2 x i1> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_1 = extractelement <4 x i1> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_1 = extractelement <8 x i1> undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_1 = extractelement <16 x i1> undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = extractelement <16 x i1> undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_1 = extractelement <vscale x 2 x i1> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_1 = extractelement <vscale x 4 x i1> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_1 = extractelement <vscale x 8 x i1> undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = extractelement <2 x i8> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = extractelement <4 x i8> undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = extractelement <8 x i8> undef, i32 1
@@ -449,13 +449,13 @@ define void @extractelement_int(i32 %x) {
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i1_x = extractelement <2 x i1> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_x = extractelement <4 x i1> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_x = extractelement <8 x i1> undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_x = extractelement <16 x i1> undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_x = extractelement <16 x i1> undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_x = extractelement <vscale x 2 x i1> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_x = extractelement <vscale x 4 x i1> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_x = extractelement <vscale x 8 x i1> undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_x = extractelement <2 x i8> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_x = extractelement <4 x i8> undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_x = extractelement <8 x i8> undef, i32 %x
@@ -506,13 +506,13 @@ define void @extractelement_int(i32 %x) {
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i1_0 = extractelement <2 x i1> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i1_0 = extractelement <4 x i1> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i1_0 = extractelement <8 x i1> undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i1_0 = extractelement <16 x i1> undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = extractelement <16 x i1> undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32i1_0 = extractelement <32 x i1> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i1_0 = extractelement <vscale x 2 x i1> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4i1_0 = extractelement <vscale x 4 x i1> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv8i1_0 = extractelement <vscale x 8 x i1> undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = extractelement <vscale x 16 x i1> undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv32i1_0 = extractelement <vscale x 32 x i1> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = extractelement <2 x i8> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = extractelement <4 x i8> undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = extractelement <8 x i8> undef, i32 0
@@ -560,13 +560,13 @@ define void @extractelement_int(i32 %x) {
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i1_1 = extractelement <2 x i1> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_1 = extractelement <4 x i1> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_1 = extractelement <8 x i1> undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_1 = extractelement <16 x i1> undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = extractelement <16 x i1> undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v32i1_1 = extractelement <32 x i1> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_1 = extractelement <vscale x 2 x i1> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_1 = extractelement <vscale x 4 x i1> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_1 = extractelement <vscale x 8 x i1> undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = extractelement <vscale x 16 x i1> undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_1 = extractelement <vscale x 32 x i1> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = extractelement <2 x i8> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = extractelement <4 x i8> undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = extractelement <8 x i8> undef, i32 1
@@ -614,13 +614,13 @@ define void @extractelement_int(i32 %x) {
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i1_x = extractelement <2 x i1> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_x = extractelement <4 x i1> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_x = extractelement <8 x i1> undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i1_x = extractelement <16 x i1> undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_x = extractelement <16 x i1> undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v32i1_x = extractelement <32 x i1> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_x = extractelement <vscale x 2 x i1> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_x = extractelement <vscale x 4 x i1> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i1_x = extractelement <vscale x 8 x i1> undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_x = extractelement <vscale x 16 x i1> undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %nxv32i1_x = extractelement <vscale x 32 x i1> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_x = extractelement <2 x i8> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_x = extractelement <4 x i8> undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_x = extractelement <8 x i8> undef, i32 %x
diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll
index 5387c8dc3594..6e1ae0216f76 100644
--- a/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/rvv-insertelement.ll
@@ -12,12 +12,12 @@ define void @insertelement_int(i32 %x) {
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v4i1_0 = insertelement <4 x i1> undef, i1 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v8i1_0 = insertelement <8 x i1> undef, i1 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv2i1_0 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv4i1_0 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv8i1_0 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 0
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = insertelement <2 x i8> undef, i8 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = insertelement <4 x i8> undef, i8 undef, i32 0
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = insertelement <8 x i8> undef, i8 undef, i32 0
@@ -66,12 +66,12 @@ define void @insertelement_int(i32 %x) {
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v4i1_1 = insertelement <4 x i1> undef, i1 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i1_1 = insertelement <8 x i1> undef, i1 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv2i1_1 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i1_1 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8i1_1 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 1
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = insertelement <2 x i8> undef, i8 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = insertelement <4 x i8> undef, i8 undef, i32 1
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = insertelement <8 x i8> undef, i8 undef, i32 1
@@ -120,12 +120,12 @@ define void @insertelement_int(i32 %x) {
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i1_x = insertelement <4 x i1> undef, i1 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8i1_x = insertelement <8 x i1> undef, i1 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv2i1_x = insertelement <vscale x 2 x i1> undef, i1 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv4i1_x = insertelement <vscale x 4 x i1> undef, i1 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv8i1_x = insertelement <vscale x 8 x i1> undef, i1 undef, i32 %x
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
-; RV32V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
+; RV32V-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i8_x = insertelement <2 x i8> undef, i8 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i8_x = insertelement <4 x i8> undef, i8 undef, i32 %x
 ; RV32V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i8_x = insertelement <8 x i8> undef, i8 undef, i32 %x
@@ -177,12 +177,12 @@ define void @insertelement_int(i32 %x) {
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v4i1_0 = insertelement <4 x i1> undef, i1 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v8i1_0 = insertelement <8 x i1> undef, i1 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv2i1_0 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv4i1_0 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv8i1_0 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 0
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = insertelement <2 x i8> undef, i8 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = insertelement <4 x i8> undef, i8 undef, i32 0
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = insertelement <8 x i8> undef, i8 undef, i32 0
@@ -231,12 +231,12 @@ define void @insertelement_int(i32 %x) {
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v4i1_1 = insertelement <4 x i1> undef, i1 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i1_1 = insertelement <8 x i1> undef, i1 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv2i1_1 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i1_1 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8i1_1 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 1
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = insertelement <2 x i8> undef, i8 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = insertelement <4 x i8> undef, i8 undef, i32 1
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = insertelement <8 x i8> undef, i8 undef, i32 1
@@ -285,12 +285,12 @@ define void @insertelement_int(i32 %x) {
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i1_x = insertelement <4 x i1> undef, i1 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8i1_x = insertelement <8 x i1> undef, i1 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv2i1_x = insertelement <vscale x 2 x i1> undef, i1 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv4i1_x = insertelement <vscale x 4 x i1> undef, i1 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv8i1_x = insertelement <vscale x 8 x i1> undef, i1 undef, i32 %x
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
-; RV64V-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
+; RV64V-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i8_x = insertelement <2 x i8> undef, i8 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i8_x = insertelement <4 x i8> undef, i8 undef, i32 %x
 ; RV64V-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i8_x = insertelement <8 x i8> undef, i8 undef, i32 %x
@@ -341,13 +341,13 @@ define void @insertelement_int(i32 %x) {
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v2i1_0 = insertelement <2 x i1> undef, i1 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v4i1_0 = insertelement <4 x i1> undef, i1 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v8i1_0 = insertelement <8 x i1> undef, i1 undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv2i1_0 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv4i1_0 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv8i1_0 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = insertelement <2 x i8> undef, i8 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = insertelement <4 x i8> undef, i8 undef, i32 0
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = insertelement <8 x i8> undef, i8 undef, i32 0
@@ -395,13 +395,13 @@ define void @insertelement_int(i32 %x) {
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v2i1_1 = insertelement <2 x i1> undef, i1 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v4i1_1 = insertelement <4 x i1> undef, i1 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i1_1 = insertelement <8 x i1> undef, i1 undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv2i1_1 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i1_1 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8i1_1 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = insertelement <2 x i8> undef, i8 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = insertelement <4 x i8> undef, i8 undef, i32 1
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = insertelement <8 x i8> undef, i8 undef, i32 1
@@ -449,13 +449,13 @@ define void @insertelement_int(i32 %x) {
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2i1_x = insertelement <2 x i1> undef, i1 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i1_x = insertelement <4 x i1> undef, i1 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8i1_x = insertelement <8 x i1> undef, i1 undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv2i1_x = insertelement <vscale x 2 x i1> undef, i1 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv4i1_x = insertelement <vscale x 4 x i1> undef, i1 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv8i1_x = insertelement <vscale x 8 x i1> undef, i1 undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
-; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
+; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i8_x = insertelement <2 x i8> undef, i8 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i8_x = insertelement <4 x i8> undef, i8 undef, i32 %x
 ; RV32ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i8_x = insertelement <8 x i8> undef, i8 undef, i32 %x
@@ -506,13 +506,13 @@ define void @insertelement_int(i32 %x) {
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v2i1_0 = insertelement <2 x i1> undef, i1 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v4i1_0 = insertelement <4 x i1> undef, i1 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v8i1_0 = insertelement <8 x i1> undef, i1 undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16i1_0 = insertelement <16 x i1> undef, i1 undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v32i1_0 = insertelement <32 x i1> undef, i1 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv2i1_0 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv4i1_0 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv8i1_0 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_0 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 0
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %nxv32i1_0 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8_0 = insertelement <2 x i8> undef, i8 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = insertelement <4 x i8> undef, i8 undef, i32 0
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = insertelement <8 x i8> undef, i8 undef, i32 0
@@ -560,13 +560,13 @@ define void @insertelement_int(i32 %x) {
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v2i1_1 = insertelement <2 x i1> undef, i1 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v4i1_1 = insertelement <4 x i1> undef, i1 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i1_1 = insertelement <8 x i1> undef, i1 undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_1 = insertelement <16 x i1> undef, i1 undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i1_1 = insertelement <32 x i1> undef, i1 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv2i1_1 = insertelement <vscale x 2 x i1> undef, i1 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i1_1 = insertelement <vscale x 4 x i1> undef, i1 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8i1_1 = insertelement <vscale x 8 x i1> undef, i1 undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_1 = insertelement <vscale x 16 x i1> undef, i1 undef, i32 1
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv32i1_1 = insertelement <vscale x 32 x i1> undef, i1 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_1 = insertelement <2 x i8> undef, i8 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_1 = insertelement <4 x i8> undef, i8 undef, i32 1
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8_1 = insertelement <8 x i8> undef, i8 undef, i32 1
@@ -614,13 +614,13 @@ define void @insertelement_int(i32 %x) {
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2i1_x = insertelement <2 x i1> undef, i1 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i1_x = insertelement <4 x i1> undef, i1 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v8i1_x = insertelement <8 x i1> undef, i1 undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v16i1_x = insertelement <16 x i1> undef, i1 undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %v32i1_x = insertelement <32 x i1> undef, i1 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv2i1_x = insertelement <vscale x 2 x i1> undef, i1 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv4i1_x = insertelement <vscale x 4 x i1> undef, i1 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv8i1_x = insertelement <vscale x 8 x i1> undef, i1 undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
-; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16i1_x = insertelement <vscale x 16 x i1> undef, i1 undef, i32 %x
+; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv32i1_x = insertelement <vscale x 32 x i1> undef, i1 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i8_x = insertelement <2 x i8> undef, i8 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i8_x = insertelement <4 x i8> undef, i8 undef, i32 %x
 ; RV64ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i8_x = insertelement <8 x i8> undef, i8 undef, i32 %x
diff --git a/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll b/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll
index 46bf3152ac5b..b763198e98ba 100644
--- a/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/shuffle-broadcast.ll
@@ -197,7 +197,7 @@ define void  @broadcast_fixed() #0{
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %41 = shufflevector <32 x i1> undef, <32 x i1> undef, <32 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %42 = shufflevector <64 x i1> undef, <64 x i1> undef, <64 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %43 = shufflevector <128 x i1> undef, <128 x i1> undef, <128 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %ins1 = insertelement <128 x i1> poison, i1 poison, i32 0
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %ins1 = insertelement <128 x i1> poison, i1 poison, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %44 = shufflevector <128 x i1> %ins1, <128 x i1> poison, <128 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %ins2 = insertelement <2 x i8> poison, i8 3, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %45 = shufflevector <2 x i8> %ins2, <2 x i8> undef, <2 x i32> zeroinitializer
-- 
GitLab


From 6d13263d4a723689d025423562269ea6ccb6bfc2 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Wed, 27 Mar 2024 15:23:34 +0800
Subject: [PATCH 384/404] [RISCV] Add tests for combineBinOpOfZExts. NFC
 (#86689)

Unlike add, sub and mul, we don't have widening instructions for div,
rem and logical ops, so we don't have any test coverage if we were to
extend combineBinOpOfZExts to handle them.

Adding tests coincidentally revealed that logical ops are already
narrowed as a generic DAG combine via
DAGCombiner::hoistLogicOpWithSameOpcodeHands. So we don't actually need
to run combineBinOpOfZExts on them.
---
 llvm/test/CodeGen/RISCV/rvv/binop-zext.ll | 146 ++++++++++++++++++++++
 1 file changed, 146 insertions(+)
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/binop-zext.ll

diff --git a/llvm/test/CodeGen/RISCV/rvv/binop-zext.ll b/llvm/test/CodeGen/RISCV/rvv/binop-zext.ll
new file mode 100644
index 000000000000..e050240f0de1
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/binop-zext.ll
@@ -0,0 +1,146 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s
+
+; Check that we perform binary arithmetic in a narrower type where possible, via
+; combineBinOpOfZExt or otherwise.
+
+define <vscale x 8 x i32> @add(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
+; CHECK-LABEL: add:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwaddu.vv v12, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v12
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 8 x i8> %a to <vscale x 8 x i32>
+  %b.zext = zext <vscale x 8 x i8> %b to <vscale x 8 x i32>
+  %add = add <vscale x 8 x i32> %a.zext, %b.zext
+  ret <vscale x 8 x i32> %add
+}
+
+define <vscale x 8 x i32> @sub(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
+; CHECK-LABEL: sub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwsubu.vv v12, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vsext.vf2 v8, v12
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 8 x i8> %a to <vscale x 8 x i32>
+  %b.zext = zext <vscale x 8 x i8> %b to <vscale x 8 x i32>
+  %sub = sub <vscale x 8 x i32> %a.zext, %b.zext
+  ret <vscale x 8 x i32> %sub
+}
+
+define <vscale x 8 x i32> @mul(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
+; CHECK-LABEL: mul:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vwmulu.vv v12, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vzext.vf2 v8, v12
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 8 x i8> %a to <vscale x 8 x i32>
+  %b.zext = zext <vscale x 8 x i8> %b to <vscale x 8 x i32>
+  %mul = mul <vscale x 8 x i32> %a.zext, %b.zext
+  ret <vscale x 8 x i32> %mul
+}
+
+define <vscale x 8 x i32> @sdiv(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
+; CHECK-LABEL: sdiv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vzext.vf4 v12, v8
+; CHECK-NEXT:    vzext.vf4 v16, v9
+; CHECK-NEXT:    vdivu.vv v8, v12, v16
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 8 x i8> %a to <vscale x 8 x i32>
+  %b.zext = zext <vscale x 8 x i8> %b to <vscale x 8 x i32>
+  %sdiv = sdiv <vscale x 8 x i32> %a.zext, %b.zext
+  ret <vscale x 8 x i32> %sdiv
+}
+
+define <vscale x 8 x i32> @udiv(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
+; CHECK-LABEL: udiv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vzext.vf4 v12, v8
+; CHECK-NEXT:    vzext.vf4 v16, v9
+; CHECK-NEXT:    vdivu.vv v8, v12, v16
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 8 x i8> %a to <vscale x 8 x i32>
+  %b.zext = zext <vscale x 8 x i8> %b to <vscale x 8 x i32>
+  %udiv = udiv <vscale x 8 x i32> %a.zext, %b.zext
+  ret <vscale x 8 x i32> %udiv
+}
+
+define <vscale x 8 x i32> @srem(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
+; CHECK-LABEL: srem:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vzext.vf4 v12, v8
+; CHECK-NEXT:    vzext.vf4 v16, v9
+; CHECK-NEXT:    vremu.vv v8, v12, v16
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 8 x i8> %a to <vscale x 8 x i32>
+  %b.zext = zext <vscale x 8 x i8> %b to <vscale x 8 x i32>
+  %srem = srem <vscale x 8 x i32> %a.zext, %b.zext
+  ret <vscale x 8 x i32> %srem
+}
+
+define <vscale x 8 x i32> @urem(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
+; CHECK-LABEL: urem:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vzext.vf4 v12, v8
+; CHECK-NEXT:    vzext.vf4 v16, v9
+; CHECK-NEXT:    vremu.vv v8, v12, v16
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 8 x i8> %a to <vscale x 8 x i32>
+  %b.zext = zext <vscale x 8 x i8> %b to <vscale x 8 x i32>
+  %urem = urem <vscale x 8 x i32> %a.zext, %b.zext
+  ret <vscale x 8 x i32> %urem
+}
+
+define <vscale x 8 x i32> @and(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
+; CHECK-LABEL: and:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vand.vv v12, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v12
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 8 x i8> %a to <vscale x 8 x i32>
+  %b.zext = zext <vscale x 8 x i8> %b to <vscale x 8 x i32>
+  %shl = and <vscale x 8 x i32> %a.zext, %b.zext
+  ret <vscale x 8 x i32> %shl
+}
+
+define <vscale x 8 x i32> @or(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
+; CHECK-LABEL: or:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vor.vv v12, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v12
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 8 x i8> %a to <vscale x 8 x i32>
+  %b.zext = zext <vscale x 8 x i8> %b to <vscale x 8 x i32>
+  %or = or <vscale x 8 x i32> %a.zext, %b.zext
+  ret <vscale x 8 x i32> %or
+}
+
+define <vscale x 8 x i32> @xor(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {
+; CHECK-LABEL: xor:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vxor.vv v12, v8, v9
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vzext.vf4 v8, v12
+; CHECK-NEXT:    ret
+  %a.zext = zext <vscale x 8 x i8> %a to <vscale x 8 x i32>
+  %b.zext = zext <vscale x 8 x i8> %b to <vscale x 8 x i32>
+  %xor = xor <vscale x 8 x i32> %a.zext, %b.zext
+  ret <vscale x 8 x i32> %xor
+}
-- 
GitLab


From defc4859b032ccaec69f24b6cfd9882fece5f093 Mon Sep 17 00:00:00 2001
From: Jack Styles <99514724+Stylie777@users.noreply.github.com>
Date: Wed, 27 Mar 2024 07:49:38 +0000
Subject: [PATCH 385/404] [AArch64] Remove Automatic Enablement of FEAT_F32MM
 (#85203)

When `+sve` is passed in the command line, if the Architecture being
targeted is V8.6A/V9.1A or later, `+f32mm` is also added. This enables
FEAT_32MM, however at the time of writing no CPU's support this. This
leads to the FEAT_32MM instructions being compiled for CPU's that do not
support them.

This commit removes the automatic enablement, however the option is
still able to be used by passing `+f32mm`.
---
 clang/test/Driver/aarch64-sve.c                   |  9 ++++-----
 clang/test/Preprocessor/aarch64-target-features.c |  2 +-
 llvm/docs/ReleaseNotes.rst                        |  1 +
 llvm/lib/TargetParser/AArch64TargetParser.cpp     |  5 -----
 llvm/unittests/TargetParser/TargetParserTest.cpp  | 15 ++++-----------
 5 files changed, 10 insertions(+), 22 deletions(-)

diff --git a/clang/test/Driver/aarch64-sve.c b/clang/test/Driver/aarch64-sve.c
index f34b2700deb9..4a33c2e3c8d3 100644
--- a/clang/test/Driver/aarch64-sve.c
+++ b/clang/test/Driver/aarch64-sve.c
@@ -6,12 +6,11 @@
 // RUN: %clang --target=aarch64 -march=armv8.6a -### -c %s 2>&1 | FileCheck -check-prefix=GENERICV8A-NOSVE %s
 // GENERICV8A-NOSVE-NOT: "-target-feature" "+sve"
 
-// The 32-bit floating point matrix multiply extension is enabled by default
-// for armv8.6-a targets (or later) with SVE, and can optionally be enabled for
-// any target from armv8.2a onwards (we don't enforce not using it with earlier
-// targets).
+// The 32-bit floating point matrix multiply extension is an optional feature
+// that can be used for any target from armv8.2a and onwards. This can be
+// enabled using the `+f32mm` option.`.
 // RUN: %clang --target=aarch64 -march=armv8.6a       -### -c %s 2>&1 | FileCheck -check-prefix=NO-F32MM %s
-// RUN: %clang --target=aarch64 -march=armv8.6a+sve   -### -c %s 2>&1 | FileCheck -check-prefix=F32MM %s
+// RUN: %clang --target=aarch64 -march=armv8.6a+sve+f32mm   -### -c %s 2>&1 | FileCheck -check-prefix=F32MM %s
 // RUN: %clang --target=aarch64 -march=armv8.5a+f32mm -### -c %s 2>&1 | FileCheck -check-prefix=F32MM %s
 // NO-F32MM-NOT: "-target-feature" "+f32mm"
 // F32MM: "-target-feature" "+f32mm"
diff --git a/clang/test/Preprocessor/aarch64-target-features.c b/clang/test/Preprocessor/aarch64-target-features.c
index 9f8a8bdeeb9c..85762b7fed4d 100644
--- a/clang/test/Preprocessor/aarch64-target-features.c
+++ b/clang/test/Preprocessor/aarch64-target-features.c
@@ -196,7 +196,7 @@
 // CHECK-8_6-NOT: __ARM_FEATURE_SHA3 1
 // CHECK-8_6-NOT: __ARM_FEATURE_SM4 1
 
-// RUN: %clang -target aarch64-none-linux-gnu -march=armv8.6-a+sve -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SVE-8_6 %s
+// RUN: %clang -target aarch64-none-linux-gnu -march=armv8.6-a+sve+f32mm -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SVE-8_6 %s
 // CHECK-SVE-8_6: __ARM_FEATURE_SVE 1
 // CHECK-SVE-8_6: __ARM_FEATURE_SVE_BF16 1
 // CHECK-SVE-8_6: __ARM_FEATURE_SVE_MATMUL_FP32 1
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index c2b1a9d3d738..7588048334d7 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -76,6 +76,7 @@ Changes to the AMDGPU Backend
 
 Changes to the ARM Backend
 --------------------------
+* FEAT_F32MM is no longer activated by default when using `+sve` on v8.6-A or greater. The feature is still available and can be used by adding `+f32mm` to the command line options.
 
 Changes to the AVR Backend
 --------------------------
diff --git a/llvm/lib/TargetParser/AArch64TargetParser.cpp b/llvm/lib/TargetParser/AArch64TargetParser.cpp
index e36832f563ee..71099462d5ec 100644
--- a/llvm/lib/TargetParser/AArch64TargetParser.cpp
+++ b/llvm/lib/TargetParser/AArch64TargetParser.cpp
@@ -186,11 +186,6 @@ void AArch64::ExtensionSet::enable(ArchExtKind E) {
   // Special cases for dependencies which vary depending on the base
   // architecture version.
   if (BaseArch) {
-    // +sve implies +f32mm if the base architecture is v8.6A+ or v9.1A+
-    // It isn't the case in general that sve implies both f64mm and f32mm
-    if (E == AEK_SVE && BaseArch->is_superset(ARMV8_6A))
-      enable(AEK_F32MM);
-
     // +fp16 implies +fp16fml for v8.4A+, but not v9.0-A+
     if (E == AEK_FP16 && BaseArch->is_superset(ARMV8_4A) &&
         !BaseArch->is_superset(ARMV9A))
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index a7d0b1687a7f..2c72a7229b52 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -2347,13 +2347,6 @@ AArch64ExtensionDependenciesBaseArchTestParams
          {},
          {"aes", "sha2", "sha3", "sm4"}},
 
-        // +sve implies +f32mm if the base architecture is v8.6A+ or v9.1A+, but
-        // not earlier architectures.
-        {AArch64::ARMV8_5A, {"sve"}, {"sve"}, {"f32mm"}},
-        {AArch64::ARMV9A, {"sve"}, {"sve"}, {"f32mm"}},
-        {AArch64::ARMV8_6A, {"sve"}, {"sve", "f32mm"}, {}},
-        {AArch64::ARMV9_1A, {"sve"}, {"sve", "f32mm"}, {}},
-
         // +fp16 implies +fp16fml for v8.4A+, but not v9.0-A+
         {AArch64::ARMV8_3A, {"fp16"}, {"fullfp16"}, {"fp16fml"}},
         {AArch64::ARMV9A, {"fp16"}, {"fullfp16"}, {"fp16fml"}},
@@ -2520,10 +2513,10 @@ AArch64ExtensionDependenciesBaseCPUTestParams
          {}},
         {"cortex-a520",
          {},
-         {"v9.2a",    "bf16",     "crc",     "dotprod", "f32mm",        "flagm",
-          "fp-armv8", "fullfp16", "fp16fml", "i8mm",    "lse",          "mte",
-          "pauth",    "perfmon",  "predres", "ras",     "rcpc",         "rdm",
-          "sb",       "neon",     "ssbs",    "sve",     "sve2-bitperm", "sve2"},
+         {"v9.2a",    "bf16",    "crc",  "dotprod",      "flagm", "fp-armv8",
+          "fullfp16", "fp16fml", "i8mm", "lse",          "mte",   "pauth",
+          "perfmon",  "predres", "ras",  "rcpc",         "rdm",   "sb",
+          "neon",     "ssbs",    "sve",  "sve2-bitperm", "sve2"},
          {}},
 
         // Negative modifiers
-- 
GitLab


From 2938f1cff9f880d03c900a2bdcd078af937d9433 Mon Sep 17 00:00:00 2001
From: zhongyunde 00443407 <zhongyunde@huawei.com>
Date: Sun, 24 Mar 2024 05:56:52 -0400
Subject: [PATCH 386/404] [InstCombine] Refactor powi(X,Y) / X to call
 foldPowiReassoc, NFC

---
 .../InstCombine/InstCombineMulDivRem.cpp      | 28 +++++++++----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index af238a43b11a..a9f0a16be0b8 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -611,6 +611,18 @@ Instruction *InstCombinerImpl::foldPowiReassoc(BinaryOperator &I) {
       Y->getType() == Z->getType())
     return createPowiExpr(I, *this, X, Y, Z);
 
+  // powi(X, Y) / X --> powi(X, Y-1)
+  // This is legal when (Y - 1) can't wraparound, in which case reassoc and nnan
+  // are required.
+  // TODO: Multi-use may be also better off creating Powi(x,y-1)
+  if (I.hasAllowReassoc() && I.hasNoNaNs() &&
+      match(Op0, m_OneUse(m_Intrinsic<Intrinsic::powi>(m_Specific(Op1),
+                                                       m_Value(Y)))) &&
+      willNotOverflowSignedSub(Y, ConstantInt::get(Y->getType(), 1), I)) {
+    Constant *NegOne = ConstantInt::getAllOnesValue(Y->getType());
+    return createPowiExpr(I, *this, Op1, Y, NegOne);
+  }
+
   return nullptr;
 }
 
@@ -1904,20 +1916,8 @@ Instruction *InstCombinerImpl::visitFDiv(BinaryOperator &I) {
     return replaceInstUsesWith(I, Pow);
   }
 
-  // powi(X, Y) / X --> powi(X, Y-1)
-  // This is legal when (Y - 1) can't wraparound, in which case reassoc and nnan
-  // are required.
-  // TODO: Multi-use may be also better off creating Powi(x,y-1)
-  if (I.hasAllowReassoc() && I.hasNoNaNs() &&
-      match(Op0, m_OneUse(m_Intrinsic<Intrinsic::powi>(m_Specific(Op1),
-                                                       m_Value(Y)))) &&
-      willNotOverflowSignedSub(Y, ConstantInt::get(Y->getType(), 1), I)) {
-    Constant *NegOne = ConstantInt::getAllOnesValue(Y->getType());
-    Value *Y1 = Builder.CreateAdd(Y, NegOne);
-    Type *Types[] = {Op1->getType(), Y1->getType()};
-    Value *Pow = Builder.CreateIntrinsic(Intrinsic::powi, Types, {Op1, Y1}, &I);
-    return replaceInstUsesWith(I, Pow);
-  }
+  if (Instruction *FoldedPowi = foldPowiReassoc(I))
+    return FoldedPowi;
 
   return nullptr;
 }
-- 
GitLab


From bd9bb31bce0754c0a04d5c842ab3e7f8dd467861 Mon Sep 17 00:00:00 2001
From: zhongyunde 00443407 <zhongyunde@huawei.com>
Date: Sun, 24 Mar 2024 06:16:41 -0400
Subject: [PATCH 387/404] [InstCombine] add restrict reassoc for the powi(X,Y)
 / X

add restrict reassoc for the powi(X,Y) / X according the discuss on PR69998.
---
 .../InstCombine/InstCombineMulDivRem.cpp        |  4 ++--
 llvm/test/Transforms/InstCombine/powi.ll        | 17 ++++++++++++++---
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index a9f0a16be0b8..8c698e52b5a0 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -616,8 +616,8 @@ Instruction *InstCombinerImpl::foldPowiReassoc(BinaryOperator &I) {
   // are required.
   // TODO: Multi-use may be also better off creating Powi(x,y-1)
   if (I.hasAllowReassoc() && I.hasNoNaNs() &&
-      match(Op0, m_OneUse(m_Intrinsic<Intrinsic::powi>(m_Specific(Op1),
-                                                       m_Value(Y)))) &&
+      match(Op0, m_OneUse(m_AllowReassoc(m_Intrinsic<Intrinsic::powi>(
+                     m_Specific(Op1), m_Value(Y))))) &&
       willNotOverflowSignedSub(Y, ConstantInt::get(Y->getType(), 1), I)) {
     Constant *NegOne = ConstantInt::getAllOnesValue(Y->getType());
     return createPowiExpr(I, *this, Op1, Y, NegOne);
diff --git a/llvm/test/Transforms/InstCombine/powi.ll b/llvm/test/Transforms/InstCombine/powi.ll
index 43e34c889106..6c0575e8b719 100644
--- a/llvm/test/Transforms/InstCombine/powi.ll
+++ b/llvm/test/Transforms/InstCombine/powi.ll
@@ -313,7 +313,7 @@ define double @fdiv_pow_powi(double %x) {
 ; CHECK-NEXT:    [[DIV:%.*]] = fmul reassoc nnan double [[X:%.*]], [[X]]
 ; CHECK-NEXT:    ret double [[DIV]]
 ;
-  %p1 = call double @llvm.powi.f64.i32(double %x, i32 3)
+  %p1 = call reassoc double @llvm.powi.f64.i32(double %x, i32 3)
   %div = fdiv reassoc nnan double %p1, %x
   ret double %div
 }
@@ -323,7 +323,7 @@ define float @fdiv_powf_powi(float %x) {
 ; CHECK-NEXT:    [[DIV:%.*]] = call reassoc nnan float @llvm.powi.f32.i32(float [[X:%.*]], i32 99)
 ; CHECK-NEXT:    ret float [[DIV]]
 ;
-  %p1 = call float @llvm.powi.f32.i32(float %x, i32 100)
+  %p1 = call reassoc float @llvm.powi.f32.i32(float %x, i32 100)
   %div = fdiv reassoc nnan float %p1, %x
   ret float %div
 }
@@ -347,10 +347,21 @@ define double @fdiv_pow_powi_multi_use(double %x) {
 define float @fdiv_powf_powi_missing_reassoc(float %x) {
 ; CHECK-LABEL: @fdiv_powf_powi_missing_reassoc(
 ; CHECK-NEXT:    [[P1:%.*]] = call float @llvm.powi.f32.i32(float [[X:%.*]], i32 100)
-; CHECK-NEXT:    [[DIV:%.*]] = fdiv nnan float [[P1]], [[X]]
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv reassoc nnan float [[P1]], [[X]]
 ; CHECK-NEXT:    ret float [[DIV]]
 ;
   %p1 = call float @llvm.powi.f32.i32(float %x, i32 100)
+  %div = fdiv reassoc nnan float %p1, %x
+  ret float %div
+}
+
+define float @fdiv_powf_powi_missing_reassoc1(float %x) {
+; CHECK-LABEL: @fdiv_powf_powi_missing_reassoc1(
+; CHECK-NEXT:    [[P1:%.*]] = call reassoc float @llvm.powi.f32.i32(float [[X:%.*]], i32 100)
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv nnan float [[P1]], [[X]]
+; CHECK-NEXT:    ret float [[DIV]]
+;
+  %p1 = call reassoc float @llvm.powi.f32.i32(float %x, i32 100)
   %div = fdiv nnan float %p1, %x
   ret float %div
 }
-- 
GitLab


From df75183d70e029352a49c93f275db703c81a65c1 Mon Sep 17 00:00:00 2001
From: Julian Nagele <j.nagele@apple.com>
Date: Wed, 27 Mar 2024 09:30:27 +0000
Subject: [PATCH 388/404] [TBAA] Add verifier for tbaa.struct metadata (#86709)

Adds logic to the IR verifier that checks whether !tbaa.struct nodes are
well-formed. That is, it checks that the operands of !tbaa.struct nodes
are in groups of three, that each group of three operands consists of
two integers and a valid tbaa node, and that the regions described by
the offset and size operands are non-overlapping.

PR: https://github.com/llvm/llvm-project/pull/86709
---
 llvm/include/llvm/IR/Verifier.h               |  1 +
 llvm/lib/IR/Verifier.cpp                      | 32 +++++++++++++++++++
 llvm/test/CodeGen/AArch64/arm64-abi_align.ll  |  4 ++-
 .../AMDGPU/mem-intrinsics.ll                  |  2 +-
 .../InstCombine/struct-assign-tbaa.ll         |  2 +-
 llvm/test/Transforms/SROA/tbaa-struct3.ll     |  2 +-
 .../Scalarizer/basic-inseltpoison.ll          |  3 +-
 llvm/test/Transforms/Scalarizer/basic.ll      |  3 +-
 llvm/test/Verifier/tbaa-struct.ll             | 14 ++++++--
 9 files changed, 54 insertions(+), 9 deletions(-)

diff --git a/llvm/include/llvm/IR/Verifier.h b/llvm/include/llvm/IR/Verifier.h
index b25f8eb77ee3..b7db6e0bbfb7 100644
--- a/llvm/include/llvm/IR/Verifier.h
+++ b/llvm/include/llvm/IR/Verifier.h
@@ -77,6 +77,7 @@ public:
   /// Visit an instruction and return true if it is valid, return false if an
   /// invalid TBAA is attached.
   bool visitTBAAMetadata(Instruction &I, const MDNode *MD);
+  bool visitTBAAStructMetadata(Instruction &I, const MDNode *MD);
 };
 
 /// Check a function for errors, useful for use when debugging a
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 33f358440a31..e16572540f96 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -5096,6 +5096,9 @@ void Verifier::visitInstruction(Instruction &I) {
   if (MDNode *TBAA = I.getMetadata(LLVMContext::MD_tbaa))
     TBAAVerifyHelper.visitTBAAMetadata(I, TBAA);
 
+  if (MDNode *TBAA = I.getMetadata(LLVMContext::MD_tbaa_struct))
+    TBAAVerifyHelper.visitTBAAStructMetadata(I, TBAA);
+
   if (MDNode *MD = I.getMetadata(LLVMContext::MD_noalias))
     visitAliasScopeListMetadata(MD);
   if (MDNode *MD = I.getMetadata(LLVMContext::MD_alias_scope))
@@ -7419,6 +7422,35 @@ bool TBAAVerifier::visitTBAAMetadata(Instruction &I, const MDNode *MD) {
   return true;
 }
 
+bool TBAAVerifier::visitTBAAStructMetadata(Instruction &I, const MDNode *MD) {
+  CheckTBAA(MD->getNumOperands() % 3 == 0,
+            "tbaa.struct operands must occur in groups of three", &I, MD);
+
+  // Each group of three operands must consist of two integers and a
+  // tbaa node. Moreover, the regions described by the offset and size
+  // operands must be non-overlapping.
+  std::optional<APInt> NextFree;
+  for (unsigned int Idx = 0; Idx < MD->getNumOperands(); Idx += 3) {
+    auto *OffsetCI =
+        mdconst::dyn_extract_or_null<ConstantInt>(MD->getOperand(Idx));
+    CheckTBAA(OffsetCI, "Offset must be a constant integer", &I, MD);
+
+    auto *SizeCI =
+        mdconst::dyn_extract_or_null<ConstantInt>(MD->getOperand(Idx + 1));
+    CheckTBAA(SizeCI, "Size must be a constant integer", &I, MD);
+
+    MDNode *TBAA = dyn_cast_or_null<MDNode>(MD->getOperand(Idx + 2));
+    CheckTBAA(TBAA, "TBAA tag missing", &I, MD);
+    visitTBAAMetadata(I, TBAA);
+
+    bool NonOverlapping = !NextFree || NextFree->ule(OffsetCI->getValue());
+    CheckTBAA(NonOverlapping, "Overlapping tbaa.struct regions", &I, MD);
+
+    NextFree = OffsetCI->getValue() + SizeCI->getValue();
+  }
+  return true;
+}
+
 char VerifierLegacyPass::ID = 0;
 INITIALIZE_PASS(VerifierLegacyPass, "verify", "Module Verifier", false, false)
 
diff --git a/llvm/test/CodeGen/AArch64/arm64-abi_align.ll b/llvm/test/CodeGen/AArch64/arm64-abi_align.ll
index 089e171e5a4a..c9fd2d38e27a 100644
--- a/llvm/test/CodeGen/AArch64/arm64-abi_align.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-abi_align.ll
@@ -518,4 +518,6 @@ attributes #5 = { nobuiltin }
 !1 = !{!"omnipotent char", !2}
 !2 = !{!"Simple C/C++ TBAA"}
 !3 = !{!"short", !1}
-!4 = !{i64 0, i64 4, !0, i64 4, i64 2, !3, i64 8, i64 4, !0, i64 12, i64 2, !3, i64 16, i64 4, !0, i64 20, i64 2, !3}
+!4 = !{i64 0, i64 4, !5, i64 4, i64 2, !6, i64 8, i64 4, !5, i64 12, i64 2, !6, i64 16, i64 4, !5, i64 20, i64 2, !6}
+!5 = !{!0, !0, i64 0}
+!6 = !{!3, !3, i64 0}
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/mem-intrinsics.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/mem-intrinsics.ll
index 50b0e7a0f547..2f264a2432fc 100644
--- a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/mem-intrinsics.ll
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/mem-intrinsics.ll
@@ -141,4 +141,4 @@ attributes #1 = { argmemonly nounwind }
 !5 = distinct !{!5, !"some domain"}
 !6 = !{!7}
 !7 = distinct !{!7, !5, !"some scope 2"}
-!8 = !{i64 0, i64 8, null}
+!8 = !{i64 0, i64 8, !0}
diff --git a/llvm/test/Transforms/InstCombine/struct-assign-tbaa.ll b/llvm/test/Transforms/InstCombine/struct-assign-tbaa.ll
index 996d2c0e67e1..d079c03f1dcb 100644
--- a/llvm/test/Transforms/InstCombine/struct-assign-tbaa.ll
+++ b/llvm/test/Transforms/InstCombine/struct-assign-tbaa.ll
@@ -75,7 +75,7 @@ entry:
 !1 = !{!"omnipotent char", !0}
 !2 = !{!5, !5, i64 0}
 !3 = !{i64 0, i64 4, !2}
-!4 = !{i64 0, i64 8, null}
+!4 = !{i64 0, i64 8, !2}
 !5 = !{!"float", !0}
 !6 = !{i64 0, i64 4, !2, i64 4, i64 4, !2}
 !7 = !{i64 0, i64 2, !2, i64 4, i64 6, !2}
diff --git a/llvm/test/Transforms/SROA/tbaa-struct3.ll b/llvm/test/Transforms/SROA/tbaa-struct3.ll
index 0fcd787fef97..61034de81e4b 100644
--- a/llvm/test/Transforms/SROA/tbaa-struct3.ll
+++ b/llvm/test/Transforms/SROA/tbaa-struct3.ll
@@ -539,7 +539,7 @@ declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias
 !6 = !{!5, !5, i64 0}
 !7 = !{i64 0, i64 8, !6, i64 8, i64 4, !1}
 !8 = !{i64 0, i64 4, !1, i64 4, i64 8, !6}
-!9 = !{i64 0, i64 8, !6, i64 4, i64 8, !1}
+!9 = !{i64 0, i64 8, !6, i64 8, i64 8, !1}
 !10 = !{i64 0, i64 2, !1, i64 2, i64 2, !1}
 !11 = !{i64 0, i64 1, !1, i64 1, i64 3, !1}
 !12 = !{i64 0, i64 2, !1, i64 2, i64 6, !1}
diff --git a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
index bbcdcb6f5867..73ae66dd76c6 100644
--- a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
+++ b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
@@ -836,5 +836,6 @@ define <2 x i32> @f23_crash(<2 x i32> %srcvec, i32 %v1) {
 !2 = !{ !"set2", !0 }
 !3 = !{ !3, !{!"llvm.loop.parallel_accesses", !13} }
 !4 = !{ float 4.0 }
-!5 = !{ i64 0, i64 8, null }
+!5 = !{ i64 0, i64 8, !6 }
+!6 = !{ !1, !1, i64 0 }
 !13 = distinct !{}
diff --git a/llvm/test/Transforms/Scalarizer/basic.ll b/llvm/test/Transforms/Scalarizer/basic.ll
index db7c5f535f7e..87a70ccd3fc7 100644
--- a/llvm/test/Transforms/Scalarizer/basic.ll
+++ b/llvm/test/Transforms/Scalarizer/basic.ll
@@ -870,5 +870,6 @@ define <2 x float> @f25(<2 x float> %src) {
 !2 = !{ !"set2", !0 }
 !3 = !{ !3, !{!"llvm.loop.parallel_accesses", !13} }
 !4 = !{ float 4.0 }
-!5 = !{ i64 0, i64 8, null }
+!5 = !{ i64 0, i64 8, !6 }
+!6 = !{ !1, !1, i64 0 }
 !13 = distinct !{}
diff --git a/llvm/test/Verifier/tbaa-struct.ll b/llvm/test/Verifier/tbaa-struct.ll
index b8ddc7cee496..14c19a19d5ae 100644
--- a/llvm/test/Verifier/tbaa-struct.ll
+++ b/llvm/test/Verifier/tbaa-struct.ll
@@ -1,28 +1,36 @@
-; RUN: llvm-as < %s 2>&1
-
-; FIXME: The verifer should reject the invalid !tbaa.struct nodes below.
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
 
 define void @test_overlapping_regions(ptr %a1) {
+; CHECK: Overlapping tbaa.struct regions
+; CHECK-NEXT:  %ld = load i8, ptr %a1, align 1, !tbaa.struct !0
   %ld = load i8, ptr %a1, align 1, !tbaa.struct !0
   ret void
 }
 
 define void @test_size_not_integer(ptr %a1) {
+; CHECK: Size must be a constant integer
+; CHECK-NEXT:  store i8 1, ptr %a1, align 1, !tbaa.struct !5
   store i8 1, ptr %a1, align 1, !tbaa.struct !5
   ret void
 }
 
 define void @test_offset_not_integer(ptr %a1, ptr %a2) {
+; CHECK: Offset must be a constant integer
+; CHECK-NEXT:  tail call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a1, ptr align 8 %a2, i64 16, i1 false), !tbaa.struct !6
   tail call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a1, ptr align 8 %a2, i64 16, i1 false), !tbaa.struct !6
   ret void
 }
 
 define void @test_tbaa_missing(ptr %a1, ptr %a2) {
+; CHECK: TBAA tag missing
+; CHECK-NEXT:  tail call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a1, ptr align 8 %a2, i64 16, i1 false), !tbaa.struct !7
   tail call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a1, ptr align 8 %a2, i64 16, i1 false), !tbaa.struct !7
   ret void
 }
 
 define void @test_tbaa_invalid(ptr %a1) {
+; CHECK: Old-style TBAA is no longer allowed, use struct-path TBAA instead
+; CHECK-NEXT:  store i8 1, ptr %a1, align 1, !tbaa.struct !8
   store i8 1, ptr %a1, align 1, !tbaa.struct !8
   ret void
 }
-- 
GitLab


From f15b7deeaaf9028a31f66110a10f1313ed5e57f7 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Wed, 27 Mar 2024 17:40:44 +0800
Subject: [PATCH 389/404] [RISCV] Add test case to show missing vmerge fold on
 tied pseudos. NFC

Note we can't use vwaddu.wv because it will get combined away with #78403
---
 .../RISCV/rvv/rvv-peephole-vmerge-vops.ll        | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
index a4aef577bc9a..571e2df13c26 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops.ll
@@ -1187,3 +1187,19 @@ define <vscale x 2 x i32> @vmerge_larger_vl_false_becomes_tail(<vscale x 2 x i32
   %b = call <vscale x 2 x i32> @llvm.riscv.vmerge.nxv2i32.nxv2i32(<vscale x 2 x i32> poison, <vscale x 2 x i32> %false, <vscale x 2 x i32> %a, <vscale x 2 x i1> %m, i64 3)
   ret <vscale x 2 x i32> %b
 }
+
+; Test widening pseudos with their TIED variant (passthru same as first op).
+define <vscale x 2 x i64> @vpmerge_vwsub.w_tied(<vscale x 2 x i64> %passthru, <vscale x 2 x i64> %x, <vscale x 2 x i32> %y, <vscale x 2 x i1> %mask, i32 zeroext %vl) {
+; CHECK-LABEL: vpmerge_vwsub.w_tied:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, tu, ma
+; CHECK-NEXT:    vmv2r.v v10, v8
+; CHECK-NEXT:    vwsub.wv v10, v10, v12
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, tu, ma
+; CHECK-NEXT:    vmerge.vvm v8, v8, v10, v0
+; CHECK-NEXT:    ret
+  %vl.zext = zext i32 %vl to i64
+  %a = call <vscale x 2 x i64> @llvm.riscv.vwsub.w.nxv2i64.nxv2i32(<vscale x 2 x i64> %passthru, <vscale x 2 x i64> %passthru, <vscale x 2 x i32> %y, i64 %vl.zext)
+  %b = call <vscale x 2 x i64> @llvm.vp.merge.nxv2i64(<vscale x 2 x i1> %mask, <vscale x 2 x i64> %a, <vscale x 2 x i64> %passthru, i32 %vl)
+  ret <vscale x 2 x i64> %b
+}
-- 
GitLab


From cc23ee8250c2eda3f28c4d25c412e68ec78ecbe1 Mon Sep 17 00:00:00 2001
From: Jacek Caban <jacek@codeweavers.com>
Date: Wed, 27 Mar 2024 11:37:02 +0100
Subject: [PATCH 390/404] [LLD][COFF] Add support for EXPORTAS import name
 type. (#86541)

#78772 added similar support for .def file parser and import library
writer. This PR adds missing bits in LLD to propagate EXPORTAS name and
allow it in `/export` parser. This is syntax is used by MSVC for ARM64EC
`__declspec(dllexport)` handling.
---
 lld/COFF/Config.h           |  8 ++--
 lld/COFF/Driver.cpp         |  2 +
 lld/COFF/DriverUtils.cpp    | 14 +++++-
 lld/test/COFF/exportas.test | 88 +++++++++++++++++++++++++++++++++++++
 4 files changed, 106 insertions(+), 6 deletions(-)

diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h
index 8f85929f1bea..917f88fc2828 100644
--- a/lld/COFF/Config.h
+++ b/lld/COFF/Config.h
@@ -54,6 +54,7 @@ enum class EmitKind { Obj, LLVM, ASM };
 struct Export {
   StringRef name;       // N in /export:N or /export:E=N
   StringRef extName;    // E in /export:E=N
+  StringRef exportAs;   // E in /export:N,EXPORTAS,E
   StringRef aliasTarget; // GNU specific: N in "alias == N"
   Symbol *sym = nullptr;
   uint16_t ordinal = 0;
@@ -73,10 +74,9 @@ struct Export {
   StringRef exportName; // Name in DLL
 
   bool operator==(const Export &e) const {
-    return (name == e.name && extName == e.extName &&
-            aliasTarget == e.aliasTarget &&
-            ordinal == e.ordinal && noname == e.noname &&
-            data == e.data && isPrivate == e.isPrivate);
+    return (name == e.name && extName == e.extName && exportAs == e.exportAs &&
+            aliasTarget == e.aliasTarget && ordinal == e.ordinal &&
+            noname == e.noname && data == e.data && isPrivate == e.isPrivate);
   }
 };
 
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index 181492913c0d..2b1d4abb6ed0 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -945,6 +945,7 @@ void LinkerDriver::createImportLibrary(bool asLib) {
     e2.Name = std::string(e1.name);
     e2.SymbolName = std::string(e1.symbolName);
     e2.ExtName = std::string(e1.extName);
+    e2.ExportAs = std::string(e1.exportAs);
     e2.AliasTarget = std::string(e1.aliasTarget);
     e2.Ordinal = e1.ordinal;
     e2.Noname = e1.noname;
@@ -1044,6 +1045,7 @@ void LinkerDriver::parseModuleDefs(StringRef path) {
       e2.name = saver().save(e1.Name);
       e2.extName = saver().save(e1.ExtName);
     }
+    e2.exportAs = saver().save(e1.ExportAs);
     e2.aliasTarget = saver().save(e1.AliasTarget);
     e2.ordinal = e1.Ordinal;
     e2.noname = e1.Noname;
diff --git a/lld/COFF/DriverUtils.cpp b/lld/COFF/DriverUtils.cpp
index 0fa4769bab19..b4ff31a606da 100644
--- a/lld/COFF/DriverUtils.cpp
+++ b/lld/COFF/DriverUtils.cpp
@@ -585,7 +585,8 @@ Export LinkerDriver::parseExport(StringRef arg) {
     }
   }
 
-  // Optional parameters "[,@ordinal[,NONAME]][,DATA][,PRIVATE]"
+  // Optional parameters
+  // "[,@ordinal[,NONAME]][,DATA][,PRIVATE][,EXPORTAS,exportname]"
   while (!rest.empty()) {
     StringRef tok;
     std::tie(tok, rest) = rest.split(",");
@@ -607,6 +608,13 @@ Export LinkerDriver::parseExport(StringRef arg) {
       e.isPrivate = true;
       continue;
     }
+    if (tok.equals_insensitive("exportas")) {
+      if (!rest.empty() && !rest.contains(','))
+        e.exportAs = rest;
+      else
+        error("invalid EXPORTAS value: " + rest);
+      break;
+    }
     if (tok.starts_with("@")) {
       int32_t ord;
       if (tok.substr(1).getAsInteger(0, ord))
@@ -683,7 +691,9 @@ void LinkerDriver::fixupExports() {
   }
 
   for (Export &e : ctx.config.exports) {
-    if (!e.forwardTo.empty()) {
+    if (!e.exportAs.empty()) {
+      e.exportName = e.exportAs;
+    } else if (!e.forwardTo.empty()) {
       e.exportName = undecorate(ctx, e.name);
     } else {
       e.exportName = undecorate(ctx, e.extName.empty() ? e.name : e.extName);
diff --git a/lld/test/COFF/exportas.test b/lld/test/COFF/exportas.test
index c0295c3d7fb7..d70547c39b40 100644
--- a/lld/test/COFF/exportas.test
+++ b/lld/test/COFF/exportas.test
@@ -9,6 +9,77 @@ RUN: lld-link -out:out1.dll -dll -noentry test.obj test.lib
 RUN: llvm-readobj --coff-imports out1.dll | FileCheck --check-prefix=IMPORT %s
 IMPORT: Symbol: expfunc
 
+Pass -export argument with EXPORTAS.
+
+RUN: llvm-mc -filetype=obj -triple=x86_64-windows func.s -o func.obj
+RUN: lld-link -out:out2.dll -dll -noentry func.obj -export:func,EXPORTAS,expfunc
+RUN: llvm-readobj --coff-exports out2.dll | FileCheck --check-prefix=EXPORT %s
+EXPORT: Name: expfunc
+
+RUN: llvm-readobj out2.lib | FileCheck --check-prefix=IMPLIB %s
+IMPLIB:      Name type: export as
+IMPLIB-NEXT: Export name: expfunc
+IMPLIB-NEXT: Symbol: __imp_func
+IMPLIB-NEXT: Symbol: func
+
+Use .drectve section with EXPORTAS.
+
+RUN: llvm-mc -filetype=obj -triple=x86_64-windows drectve.s -o drectve.obj
+RUN: lld-link -out:out3.dll -dll -noentry func.obj drectve.obj
+RUN: llvm-readobj --coff-exports out3.dll | FileCheck --check-prefix=EXPORT %s
+RUN: llvm-readobj out3.lib | FileCheck --check-prefix=IMPLIB %s
+
+Use a .def file with EXPORTAS.
+
+RUN: lld-link -out:out4.dll -dll -noentry func.obj -def:test.def
+RUN: llvm-readobj --coff-exports out4.dll | FileCheck --check-prefix=EXPORT %s
+RUN: llvm-readobj out4.lib | FileCheck --check-prefix=IMPLIB %s
+
+Use a .def file with EXPORTAS in a forwarding export.
+
+RUN: lld-link -out:out5.dll -dll -noentry func.obj -def:test2.def
+RUN: llvm-readobj --coff-exports out5.dll | FileCheck --check-prefix=FORWARD-EXPORT %s
+FORWARD-EXPORT:      Export {
+FORWARD-EXPORT-NEXT:   Ordinal: 1
+FORWARD-EXPORT-NEXT:   Name: expfunc
+FORWARD-EXPORT-NEXT:   ForwardedTo: otherdll.otherfunc
+FORWARD-EXPORT-NEXT: }
+
+RUN: llvm-readobj out5.lib | FileCheck --check-prefix=FORWARD-IMPLIB %s
+FORWARD-IMPLIB:      Name type: export as
+FORWARD-IMPLIB-NEXT: Export name: expfunc
+FORWARD-IMPLIB-NEXT: Symbol: __imp_func
+FORWARD-IMPLIB-NEXT: Symbol: func
+
+Pass -export argument with EXPORTAS in a forwarding export.
+
+RUN: lld-link -out:out6.dll -dll -noentry func.obj -export:func=otherdll.otherfunc,EXPORTAS,expfunc
+RUN: llvm-readobj --coff-exports out6.dll | FileCheck --check-prefix=FORWARD-EXPORT %s
+RUN: llvm-readobj out6.lib | FileCheck --check-prefix=FORWARD-IMPLIB %s
+
+Pass -export argument with EXPORTAS in a data export.
+
+RUN: lld-link -out:out7.dll -dll -noentry func.obj -export:func,DATA,@5,EXPORTAS,expfunc
+RUN: llvm-readobj --coff-exports out7.dll | FileCheck --check-prefix=ORD %s
+ORD:      Ordinal: 5
+ORD-NEXT: Name: expfunc
+
+RUN: llvm-readobj out7.lib | FileCheck --check-prefix=ORD-IMPLIB %s
+ORD-IMPLIB:      Type: data
+ORD-IMPLIB-NEXT: Name type: export as
+ORD-IMPLIB-NEXT: Export name: expfunc
+ORD-IMPLIB-NEXT: Symbol: __imp_func
+
+Check invalid EXPORTAS syntax.
+
+RUN: not lld-link -out:err1.dll -dll -noentry func.obj -export:func,EXPORTAS, 2>&1 | \
+RUN:     FileCheck --check-prefix=ERR1 %s
+ERR1: error: invalid EXPORTAS value: {{$}}
+
+RUN: not lld-link -out:err2.dll -dll -noentry func.obj -export:func,EXPORTAS,expfunc,DATA 2>&1 | \
+RUN:     FileCheck --check-prefix=ERR2 %s
+ERR2: error: invalid EXPORTAS value: expfunc,DATA
+
 #--- test.s
     .section ".test", "rd"
     .rva __imp_func
@@ -17,3 +88,20 @@ IMPORT: Symbol: expfunc
 LIBRARY test.dll
 EXPORTS
     func EXPORTAS expfunc
+
+#--- test2.def
+LIBRARY test.dll
+EXPORTS
+    func=otherdll.otherfunc EXPORTAS expfunc
+
+#--- func.s
+    .text
+    .globl func
+    .p2align 2, 0x0
+func:
+    movl $1, %eax
+    retq
+
+#--- drectve.s
+    .section .drectve, "yn"
+    .ascii " -export:func,EXPORTAS,expfunc"
-- 
GitLab


From c9d12664f2f967ec170ed16d9a57af2f48e832c8 Mon Sep 17 00:00:00 2001
From: Jacek Caban <jacek@codeweavers.com>
Date: Wed, 27 Mar 2024 11:41:02 +0100
Subject: [PATCH 391/404] [llvm-dlltool][llvm-lib][COFF] Don't override NONAME
 exports with demangled ARM64EC symbols. (#86722)

---
 llvm/lib/Object/COFFImportFile.cpp           |  4 +-
 llvm/test/tools/llvm-lib/arm64ec-implib.test | 93 ++++++++++++++++++++
 2 files changed, 95 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Object/COFFImportFile.cpp b/llvm/lib/Object/COFFImportFile.cpp
index 8224a1492502..477c5bf98249 100644
--- a/llvm/lib/Object/COFFImportFile.cpp
+++ b/llvm/lib/Object/COFFImportFile.cpp
@@ -690,12 +690,12 @@ Error writeImportLibrary(StringRef ImportName, StringRef Path,
       if (ImportType == IMPORT_CODE && isArm64EC(M)) {
         if (std::optional<std::string> MangledName =
                 getArm64ECMangledFunctionName(Name)) {
-          if (ExportName.empty()) {
+          if (!E.Noname && ExportName.empty()) {
             NameType = IMPORT_NAME_EXPORTAS;
             ExportName.swap(Name);
           }
           Name = std::move(*MangledName);
-        } else if (ExportName.empty()) {
+        } else if (!E.Noname && ExportName.empty()) {
           NameType = IMPORT_NAME_EXPORTAS;
           ExportName = std::move(*getArm64ECDemangledFunctionName(Name));
         }
diff --git a/llvm/test/tools/llvm-lib/arm64ec-implib.test b/llvm/test/tools/llvm-lib/arm64ec-implib.test
index 9ce53fe0fea0..e9987d0ca2e6 100644
--- a/llvm/test/tools/llvm-lib/arm64ec-implib.test
+++ b/llvm/test/tools/llvm-lib/arm64ec-implib.test
@@ -14,6 +14,8 @@ ARMAP-NEXT: Archive EC map
 ARMAP-NEXT: #expname in test.dll
 ARMAP-NEXT: #funcexp in test.dll
 ARMAP-NEXT: #mangledfunc in test.dll
+ARMAP-NEXT: #manglednonamefunc in test.dll
+ARMAP-NEXT: #nonamefunc in test.dll
 ARMAP-NEXT: ?test_cpp_func@@$$hYAHPEAX@Z in test.dll
 ARMAP-NEXT: ?test_cpp_func@@YAHPEAX@Z in test.dll
 ARMAP-NEXT: __IMPORT_DESCRIPTOR_test in test.dll
@@ -23,13 +25,19 @@ ARMAP-NEXT: __imp_aux_?test_cpp_func@@YAHPEAX@Z in test.dll
 ARMAP-NEXT: __imp_aux_expname in test.dll
 ARMAP-NEXT: __imp_aux_funcexp in test.dll
 ARMAP-NEXT: __imp_aux_mangledfunc in test.dll
+ARMAP-NEXT: __imp_aux_manglednonamefunc in test.dll
+ARMAP-NEXT: __imp_aux_nonamefunc in test.dll
 ARMAP-NEXT: __imp_dataexp in test.dll
 ARMAP-NEXT: __imp_expname in test.dll
 ARMAP-NEXT: __imp_funcexp in test.dll
 ARMAP-NEXT: __imp_mangledfunc in test.dll
+ARMAP-NEXT: __imp_manglednonamefunc in test.dll
+ARMAP-NEXT: __imp_nonamefunc in test.dll
 ARMAP-NEXT: expname in test.dll
 ARMAP-NEXT: funcexp in test.dll
 ARMAP-NEXT: mangledfunc in test.dll
+ARMAP-NEXT: manglednonamefunc in test.dll
+ARMAP-NEXT: nonamefunc in test.dll
 ARMAP-NEXT: test_NULL_THUNK_DATA in test.dll
 
 RUN: llvm-readobj test.lib | FileCheck -check-prefix=READOBJ %s
@@ -95,6 +103,25 @@ READOBJ-NEXT: Type: data
 READOBJ-NEXT: Name type: name
 READOBJ-NEXT: Export name: dataexp
 READOBJ-NEXT: Symbol: __imp_dataexp
+READOBJ-EMPTY:
+READOBJ-NEXT: File: test.dll
+READOBJ-NEXT: Format: COFF-import-file-ARM64EC
+READOBJ-NEXT: Type: code
+READOBJ-NEXT: Name type: ordinal
+READOBJ-NEXT: Symbol: __imp_nonamefunc
+READOBJ-NEXT: Symbol: nonamefunc
+READOBJ-NEXT: Symbol: __imp_aux_nonamefunc
+READOBJ-NEXT: Symbol: #nonamefunc
+READOBJ-EMPTY:
+READOBJ-NEXT: File: test.dll
+READOBJ-NEXT: Format: COFF-import-file-ARM64EC
+READOBJ-NEXT: Type: code
+READOBJ-NEXT: Name type: ordinal
+READOBJ-NEXT: Symbol: __imp_manglednonamefunc
+READOBJ-NEXT: Symbol: manglednonamefunc
+READOBJ-NEXT: Symbol: __imp_aux_manglednonamefunc
+READOBJ-NEXT: Symbol: #manglednonamefunc
+
 
 Using -machine:arm64x gives the same output.
 RUN: llvm-lib -machine:arm64x -def:test.def -out:testx.lib
@@ -112,22 +139,28 @@ RUN: llvm-nm --print-armap testx.lib | FileCheck -check-prefix=ARMAPX %s
 
 ARMAPX:      Archive map
 ARMAPX-NEXT: #mangledfunc in test.dll
+ARMAPX-NEXT: #manglednonamefunc in test.dll
 ARMAPX-NEXT: ?test_cpp_func@@YAHPEAX@Z in test.dll
 ARMAPX-NEXT: __IMPORT_DESCRIPTOR_test in test.dll
 ARMAPX-NEXT: __NULL_IMPORT_DESCRIPTOR in test.dll
 ARMAPX-NEXT: __imp_#mangledfunc in test.dll
+ARMAPX-NEXT: __imp_#manglednonamefunc in test.dll
 ARMAPX-NEXT: __imp_?test_cpp_func@@YAHPEAX@Z in test.dll
 ARMAPX-NEXT: __imp_dataexp in test.dll
 ARMAPX-NEXT: __imp_expname in test.dll
 ARMAPX-NEXT: __imp_funcexp in test.dll
+ARMAPX-NEXT: __imp_nonamefunc in test.dll
 ARMAPX-NEXT: expname in test.dll
 ARMAPX-NEXT: funcexp in test.dll
+ARMAPX-NEXT: nonamefunc in test.dll
 ARMAPX-NEXT: test_NULL_THUNK_DATA in test.dll
 ARMAPX-EMPTY:
 ARMAPX-NEXT: Archive EC map
 ARMAPX-NEXT: #expname in test.dll
 ARMAPX-NEXT: #funcexp in test.dll
 ARMAPX-NEXT: #mangledfunc in test.dll
+ARMAPX-NEXT: #manglednonamefunc in test.dll
+ARMAPX-NEXT: #nonamefunc in test.dll
 ARMAPX-NEXT: ?test_cpp_func@@$$hYAHPEAX@Z in test.dll
 ARMAPX-NEXT: ?test_cpp_func@@YAHPEAX@Z in test.dll
 ARMAPX-NEXT: __IMPORT_DESCRIPTOR_test in test.dll
@@ -137,13 +170,19 @@ ARMAPX-NEXT: __imp_aux_?test_cpp_func@@YAHPEAX@Z in test.dll
 ARMAPX-NEXT: __imp_aux_expname in test.dll
 ARMAPX-NEXT: __imp_aux_funcexp in test.dll
 ARMAPX-NEXT: __imp_aux_mangledfunc in test.dll
+ARMAPX-NEXT: __imp_aux_manglednonamefunc in test.dll
+ARMAPX-NEXT: __imp_aux_nonamefunc in test.dll
 ARMAPX-NEXT: __imp_dataexp in test.dll
 ARMAPX-NEXT: __imp_expname in test.dll
 ARMAPX-NEXT: __imp_funcexp in test.dll
 ARMAPX-NEXT: __imp_mangledfunc in test.dll
+ARMAPX-NEXT: __imp_manglednonamefunc in test.dll
+ARMAPX-NEXT: __imp_nonamefunc in test.dll
 ARMAPX-NEXT: expname in test.dll
 ARMAPX-NEXT: funcexp in test.dll
 ARMAPX-NEXT: mangledfunc in test.dll
+ARMAPX-NEXT: manglednonamefunc in test.dll
+ARMAPX-NEXT: nonamefunc in test.dll
 ARMAPX-NEXT: test_NULL_THUNK_DATA in test.dll
 
 RUN: llvm-readobj testx.lib | FileCheck -check-prefix=READOBJX %s
@@ -211,6 +250,24 @@ READOBJX-NEXT: Export name: dataexp
 READOBJX-NEXT: Symbol: __imp_dataexp
 READOBJX-EMPTY:
 READOBJX-NEXT: File: test.dll
+READOBJX-NEXT: Format: COFF-import-file-ARM64EC
+READOBJX-NEXT: Type: code
+READOBJX-NEXT: Name type: ordinal
+READOBJX-NEXT: Symbol: __imp_nonamefunc
+READOBJX-NEXT: Symbol: nonamefunc
+READOBJX-NEXT: Symbol: __imp_aux_nonamefunc
+READOBJX-NEXT: Symbol: #nonamefunc
+READOBJX-EMPTY:
+READOBJX-NEXT: File: test.dll
+READOBJX-NEXT: Format: COFF-import-file-ARM64EC
+READOBJX-NEXT: Type: code
+READOBJX-NEXT: Name type: ordinal
+READOBJX-NEXT: Symbol: __imp_manglednonamefunc
+READOBJX-NEXT: Symbol: manglednonamefunc
+READOBJX-NEXT: Symbol: __imp_aux_manglednonamefunc
+READOBJX-NEXT: Symbol: #manglednonamefunc
+READOBJX-EMPTY:
+READOBJX-NEXT: File: test.dll
 READOBJX-NEXT: Format: COFF-import-file-ARM64
 READOBJX-NEXT: Type: code
 READOBJX-NEXT: Name type: name
@@ -248,6 +305,20 @@ READOBJX-NEXT: Type: data
 READOBJX-NEXT: Name type: name
 READOBJX-NEXT: Export name: dataexp
 READOBJX-NEXT: Symbol: __imp_dataexp
+READOBJX-EMPTY:
+READOBJX-NEXT: File: test.dll
+READOBJX-NEXT: Format: COFF-import-file-ARM64
+READOBJX-NEXT: Type: code
+READOBJX-NEXT: Name type: ordinal
+READOBJX-NEXT: Symbol: __imp_nonamefunc
+READOBJX-NEXT: Symbol: nonamefunc
+READOBJX-EMPTY:
+READOBJX-NEXT: File: test.dll
+READOBJX-NEXT: Format: COFF-import-file-ARM64
+READOBJX-NEXT: Type: code
+READOBJX-NEXT: Name type: ordinal
+READOBJX-NEXT: Symbol: __imp_#manglednonamefunc
+READOBJX-NEXT: Symbol: #manglednonamefunc
 
 
 RUN: llvm-lib -machine:arm64ec -def:test.def -defArm64Native:test2.def -out:test2.lib
@@ -266,6 +337,8 @@ ARMAPX2-NEXT: Archive EC map
 ARMAPX2-NEXT: #expname in test2.dll
 ARMAPX2-NEXT: #funcexp in test2.dll
 ARMAPX2-NEXT: #mangledfunc in test2.dll
+ARMAPX2-NEXT: #manglednonamefunc in test2.dll
+ARMAPX2-NEXT: #nonamefunc in test2.dll
 ARMAPX2-NEXT: ?test_cpp_func@@$$hYAHPEAX@Z in test2.dll
 ARMAPX2-NEXT: ?test_cpp_func@@YAHPEAX@Z in test2.dll
 ARMAPX2-NEXT: __IMPORT_DESCRIPTOR_test2 in test2.dll
@@ -275,13 +348,19 @@ ARMAPX2-NEXT: __imp_aux_?test_cpp_func@@YAHPEAX@Z in test2.dll
 ARMAPX2-NEXT: __imp_aux_expname in test2.dll
 ARMAPX2-NEXT: __imp_aux_funcexp in test2.dll
 ARMAPX2-NEXT: __imp_aux_mangledfunc in test2.dll
+ARMAPX2-NEXT: __imp_aux_manglednonamefunc in test2.dll
+ARMAPX2-NEXT: __imp_aux_nonamefunc in test2.dll
 ARMAPX2-NEXT: __imp_dataexp in test2.dll
 ARMAPX2-NEXT: __imp_expname in test2.dll
 ARMAPX2-NEXT: __imp_funcexp in test2.dll
 ARMAPX2-NEXT: __imp_mangledfunc in test2.dll
+ARMAPX2-NEXT: __imp_manglednonamefunc in test2.dll
+ARMAPX2-NEXT: __imp_nonamefunc in test2.dll
 ARMAPX2-NEXT: expname in test2.dll
 ARMAPX2-NEXT: funcexp in test2.dll
 ARMAPX2-NEXT: mangledfunc in test2.dll
+ARMAPX2-NEXT: manglednonamefunc in test2.dll
+ARMAPX2-NEXT: nonamefunc in test2.dll
 ARMAPX2-NEXT: test2_NULL_THUNK_DATA in test2.dll
 
 ARMAPX2:      test2.dll:
@@ -312,6 +391,18 @@ ARMAPX2-NEXT: test2.dll:
 ARMAPX2-NEXT: 00000000 D __imp_dataexp
 ARMAPX2-EMPTY:
 ARMAPX2-NEXT: test2.dll:
+ARMAPX2-NEXT: 00000000 T #nonamefunc
+ARMAPX2-NEXT: 00000000 T __imp_aux_nonamefunc
+ARMAPX2-NEXT: 00000000 T __imp_nonamefunc
+ARMAPX2-NEXT: 00000000 T nonamefunc
+ARMAPX2-EMPTY:
+ARMAPX2-NEXT: test2.dll:
+ARMAPX2-NEXT: 00000000 T #manglednonamefunc
+ARMAPX2-NEXT: 00000000 T __imp_aux_manglednonamefunc
+ARMAPX2-NEXT: 00000000 T __imp_manglednonamefunc
+ARMAPX2-NEXT: 00000000 T manglednonamefunc
+ARMAPX2-EMPTY:
+ARMAPX2-NEXT: test2.dll:
 ARMAPX2-NEXT: 00000000 T __imp_otherfunc
 ARMAPX2-NEXT: 00000000 T otherfunc
 
@@ -406,6 +497,8 @@ EXPORTS
     ?test_cpp_func@@YAHPEAX@Z
     expname=impname
     dataexp DATA
+    nonamefunc @1 NONAME
+    #manglednonamefunc @2 NONAME
 
 #--- test2.def
 LIBRARY test2.dll
-- 
GitLab


From ab7dba233a058cc8310ef829929238b5d8440b30 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu@amd.com>
Date: Wed, 27 Mar 2024 13:41:34 +0200
Subject: [PATCH 392/404] [CodeGen][LLVM] Make the `va_list` related intrinsics
 generic. (#85460)

Currently, the builtins used for implementing `va_list` handling
unconditionally take their arguments as unqualified `ptr`s i.e. pointers
to AS 0. This does not work for targets where the default AS is not 0 or
AS 0 is not a viable AS (for example, a target might choose 0 to
represent the constant address space). This patch changes the builtins'
signature to take generic `anyptr` args, which corrects this issue. It
is noisy due to the number of tests affected. A test for an upstream
target which does not use 0 as its default AS (SPIRV for HIP device
compilations) is added as well.
---
 clang/lib/CodeGen/CGBuiltin.cpp               |  6 ++-
 clang/test/CodeGen/CSKY/csky-abi.c            | 16 +++----
 clang/test/CodeGen/LoongArch/abi-lp64d.c      |  4 +-
 .../test/CodeGen/PowerPC/aix-altivec-vaargs.c |  4 +-
 clang/test/CodeGen/PowerPC/aix-vaargs.c       | 14 +++---
 .../CodeGen/PowerPC/ppc64le-varargs-f128.c    | 18 +++----
 clang/test/CodeGen/RISCV/riscv32-vararg.c     | 40 ++++++++--------
 clang/test/CodeGen/RISCV/riscv64-vararg.c     | 16 +++----
 clang/test/CodeGen/WebAssembly/wasm-varargs.c | 16 +++----
 clang/test/CodeGen/X86/va-arg-sse.c           |  4 +-
 clang/test/CodeGen/X86/x86_64-vaarg.c         |  4 +-
 clang/test/CodeGen/aarch64-ABI-align-packed.c | 14 +++---
 clang/test/CodeGen/aarch64-varargs.c          |  2 +-
 clang/test/CodeGen/arm-varargs.c              |  2 +-
 clang/test/CodeGen/hexagon-linux-vararg.c     |  2 +-
 clang/test/CodeGen/mips-varargs.c             | 16 +++----
 clang/test/CodeGen/pr53127.cpp                |  4 +-
 ...rargs-with-nonzero-default-address-space.c | 46 ++++++++++++++++++
 clang/test/CodeGen/xcore-abi.c                |  2 +-
 clang/test/CodeGenCXX/ext-int.cpp             | 12 ++---
 clang/test/CodeGenCXX/ibm128-declarations.cpp |  4 +-
 clang/test/CodeGenCXX/x86_64-vaarg.cpp        |  4 +-
 clang/test/Modules/codegen.test               |  2 +-
 llvm/docs/LangRef.rst                         | 29 ++++++-----
 llvm/include/llvm/IR/Intrinsics.td            | 11 +++--
 llvm/test/Bitcode/compatibility-3.6.ll        | 16 +++----
 llvm/test/Bitcode/compatibility-3.7.ll        | 16 +++----
 llvm/test/Bitcode/compatibility-3.8.ll        | 16 +++----
 llvm/test/Bitcode/compatibility-3.9.ll        | 16 +++----
 llvm/test/Bitcode/compatibility-4.0.ll        | 16 +++----
 llvm/test/Bitcode/compatibility-5.0.ll        | 16 +++----
 llvm/test/Bitcode/compatibility-6.0.ll        | 16 +++----
 llvm/test/Bitcode/compatibility.ll            | 18 +++----
 llvm/test/Bitcode/thinlto-function-summary.ll |  6 +--
 .../Bitcode/variableArgumentIntrinsic.3.2.ll  |  8 ++--
 .../MemorySanitizer/AArch64/vararg_shadow.ll  | 48 +++++++++----------
 .../MemorySanitizer/SystemZ/vararg-kernel.ll  |  2 +-
 .../MemorySanitizer/X86/vararg_shadow.ll      | 48 +++++++++----------
 .../MemorySanitizer/msan_debug_info.ll        |  2 +-
 .../Transforms/GlobalOpt/inalloca-varargs.ll  |  2 +-
 .../Transforms/IROutliner/illegal-vaarg.ll    | 12 ++---
 .../IROutliner/outline-vaarg-intrinsic.ll     |  8 ++--
 llvm/test/Transforms/NewGVN/pr31483.ll        |  2 +-
 .../Transforms/Reassociate/vaarg_movable.ll   |  4 +-
 .../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td   |  6 +--
 mlir/test/Target/LLVMIR/Import/basic.ll       | 14 +++---
 mlir/test/Target/LLVMIR/Import/intrinsic.ll   | 12 ++---
 mlir/test/Target/LLVMIR/llvmir.mlir           |  8 ++--
 48 files changed, 330 insertions(+), 274 deletions(-)
 create mode 100644 clang/test/CodeGen/varargs-with-nonzero-default-address-space.c

diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 3cfdb261a0ea..fdb517eb254d 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -792,7 +792,8 @@ EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) {
 
 Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) {
   Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
-  return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue);
+  return Builder.CreateCall(CGM.getIntrinsic(inst, {ArgValue->getType()}),
+                            ArgValue);
 }
 
 /// Checks if using the result of __builtin_object_size(p, @p From) in place of
@@ -3018,7 +3019,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   case Builtin::BI__builtin_va_copy: {
     Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer();
     Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer();
-    Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy), {DstPtr, SrcPtr});
+    Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy, {DstPtr->getType()}),
+                       {DstPtr, SrcPtr});
     return RValue::get(nullptr);
   }
   case Builtin::BIabs:
diff --git a/clang/test/CodeGen/CSKY/csky-abi.c b/clang/test/CodeGen/CSKY/csky-abi.c
index 2e549376ba93..29ed661aea75 100644
--- a/clang/test/CodeGen/CSKY/csky-abi.c
+++ b/clang/test/CodeGen/CSKY/csky-abi.c
@@ -185,13 +185,13 @@ void f_va_caller(void) {
 // CHECK:   [[VA:%.*]] = alloca ptr, align 4
 // CHECK:   [[V:%.*]] = alloca i32, align 4
 // CHECK:   store ptr %fmt, ptr [[FMT_ADDR]], align 4
-// CHECK:   call void @llvm.va_start(ptr [[VA]])
+// CHECK:   call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK:   [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK:   [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
 // CHECK:   store ptr [[ARGP_NEXT]], ptr [[VA]], align 4
 // CHECK:   [[TMP1:%.*]] = load i32, ptr [[ARGP_CUR]], align 4
 // CHECK:   store i32 [[TMP1]], ptr [[V]], align 4
-// CHECK:   call void @llvm.va_end(ptr [[VA]])
+// CHECK:   call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK:   [[TMP2:%.*]] = load i32, ptr [[V]], align 4
 // CHECK:   ret i32 [[TMP2]]
 // CHECK: }
@@ -210,13 +210,13 @@ int f_va_1(char *fmt, ...) {
 // CHECK-NEXT:    [[VA:%.*]] = alloca ptr, align 4
 // CHECK-NEXT:    [[V:%.*]] = alloca double, align 4
 // CHECK-NEXT:    store ptr [[FMT:%.*]], ptr [[FMT_ADDR]], align 4
-// CHECK-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 8
 // CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[VA]], align 4
 // CHECK-NEXT:    [[TMP4:%.*]] = load double, ptr [[ARGP_CUR]], align 4
 // CHECK-NEXT:    store double [[TMP4]], ptr [[V]], align 4
-// CHECK-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-NEXT:    [[TMP5:%.*]] = load double, ptr [[V]], align 4
 // CHECK-NEXT:    ret double [[TMP5]]
 double f_va_2(char *fmt, ...) {
@@ -236,7 +236,7 @@ double f_va_2(char *fmt, ...) {
 // CHECK-NEXT:    [[W:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    [[X:%.*]] = alloca double, align 4
 // CHECK-NEXT:    store ptr [[FMT:%.*]], ptr [[FMT_ADDR]], align 4
-// CHECK-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 8
 // CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[VA]], align 4
@@ -252,7 +252,7 @@ double f_va_2(char *fmt, ...) {
 // CHECK-NEXT:    store ptr [[ARGP_NEXT5]], ptr [[VA]], align 4
 // CHECK-NEXT:    [[TMP11:%.*]] = load double, ptr [[ARGP_CUR4]], align 4
 // CHECK-NEXT:    store double [[TMP11]], ptr [[X]], align 4
-// CHECK-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-NEXT:    [[TMP12:%.*]] = load double, ptr [[V]], align 4
 // CHECK-NEXT:    [[TMP13:%.*]] = load double, ptr [[X]], align 4
 // CHECK-NEXT:    [[ADD:%.*]] = fadd double [[TMP12]], [[TMP13]]
@@ -279,7 +279,7 @@ double f_va_3(char *fmt, ...) {
 // CHECK-NEXT:    [[LS:%.*]] = alloca [[STRUCT_LARGE:%.*]], align 4
 // CHECK-NEXT:    [[RET:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    store ptr [[FMT:%.*]], ptr [[FMT_ADDR]], align 4
-// CHECK-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
 // CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[VA]], align 4
@@ -302,7 +302,7 @@ double f_va_3(char *fmt, ...) {
 // CHECK-NEXT:    [[ARGP_NEXT9:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR8]], i32 16
 // CHECK-NEXT:    store ptr [[ARGP_NEXT9]], ptr [[VA]], align 4
 // CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[LS]], ptr align 4 [[ARGP_CUR8]], i32 16, i1 false)
-// CHECK-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 int f_va_4(char *fmt, ...) {
   __builtin_va_list va;
 
diff --git a/clang/test/CodeGen/LoongArch/abi-lp64d.c b/clang/test/CodeGen/LoongArch/abi-lp64d.c
index 66b480a7f068..fc7f1eada586 100644
--- a/clang/test/CodeGen/LoongArch/abi-lp64d.c
+++ b/clang/test/CodeGen/LoongArch/abi-lp64d.c
@@ -449,13 +449,13 @@ void f_va_caller(void) {
 // CHECK-NEXT:    [[VA:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    [[V:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    store ptr [[FMT:%.*]], ptr [[FMT_ADDR]], align 8
-// CHECK-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 8
 // CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i64 8
 // CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[VA]], align 8
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARGP_CUR]], align 8
 // CHECK-NEXT:    store i32 [[TMP0]], ptr [[V]], align 4
-// CHECK-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[V]], align 4
 // CHECK-NEXT:    ret i32 [[TMP1]]
 int f_va_int(char *fmt, ...) {
diff --git a/clang/test/CodeGen/PowerPC/aix-altivec-vaargs.c b/clang/test/CodeGen/PowerPC/aix-altivec-vaargs.c
index 03182423a422..b3f1e93b6394 100644
--- a/clang/test/CodeGen/PowerPC/aix-altivec-vaargs.c
+++ b/clang/test/CodeGen/PowerPC/aix-altivec-vaargs.c
@@ -17,7 +17,7 @@ vector double vector_varargs(int count, ...) {
 }
 
 // CHECK:         %arg_list = alloca ptr
-// CHECK:         call void @llvm.va_start(ptr %arg_list)
+// CHECK:         call void @llvm.va_start.p0(ptr %arg_list)
 
 // AIX32:       for.body:
 // AIX32-NEXT:    %argp.cur = load ptr, ptr %arg_list, align 4
@@ -41,4 +41,4 @@ vector double vector_varargs(int count, ...) {
 
 
 // CHECK:      for.end:
-// CHECK:        call void @llvm.va_end(ptr %arg_list)
+// CHECK:        call void @llvm.va_end.p0(ptr %arg_list)
diff --git a/clang/test/CodeGen/PowerPC/aix-vaargs.c b/clang/test/CodeGen/PowerPC/aix-vaargs.c
index 8b8417d315a5..724ba6560cdb 100644
--- a/clang/test/CodeGen/PowerPC/aix-vaargs.c
+++ b/clang/test/CodeGen/PowerPC/aix-vaargs.c
@@ -35,7 +35,7 @@ void testva (int n, ...) {
 
 // CHECK-NEXT:  %v = alloca i32, align 4
 // CHECK-NEXT:  store i32 %n, ptr %n.addr, align 4
-// CHECK-NEXT:  call void @llvm.va_start(ptr %ap)
+// CHECK-NEXT:  call void @llvm.va_start.p0(ptr %ap)
 
 // AIX32-NEXT:  %argp.cur = load ptr, ptr %ap, align 4
 // AIX32-NEXT:  %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 16
@@ -48,7 +48,7 @@ void testva (int n, ...) {
 // AIX32-NEXT:  call void @llvm.memcpy.p0.p0.i32(ptr align 8 %t, ptr align 4 %argp.cur, i32 16, i1 false)
 // AIX64-NEXT:  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %t, ptr align 8 %argp.cur, i64 16, i1 false)
 
-// CHECK-NEXT:  call void @llvm.va_copy(ptr %ap2, ptr %ap)
+// CHECK-NEXT:  call void @llvm.va_copy.p0(ptr %ap2, ptr %ap)
 
 // AIX32-NEXT:  %argp.cur1 = load ptr, ptr %ap2, align 4
 // AIX32-NEXT:  %argp.next2 = getelementptr inbounds i8, ptr %argp.cur1, i32 4
@@ -62,14 +62,14 @@ void testva (int n, ...) {
 // AIX64-NEXT:  %1 = load i32, ptr %0, align 4
 // AIX64-NEXT:  store i32 %1, ptr %v, align 4
 
-// CHECK-NEXT:  call void @llvm.va_end(ptr %ap2)
-// CHECK-NEXT:  call void @llvm.va_end(ptr %ap)
+// CHECK-NEXT:  call void @llvm.va_end.p0(ptr %ap2)
+// CHECK-NEXT:  call void @llvm.va_end.p0(ptr %ap)
 // CHECK-NEXT:  ret void
 
-// CHECK: declare void @llvm.va_start(ptr)
+// CHECK: declare void @llvm.va_start.p0(ptr)
 
 // AIX32: declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg)
 // AIX64: declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
 
-// CHECK: declare void @llvm.va_copy(ptr, ptr)
-// CHECK: declare void @llvm.va_end(ptr)
+// CHECK: declare void @llvm.va_copy.p0(ptr, ptr)
+// CHECK: declare void @llvm.va_end.p0(ptr)
diff --git a/clang/test/CodeGen/PowerPC/ppc64le-varargs-f128.c b/clang/test/CodeGen/PowerPC/ppc64le-varargs-f128.c
index 396614fe5bac..2f5459d1bb9c 100644
--- a/clang/test/CodeGen/PowerPC/ppc64le-varargs-f128.c
+++ b/clang/test/CodeGen/PowerPC/ppc64le-varargs-f128.c
@@ -31,7 +31,7 @@ void foo_ls(ldbl128_s);
 // OMP-TARGET: call void @foo_ld(ppc_fp128 noundef %[[V3]])
 
 // OMP-HOST-LABEL: define{{.*}} void @omp(
-// OMP-HOST: call void @llvm.va_start(ptr %[[AP:[0-9a-zA-Z_.]+]])
+// OMP-HOST: call void @llvm.va_start.p0(ptr %[[AP:[0-9a-zA-Z_.]+]])
 // OMP-HOST: %[[CUR:[0-9a-zA-Z_.]+]] = load ptr, ptr %[[AP]], align 8
 // OMP-HOST: %[[TMP0:[^ ]+]] = getelementptr inbounds i8, ptr %[[CUR]], i32 15
 // OMP-HOST: %[[ALIGN:[^ ]+]] = call ptr @llvm.ptrmask.p0.i64(ptr %[[TMP0]], i64 -16)
@@ -49,13 +49,13 @@ void omp(int n, ...) {
 }
 
 // IEEE-LABEL: define{{.*}} void @f128
-// IEEE: call void @llvm.va_start(ptr %[[AP:[0-9a-zA-Z_.]+]])
+// IEEE: call void @llvm.va_start.p0(ptr %[[AP:[0-9a-zA-Z_.]+]])
 // IEEE: %[[CUR:[0-9a-zA-Z_.]+]] = load ptr, ptr %[[AP]]
 // IEEE: %[[TMP0:[^ ]+]] = getelementptr inbounds i8, ptr %[[CUR]], i32 15
 // IEEE: %[[ALIGN:[^ ]+]] = call ptr @llvm.ptrmask.p0.i64(ptr %[[TMP0]], i64 -16)
 // IEEE: %[[V4:[0-9a-zA-Z_.]+]] = load fp128, ptr %[[ALIGN]], align 16
 // IEEE: call void @foo_fq(fp128 noundef %[[V4]])
-// IEEE: call void @llvm.va_end(ptr %[[AP]])
+// IEEE: call void @llvm.va_end.p0(ptr %[[AP]])
 void f128(int n, ...) {
   va_list ap;
   va_start(ap, n);
@@ -64,20 +64,20 @@ void f128(int n, ...) {
 }
 
 // IEEE-LABEL: define{{.*}} void @long_double
-// IEEE: call void @llvm.va_start(ptr %[[AP:[0-9a-zA-Z_.]+]])
+// IEEE: call void @llvm.va_start.p0(ptr %[[AP:[0-9a-zA-Z_.]+]])
 // IEEE: %[[CUR:[0-9a-zA-Z_.]+]] = load ptr, ptr %[[AP]]
 // IEEE: %[[TMP0:[^ ]+]] = getelementptr inbounds i8, ptr %[[CUR]], i32 15
 // IEEE: %[[ALIGN:[^ ]+]] = call ptr @llvm.ptrmask.p0.i64(ptr %[[TMP0]], i64 -16)
 // IEEE: %[[V4:[0-9a-zA-Z_.]+]] = load fp128, ptr %[[ALIGN]], align 16
 // IEEE: call void @foo_ld(fp128 noundef %[[V4]])
-// IEEE: call void @llvm.va_end(ptr %[[AP]])
+// IEEE: call void @llvm.va_end.p0(ptr %[[AP]])
 
 // IBM-LABEL: define{{.*}} void @long_double
-// IBM: call void @llvm.va_start(ptr  %[[AP:[0-9a-zA-Z_.]+]])
+// IBM: call void @llvm.va_start.p0(ptr  %[[AP:[0-9a-zA-Z_.]+]])
 // IBM: %[[CUR:[0-9a-zA-Z_.]+]] = load ptr, ptr %[[AP]]
 // IBM: %[[V4:[0-9a-zA-Z_.]+]] = load ppc_fp128, ptr %[[CUR]], align 8
 // IBM: call void @foo_ld(ppc_fp128 noundef %[[V4]])
-// IBM: call void @llvm.va_end(ptr %[[AP]])
+// IBM: call void @llvm.va_end.p0(ptr %[[AP]])
 void long_double(int n, ...) {
   va_list ap;
   va_start(ap, n);
@@ -86,7 +86,7 @@ void long_double(int n, ...) {
 }
 
 // IEEE-LABEL: define{{.*}} void @long_double_struct
-// IEEE: call void @llvm.va_start(ptr %[[AP:[0-9a-zA-Z_.]+]])
+// IEEE: call void @llvm.va_start.p0(ptr %[[AP:[0-9a-zA-Z_.]+]])
 // IEEE: %[[CUR:[0-9a-zA-Z_.]+]] = load ptr, ptr %[[AP]]
 // IEEE: %[[TMP0:[^ ]+]] = getelementptr inbounds i8, ptr %[[CUR]], i32 15
 // IEEE: %[[ALIGN:[^ ]+]] = call ptr @llvm.ptrmask.p0.i64(ptr %[[TMP0]], i64 -16)
@@ -96,7 +96,7 @@ void long_double(int n, ...) {
 // IEEE: %[[COERCE:[0-9a-zA-Z_.]+]] = getelementptr inbounds %struct.ldbl128_s, ptr %[[TMP]], i32 0, i32 0
 // IEEE: %[[V4:[0-9a-zA-Z_.]+]] = load fp128, ptr %[[COERCE]], align 16
 // IEEE: call void @foo_ls(fp128 inreg %[[V4]])
-// IEEE: call void @llvm.va_end(ptr %[[AP]])
+// IEEE: call void @llvm.va_end.p0(ptr %[[AP]])
 void long_double_struct(int n, ...) {
   va_list ap;
   va_start(ap, n);
diff --git a/clang/test/CodeGen/RISCV/riscv32-vararg.c b/clang/test/CodeGen/RISCV/riscv32-vararg.c
index 1c4e41f2f54c..00e04eb89467 100644
--- a/clang/test/CodeGen/RISCV/riscv32-vararg.c
+++ b/clang/test/CodeGen/RISCV/riscv32-vararg.c
@@ -80,13 +80,13 @@ void f_va_caller(void) {
 // CHECK-NEXT:    [[VA:%.*]] = alloca ptr, align 4
 // CHECK-NEXT:    [[V:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    store ptr [[FMT]], ptr [[FMT_ADDR]], align 4
-// CHECK-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
 // CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[VA]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARGP_CUR]], align 4
 // CHECK-NEXT:    store i32 [[TMP0]], ptr [[V]], align 4
-// CHECK-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[V]], align 4
 // CHECK-NEXT:    ret i32 [[TMP1]]
 //
@@ -111,7 +111,7 @@ int f_va_1(char *fmt, ...) {
 // CHECK-ILP32F-NEXT:    [[VA:%.*]] = alloca ptr, align 4
 // CHECK-ILP32F-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK-ILP32F-NEXT:    store ptr [[FMT]], ptr [[FMT_ADDR]], align 4
-// CHECK-ILP32F-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-ILP32F-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-ILP32F-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-ILP32F-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 7
 // CHECK-ILP32F-NEXT:    [[ARGP_CUR_ALIGNED:%.*]] = call ptr @llvm.ptrmask.p0.i32(ptr [[TMP0]], i32 -8)
@@ -119,7 +119,7 @@ int f_va_1(char *fmt, ...) {
 // CHECK-ILP32F-NEXT:    store ptr [[ARGP_NEXT]], ptr [[VA]], align 4
 // CHECK-ILP32F-NEXT:    [[TMP1:%.*]] = load double, ptr [[ARGP_CUR_ALIGNED]], align 8
 // CHECK-ILP32F-NEXT:    store double [[TMP1]], ptr [[V]], align 8
-// CHECK-ILP32F-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-ILP32F-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-ILP32F-NEXT:    [[TMP2:%.*]] = load double, ptr [[V]], align 8
 // CHECK-ILP32F-NEXT:    ret double [[TMP2]]
 //
@@ -130,7 +130,7 @@ int f_va_1(char *fmt, ...) {
 // CHECK-ILP32D-NEXT:    [[VA:%.*]] = alloca ptr, align 4
 // CHECK-ILP32D-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK-ILP32D-NEXT:    store ptr [[FMT]], ptr [[FMT_ADDR]], align 4
-// CHECK-ILP32D-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-ILP32D-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-ILP32D-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-ILP32D-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 7
 // CHECK-ILP32D-NEXT:    [[ARGP_CUR_ALIGNED:%.*]] = call ptr @llvm.ptrmask.p0.i32(ptr [[TMP0]], i32 -8)
@@ -138,7 +138,7 @@ int f_va_1(char *fmt, ...) {
 // CHECK-ILP32D-NEXT:    store ptr [[ARGP_NEXT]], ptr [[VA]], align 4
 // CHECK-ILP32D-NEXT:    [[TMP1:%.*]] = load double, ptr [[ARGP_CUR_ALIGNED]], align 8
 // CHECK-ILP32D-NEXT:    store double [[TMP1]], ptr [[V]], align 8
-// CHECK-ILP32D-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-ILP32D-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-ILP32D-NEXT:    [[TMP2:%.*]] = load double, ptr [[V]], align 8
 // CHECK-ILP32D-NEXT:    ret double [[TMP2]]
 //
@@ -149,13 +149,13 @@ int f_va_1(char *fmt, ...) {
 // CHECK-ILP32E-NEXT:    [[VA:%.*]] = alloca ptr, align 4
 // CHECK-ILP32E-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK-ILP32E-NEXT:    store ptr [[FMT]], ptr [[FMT_ADDR]], align 4
-// CHECK-ILP32E-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-ILP32E-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-ILP32E-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-ILP32E-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 8
 // CHECK-ILP32E-NEXT:    store ptr [[ARGP_NEXT]], ptr [[VA]], align 4
 // CHECK-ILP32E-NEXT:    [[TMP0:%.*]] = load double, ptr [[ARGP_CUR]], align 4
 // CHECK-ILP32E-NEXT:    store double [[TMP0]], ptr [[V]], align 8
-// CHECK-ILP32E-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-ILP32E-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-ILP32E-NEXT:    [[TMP1:%.*]] = load double, ptr [[V]], align 8
 // CHECK-ILP32E-NEXT:    ret double [[TMP1]]
 //
@@ -180,7 +180,7 @@ double f_va_2(char *fmt, ...) {
 // CHECK-ILP32F-NEXT:    [[W:%.*]] = alloca i32, align 4
 // CHECK-ILP32F-NEXT:    [[X:%.*]] = alloca double, align 8
 // CHECK-ILP32F-NEXT:    store ptr [[FMT]], ptr [[FMT_ADDR]], align 4
-// CHECK-ILP32F-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-ILP32F-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-ILP32F-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-ILP32F-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 7
 // CHECK-ILP32F-NEXT:    [[ARGP_CUR_ALIGNED:%.*]] = call ptr @llvm.ptrmask.p0.i32(ptr [[TMP0]], i32 -8)
@@ -200,7 +200,7 @@ double f_va_2(char *fmt, ...) {
 // CHECK-ILP32F-NEXT:    store ptr [[ARGP_NEXT4]], ptr [[VA]], align 4
 // CHECK-ILP32F-NEXT:    [[TMP4:%.*]] = load double, ptr [[ARGP_CUR3_ALIGNED]], align 8
 // CHECK-ILP32F-NEXT:    store double [[TMP4]], ptr [[X]], align 8
-// CHECK-ILP32F-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-ILP32F-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-ILP32F-NEXT:    [[TMP5:%.*]] = load double, ptr [[V]], align 8
 // CHECK-ILP32F-NEXT:    [[TMP6:%.*]] = load double, ptr [[X]], align 8
 // CHECK-ILP32F-NEXT:    [[ADD:%.*]] = fadd double [[TMP5]], [[TMP6]]
@@ -215,7 +215,7 @@ double f_va_2(char *fmt, ...) {
 // CHECK-ILP32D-NEXT:    [[W:%.*]] = alloca i32, align 4
 // CHECK-ILP32D-NEXT:    [[X:%.*]] = alloca double, align 8
 // CHECK-ILP32D-NEXT:    store ptr [[FMT]], ptr [[FMT_ADDR]], align 4
-// CHECK-ILP32D-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-ILP32D-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-ILP32D-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-ILP32D-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 7
 // CHECK-ILP32D-NEXT:    [[ARGP_CUR_ALIGNED:%.*]] = call ptr @llvm.ptrmask.p0.i32(ptr [[TMP0]], i32 -8)
@@ -235,7 +235,7 @@ double f_va_2(char *fmt, ...) {
 // CHECK-ILP32D-NEXT:    store ptr [[ARGP_NEXT4]], ptr [[VA]], align 4
 // CHECK-ILP32D-NEXT:    [[TMP4:%.*]] = load double, ptr [[ARGP_CUR3_ALIGNED]], align 8
 // CHECK-ILP32D-NEXT:    store double [[TMP4]], ptr [[X]], align 8
-// CHECK-ILP32D-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-ILP32D-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-ILP32D-NEXT:    [[TMP5:%.*]] = load double, ptr [[V]], align 8
 // CHECK-ILP32D-NEXT:    [[TMP6:%.*]] = load double, ptr [[X]], align 8
 // CHECK-ILP32D-NEXT:    [[ADD:%.*]] = fadd double [[TMP5]], [[TMP6]]
@@ -250,7 +250,7 @@ double f_va_2(char *fmt, ...) {
 // CHECK-ILP32E-NEXT:    [[W:%.*]] = alloca i32, align 4
 // CHECK-ILP32E-NEXT:    [[X:%.*]] = alloca double, align 8
 // CHECK-ILP32E-NEXT:    store ptr [[FMT]], ptr [[FMT_ADDR]], align 4
-// CHECK-ILP32E-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-ILP32E-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-ILP32E-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-ILP32E-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 8
 // CHECK-ILP32E-NEXT:    store ptr [[ARGP_NEXT]], ptr [[VA]], align 4
@@ -266,7 +266,7 @@ double f_va_2(char *fmt, ...) {
 // CHECK-ILP32E-NEXT:    store ptr [[ARGP_NEXT4]], ptr [[VA]], align 4
 // CHECK-ILP32E-NEXT:    [[TMP2:%.*]] = load double, ptr [[ARGP_CUR3]], align 4
 // CHECK-ILP32E-NEXT:    store double [[TMP2]], ptr [[X]], align 8
-// CHECK-ILP32E-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-ILP32E-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-ILP32E-NEXT:    [[TMP3:%.*]] = load double, ptr [[V]], align 8
 // CHECK-ILP32E-NEXT:    [[TMP4:%.*]] = load double, ptr [[X]], align 8
 // CHECK-ILP32E-NEXT:    [[ADD:%.*]] = fadd double [[TMP3]], [[TMP4]]
@@ -296,7 +296,7 @@ double f_va_3(char *fmt, ...) {
 // CHECK-ILP32F-NEXT:    [[LS:%.*]] = alloca [[STRUCT_LARGE:%.*]], align 4
 // CHECK-ILP32F-NEXT:    [[RET:%.*]] = alloca i32, align 4
 // CHECK-ILP32F-NEXT:    store ptr [[FMT]], ptr [[FMT_ADDR]], align 4
-// CHECK-ILP32F-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-ILP32F-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-ILP32F-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-ILP32F-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
 // CHECK-ILP32F-NEXT:    store ptr [[ARGP_NEXT]], ptr [[VA]], align 4
@@ -321,7 +321,7 @@ double f_va_3(char *fmt, ...) {
 // CHECK-ILP32F-NEXT:    store ptr [[ARGP_NEXT8]], ptr [[VA]], align 4
 // CHECK-ILP32F-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[ARGP_CUR7]], align 4
 // CHECK-ILP32F-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[LS]], ptr align 4 [[TMP3]], i32 16, i1 false)
-// CHECK-ILP32F-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-ILP32F-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-ILP32F-NEXT:    [[TMP4:%.*]] = load i32, ptr [[V]], align 4
 // CHECK-ILP32F-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP4]] to fp128
 // CHECK-ILP32F-NEXT:    [[TMP5:%.*]] = load fp128, ptr [[LD]], align 16
@@ -384,7 +384,7 @@ double f_va_3(char *fmt, ...) {
 // CHECK-ILP32D-NEXT:    [[LS:%.*]] = alloca [[STRUCT_LARGE:%.*]], align 4
 // CHECK-ILP32D-NEXT:    [[RET:%.*]] = alloca i32, align 4
 // CHECK-ILP32D-NEXT:    store ptr [[FMT]], ptr [[FMT_ADDR]], align 4
-// CHECK-ILP32D-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-ILP32D-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-ILP32D-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-ILP32D-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
 // CHECK-ILP32D-NEXT:    store ptr [[ARGP_NEXT]], ptr [[VA]], align 4
@@ -409,7 +409,7 @@ double f_va_3(char *fmt, ...) {
 // CHECK-ILP32D-NEXT:    store ptr [[ARGP_NEXT8]], ptr [[VA]], align 4
 // CHECK-ILP32D-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[ARGP_CUR7]], align 4
 // CHECK-ILP32D-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[LS]], ptr align 4 [[TMP3]], i32 16, i1 false)
-// CHECK-ILP32D-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-ILP32D-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-ILP32D-NEXT:    [[TMP4:%.*]] = load i32, ptr [[V]], align 4
 // CHECK-ILP32D-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP4]] to fp128
 // CHECK-ILP32D-NEXT:    [[TMP5:%.*]] = load fp128, ptr [[LD]], align 16
@@ -472,7 +472,7 @@ double f_va_3(char *fmt, ...) {
 // CHECK-ILP32E-NEXT:    [[LS:%.*]] = alloca [[STRUCT_LARGE:%.*]], align 4
 // CHECK-ILP32E-NEXT:    [[RET:%.*]] = alloca i32, align 4
 // CHECK-ILP32E-NEXT:    store ptr [[FMT]], ptr [[FMT_ADDR]], align 4
-// CHECK-ILP32E-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-ILP32E-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-ILP32E-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-ILP32E-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
 // CHECK-ILP32E-NEXT:    store ptr [[ARGP_NEXT]], ptr [[VA]], align 4
@@ -497,7 +497,7 @@ double f_va_3(char *fmt, ...) {
 // CHECK-ILP32E-NEXT:    store ptr [[ARGP_NEXT8]], ptr [[VA]], align 4
 // CHECK-ILP32E-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[ARGP_CUR7]], align 4
 // CHECK-ILP32E-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[LS]], ptr align 4 [[TMP3]], i32 16, i1 false)
-// CHECK-ILP32E-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-ILP32E-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-ILP32E-NEXT:    [[TMP4:%.*]] = load i32, ptr [[V]], align 4
 // CHECK-ILP32E-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP4]] to fp128
 // CHECK-ILP32E-NEXT:    [[TMP5:%.*]] = load fp128, ptr [[LD]], align 16
diff --git a/clang/test/CodeGen/RISCV/riscv64-vararg.c b/clang/test/CodeGen/RISCV/riscv64-vararg.c
index 634cde61320c..efdffa2687e6 100644
--- a/clang/test/CodeGen/RISCV/riscv64-vararg.c
+++ b/clang/test/CodeGen/RISCV/riscv64-vararg.c
@@ -135,13 +135,13 @@ void f_va_caller(void) {
 // CHECK-NEXT:    [[VA:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    [[V:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    store ptr [[FMT]], ptr [[FMT_ADDR]], align 8
-// CHECK-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 8
 // CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i64 8
 // CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[VA]], align 8
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARGP_CUR]], align 8
 // CHECK-NEXT:    store i32 [[TMP0]], ptr [[V]], align 4
-// CHECK-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[V]], align 4
 // CHECK-NEXT:    ret i32 [[TMP1]]
 //
@@ -166,7 +166,7 @@ int f_va_1(char *fmt, ...) {
 // CHECK-NEXT:    [[VA:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    [[V:%.*]] = alloca fp128, align 16
 // CHECK-NEXT:    store ptr [[FMT]], ptr [[FMT_ADDR]], align 8
-// CHECK-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 8
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 15
 // CHECK-NEXT:    [[ARGP_CUR_ALIGNED:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[TMP0]], i64 -16)
@@ -174,7 +174,7 @@ int f_va_1(char *fmt, ...) {
 // CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[VA]], align 8
 // CHECK-NEXT:    [[TMP1:%.*]] = load fp128, ptr [[ARGP_CUR_ALIGNED]], align 16
 // CHECK-NEXT:    store fp128 [[TMP1]], ptr [[V]], align 16
-// CHECK-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-NEXT:    [[TMP2:%.*]] = load fp128, ptr [[V]], align 16
 // CHECK-NEXT:    ret fp128 [[TMP2]]
 //
@@ -199,7 +199,7 @@ long double f_va_2(char *fmt, ...) {
 // CHECK-NEXT:    [[W:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    [[X:%.*]] = alloca fp128, align 16
 // CHECK-NEXT:    store ptr [[FMT]], ptr [[FMT_ADDR]], align 8
-// CHECK-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 8
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 15
 // CHECK-NEXT:    [[ARGP_CUR_ALIGNED:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[TMP0]], i64 -16)
@@ -219,7 +219,7 @@ long double f_va_2(char *fmt, ...) {
 // CHECK-NEXT:    store ptr [[ARGP_NEXT4]], ptr [[VA]], align 8
 // CHECK-NEXT:    [[TMP4:%.*]] = load fp128, ptr [[ARGP_CUR3_ALIGNED]], align 16
 // CHECK-NEXT:    store fp128 [[TMP4]], ptr [[X]], align 16
-// CHECK-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-NEXT:    [[TMP5:%.*]] = load fp128, ptr [[V]], align 16
 // CHECK-NEXT:    [[TMP6:%.*]] = load fp128, ptr [[X]], align 16
 // CHECK-NEXT:    [[ADD:%.*]] = fadd fp128 [[TMP5]], [[TMP6]]
@@ -248,7 +248,7 @@ long double f_va_3(char *fmt, ...) {
 // CHECK-NEXT:    [[LS:%.*]] = alloca [[STRUCT_LARGE:%.*]], align 8
 // CHECK-NEXT:    [[RET:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    store ptr [[FMT]], ptr [[FMT_ADDR]], align 8
-// CHECK-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 8
 // CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i64 8
 // CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[VA]], align 8
@@ -267,7 +267,7 @@ long double f_va_3(char *fmt, ...) {
 // CHECK-NEXT:    store ptr [[ARGP_NEXT6]], ptr [[VA]], align 8
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[ARGP_CUR5]], align 8
 // CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[LS]], ptr align 8 [[TMP1]], i64 32, i1 false)
-// CHECK-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-NEXT:    [[A:%.*]] = getelementptr inbounds [[STRUCT_TINY]], ptr [[TS]], i32 0, i32 0
 // CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
 // CHECK-NEXT:    [[CONV:%.*]] = zext i16 [[TMP2]] to i64
diff --git a/clang/test/CodeGen/WebAssembly/wasm-varargs.c b/clang/test/CodeGen/WebAssembly/wasm-varargs.c
index c475de19ae44..e794857304e1 100644
--- a/clang/test/CodeGen/WebAssembly/wasm-varargs.c
+++ b/clang/test/CodeGen/WebAssembly/wasm-varargs.c
@@ -10,13 +10,13 @@
 // CHECK-NEXT:    [[VA:%.*]] = alloca ptr, align 4
 // CHECK-NEXT:    [[V:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    store ptr [[FMT]], ptr [[FMT_ADDR]], align 4
-// CHECK-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
 // CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[VA]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARGP_CUR]], align 4
 // CHECK-NEXT:    store i32 [[TMP0]], ptr [[V]], align 4
-// CHECK-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[V]], align 4
 // CHECK-NEXT:    ret i32 [[TMP1]]
 //
@@ -38,7 +38,7 @@ int test_i32(char *fmt, ...) {
 // CHECK-NEXT:    [[VA:%.*]] = alloca ptr, align 4
 // CHECK-NEXT:    [[V:%.*]] = alloca i64, align 8
 // CHECK-NEXT:    store ptr [[FMT]], ptr [[FMT_ADDR]], align 4
-// CHECK-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 7
 // CHECK-NEXT:    [[ARGP_CUR_ALIGNED:%.*]] = call ptr @llvm.ptrmask.p0.i32(ptr [[TMP0]], i32 -8)
@@ -46,7 +46,7 @@ int test_i32(char *fmt, ...) {
 // CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[VA]], align 4
 // CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr [[ARGP_CUR_ALIGNED]], align 8
 // CHECK-NEXT:    store i64 [[TMP1]], ptr [[V]], align 8
-// CHECK-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr [[V]], align 8
 // CHECK-NEXT:    ret i64 [[TMP2]]
 //
@@ -73,13 +73,13 @@ struct S {
 // CHECK-NEXT:    [[FMT_ADDR:%.*]] = alloca ptr, align 4
 // CHECK-NEXT:    [[VA:%.*]] = alloca ptr, align 4
 // CHECK-NEXT:    store ptr [[FMT]], ptr [[FMT_ADDR]], align 4
-// CHECK-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 4
 // CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[VA]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARGP_CUR]], align 4
 // CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[AGG_RESULT]], ptr align 4 [[TMP0]], i32 12, i1 false)
-// CHECK-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-NEXT:    ret void
 //
 struct S test_struct(char *fmt, ...) {
@@ -102,7 +102,7 @@ struct Z {};
 // CHECK-NEXT:    [[VA:%.*]] = alloca ptr, align 4
 // CHECK-NEXT:    [[U:%.*]] = alloca [[STRUCT_Z:%.*]], align 1
 // CHECK-NEXT:    store ptr [[FMT]], ptr [[FMT_ADDR]], align 4
-// CHECK-NEXT:    call void @llvm.va_start(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[VA]])
 // CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[VA]], align 4
 // CHECK-NEXT:    [[ARGP_NEXT:%.*]] = getelementptr inbounds i8, ptr [[ARGP_CUR]], i32 0
 // CHECK-NEXT:    store ptr [[ARGP_NEXT]], ptr [[VA]], align 4
@@ -112,7 +112,7 @@ struct Z {};
 // CHECK-NEXT:    store ptr [[ARGP_NEXT2]], ptr [[VA]], align 4
 // CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ARGP_CUR1]], align 4
 // CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[AGG_RESULT]], ptr align 4 [[TMP0]], i32 12, i1 false)
-// CHECK-NEXT:    call void @llvm.va_end(ptr [[VA]])
+// CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VA]])
 // CHECK-NEXT:    ret void
 //
 struct S test_empty_struct(char *fmt, ...) {
diff --git a/clang/test/CodeGen/X86/va-arg-sse.c b/clang/test/CodeGen/X86/va-arg-sse.c
index e040b0e5790b..b7d00dad1453 100644
--- a/clang/test/CodeGen/X86/va-arg-sse.c
+++ b/clang/test/CodeGen/X86/va-arg-sse.c
@@ -21,7 +21,7 @@ struct S a[5];
 // CHECK-NEXT:    store i32 0, ptr [[J]], align 4
 // CHECK-NEXT:    store i32 0, ptr [[K]], align 4
 // CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[AP]], i64 0, i64 0
-// CHECK-NEXT:    call void @llvm.va_start(ptr [[ARRAYDECAY]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[ARRAYDECAY]])
 // CHECK-NEXT:    store ptr getelementptr inbounds ([5 x %struct.S], ptr @a, i64 0, i64 2), ptr [[P]], align 8
 // CHECK-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[AP]], i64 0, i64 0
 // CHECK-NEXT:    [[FP_OFFSET_P:%.*]] = getelementptr inbounds [[STRUCT___VA_LIST_TAG:%.*]], ptr [[ARRAYDECAY2]], i32 0, i32 1
@@ -52,7 +52,7 @@ struct S a[5];
 // CHECK-NEXT:    [[VAARG_ADDR:%.*]] = phi ptr [ [[TMP]], [[VAARG_IN_REG]] ], [ [[OVERFLOW_ARG_AREA]], [[VAARG_IN_MEM]] ]
 // CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[ARG]], ptr align 4 [[VAARG_ADDR]], i64 12, i1 false)
 // CHECK-NEXT:    [[ARRAYDECAY3:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[AP]], i64 0, i64 0
-// CHECK-NEXT:    call void @llvm.va_end(ptr [[ARRAYDECAY3]])
+// CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[ARRAYDECAY3]])
 // CHECK-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[P]], align 8
 // CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne ptr [[TMP15]], null
 // CHECK-NEXT:    br i1 [[TOBOOL]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
diff --git a/clang/test/CodeGen/X86/x86_64-vaarg.c b/clang/test/CodeGen/X86/x86_64-vaarg.c
index a18ba8364238..07c6df14a0b8 100644
--- a/clang/test/CodeGen/X86/x86_64-vaarg.c
+++ b/clang/test/CodeGen/X86/x86_64-vaarg.c
@@ -13,7 +13,7 @@ typedef struct { struct {} a; } empty;
 // CHECK-NEXT:    [[TMP:%.*]] = alloca [[STRUCT_EMPTY]], align 1
 // CHECK-NEXT:    store i32 [[Z]], ptr [[Z_ADDR]], align 4
 // CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[LIST]], i64 0, i64 0
-// CHECK-NEXT:    call void @llvm.va_start(ptr [[ARRAYDECAY]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[ARRAYDECAY]])
 // CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[LIST]], i64 0, i64 0
 // CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[RETVAL]], ptr align 1 [[TMP]], i64 0, i1 false)
 // CHECK-NEXT:    ret void
@@ -37,7 +37,7 @@ typedef struct {
 // CHECK-NEXT:    [[LIST:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
 // CHECK-NEXT:    store i32 [[Z]], ptr [[Z_ADDR]], align 4
 // CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[LIST]], i64 0, i64 0
-// CHECK-NEXT:    call void @llvm.va_start(ptr [[ARRAYDECAY]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[ARRAYDECAY]])
 // CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[LIST]], i64 0, i64 0
 // CHECK-NEXT:    [[FP_OFFSET_P:%.*]] = getelementptr inbounds [[STRUCT___VA_LIST_TAG:%.*]], ptr [[ARRAYDECAY1]], i32 0, i32 1
 // CHECK-NEXT:    [[FP_OFFSET:%.*]] = load i32, ptr [[FP_OFFSET_P]], align 4
diff --git a/clang/test/CodeGen/aarch64-ABI-align-packed.c b/clang/test/CodeGen/aarch64-ABI-align-packed.c
index 2b029f645895..13c68fe54b84 100644
--- a/clang/test/CodeGen/aarch64-ABI-align-packed.c
+++ b/clang/test/CodeGen/aarch64-ABI-align-packed.c
@@ -73,7 +73,7 @@ __attribute__((noinline)) void named_arg_non_packed_struct(double d0, double d1,
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[VL:%.*]] = alloca [[STRUCT___VA_LIST:%.*]], align 8
 // CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6:[0-9]+]]
-// CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[VL]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[VL]])
 // CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void variadic_non_packed_struct(double d0, double d1, double d2, double d3,
@@ -128,7 +128,7 @@ __attribute__((noinline)) void named_arg_packed_struct(double d0, double d1, dou
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[VL:%.*]] = alloca [[STRUCT___VA_LIST:%.*]], align 8
 // CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
-// CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[VL]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[VL]])
 // CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void variadic_packed_struct(double d0, double d1, double d2, double d3,
@@ -183,7 +183,7 @@ __attribute__((noinline)) void named_arg_packed_member(double d0, double d1, dou
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[VL:%.*]] = alloca [[STRUCT___VA_LIST:%.*]], align 8
 // CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
-// CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[VL]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[VL]])
 // CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void variadic_packed_member(double d0, double d1, double d2, double d3,
@@ -238,7 +238,7 @@ __attribute__((noinline)) void named_arg_aligned_struct_8(double d0, double d1,
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[VL:%.*]] = alloca [[STRUCT___VA_LIST:%.*]], align 8
 // CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
-// CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[VL]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[VL]])
 // CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void variadic_aligned_struct_8(double d0, double d1, double d2, double d3,
@@ -293,7 +293,7 @@ __attribute__((noinline)) void named_arg_aligned_member_8(double d0, double d1,
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[VL:%.*]] = alloca [[STRUCT___VA_LIST:%.*]], align 8
 // CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
-// CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[VL]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[VL]])
 // CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void variadic_aligned_member_8(double d0, double d1, double d2, double d3,
@@ -348,7 +348,7 @@ __attribute__((noinline)) void named_arg_pragma_packed_struct_8(double d0, doubl
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[VL:%.*]] = alloca [[STRUCT___VA_LIST:%.*]], align 8
 // CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
-// CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[VL]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[VL]])
 // CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void variadic_pragma_packed_struct_8(double d0, double d1, double d2, double d3,
@@ -403,7 +403,7 @@ __attribute__((noinline)) void named_arg_pragma_packed_struct_4(double d0, doubl
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[VL:%.*]] = alloca [[STRUCT___VA_LIST:%.*]], align 8
 // CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
-// CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[VL]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[VL]])
 // CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void variadic_pragma_packed_struct_4(double d0, double d1, double d2, double d3,
diff --git a/clang/test/CodeGen/aarch64-varargs.c b/clang/test/CodeGen/aarch64-varargs.c
index 44b87029e7b3..ee4e88eda4ef 100644
--- a/clang/test/CodeGen/aarch64-varargs.c
+++ b/clang/test/CodeGen/aarch64-varargs.c
@@ -837,7 +837,7 @@ void check_start(int n, ...) {
   va_list the_list;
   va_start(the_list, n);
 // CHECK: [[THE_LIST:%[a-z_0-9]+]] = alloca %struct.__va_list
-// CHECK: call void @llvm.va_start(ptr [[THE_LIST]])
+// CHECK: call void @llvm.va_start.p0(ptr [[THE_LIST]])
 }
 
 typedef struct {} empty;
diff --git a/clang/test/CodeGen/arm-varargs.c b/clang/test/CodeGen/arm-varargs.c
index f754c7f52e59..ab4ac46924e6 100644
--- a/clang/test/CodeGen/arm-varargs.c
+++ b/clang/test/CodeGen/arm-varargs.c
@@ -264,5 +264,5 @@ void check_start(int n, ...) {
   va_list the_list;
   va_start(the_list, n);
 // CHECK: [[THE_LIST:%[a-z0-9._]+]] = alloca %struct.__va_list
-// CHECK: call void @llvm.va_start(ptr [[THE_LIST]])
+// CHECK: call void @llvm.va_start.p0(ptr [[THE_LIST]])
 }
diff --git a/clang/test/CodeGen/hexagon-linux-vararg.c b/clang/test/CodeGen/hexagon-linux-vararg.c
index 033e72ab449d..84945e872d28 100644
--- a/clang/test/CodeGen/hexagon-linux-vararg.c
+++ b/clang/test/CodeGen/hexagon-linux-vararg.c
@@ -9,7 +9,7 @@ struct AAA {
   int d;
 };
 
-// CHECK:   call void @llvm.va_start(ptr %arraydecay)
+// CHECK:   call void @llvm.va_start.p0(ptr %arraydecay)
 // CHECK:   %arraydecay1 = getelementptr inbounds [1 x %struct.__va_list_tag],
 // ptr %ap, i32 0, i32 0
 // CHECK:   br label %vaarg.maybe_reg
diff --git a/clang/test/CodeGen/mips-varargs.c b/clang/test/CodeGen/mips-varargs.c
index 052aedd1cd1e..029f000c121a 100644
--- a/clang/test/CodeGen/mips-varargs.c
+++ b/clang/test/CodeGen/mips-varargs.c
@@ -29,7 +29,7 @@ int test_i32(char *fmt, ...) {
 // ALL:   [[V:%.*]] = alloca i32, align 4
 // NEW:   [[PROMOTION_TEMP:%.*]] = alloca i32, align 4
 //
-// ALL:   call void @llvm.va_start(ptr %va)
+// ALL:   call void @llvm.va_start.p0(ptr %va)
 // ALL:   [[AP_CUR:%.+]] = load ptr, ptr %va, align [[$PTRALIGN]]
 // O32:   [[AP_NEXT:%.+]] = getelementptr inbounds i8, ptr [[AP_CUR]], [[$INTPTR_T:i32]] [[$CHUNKSIZE:4]]
 // NEW:   [[AP_NEXT:%.+]] = getelementptr inbounds i8, ptr [[AP_CUR]], [[$INTPTR_T:i32|i64]] [[$CHUNKSIZE:8]]
@@ -45,7 +45,7 @@ int test_i32(char *fmt, ...) {
 // NEW:   [[ARG:%.+]] = load i32, ptr [[PROMOTION_TEMP]], align 4
 // ALL:   store i32 [[ARG]], ptr [[V]], align 4
 //
-// ALL:   call void @llvm.va_end(ptr %va)
+// ALL:   call void @llvm.va_end.p0(ptr %va)
 // ALL: }
 
 long long test_i64(char *fmt, ...) {
@@ -61,7 +61,7 @@ long long test_i64(char *fmt, ...) {
 // ALL-LABEL: define{{.*}} i64 @test_i64(ptr{{.*}} %fmt, ...)
 //
 // ALL:   %va = alloca ptr, align [[$PTRALIGN]]
-// ALL:   call void @llvm.va_start(ptr %va)
+// ALL:   call void @llvm.va_start.p0(ptr %va)
 // ALL:   [[AP_CUR:%.+]] = load ptr, ptr %va, align [[$PTRALIGN]]
 //
 // i64 is 8-byte aligned, while this is within O32's stack alignment there's no
@@ -74,7 +74,7 @@ long long test_i64(char *fmt, ...) {
 //
 // ALL:   [[ARG:%.+]] = load i64, ptr [[AP_CUR]], align 8
 //
-// ALL:   call void @llvm.va_end(ptr %va)
+// ALL:   call void @llvm.va_end.p0(ptr %va)
 // ALL: }
 
 char *test_ptr(char *fmt, ...) {
@@ -92,7 +92,7 @@ char *test_ptr(char *fmt, ...) {
 // ALL:   %va = alloca ptr, align [[$PTRALIGN]]
 // ALL:   [[V:%.*]] = alloca ptr, align [[$PTRALIGN]]
 // N32:   [[AP_CAST:%.+]] = alloca ptr, align 4
-// ALL:   call void @llvm.va_start(ptr %va)
+// ALL:   call void @llvm.va_start.p0(ptr %va)
 // ALL:   [[AP_CUR:%.+]] = load ptr, ptr %va, align [[$PTRALIGN]]
 // ALL:   [[AP_NEXT:%.+]] = getelementptr inbounds i8, ptr [[AP_CUR]], [[$INTPTR_T]] [[$CHUNKSIZE]]
 // ALL:   store ptr [[AP_NEXT]], ptr %va, align [[$PTRALIGN]]
@@ -109,7 +109,7 @@ char *test_ptr(char *fmt, ...) {
 // N64:   [[ARG:%.+]] = load ptr, ptr [[AP_CUR]], align [[$PTRALIGN]]
 // ALL:   store ptr [[ARG]], ptr [[V]], align [[$PTRALIGN]]
 //
-// ALL:   call void @llvm.va_end(ptr %va)
+// ALL:   call void @llvm.va_end.p0(ptr %va)
 // ALL: }
 
 int test_v4i32(char *fmt, ...) {
@@ -128,7 +128,7 @@ int test_v4i32(char *fmt, ...) {
 //
 // ALL:   %va = alloca ptr, align [[$PTRALIGN]]
 // ALL:   [[V:%.+]] = alloca <4 x i32>, align 16
-// ALL:   call void @llvm.va_start(ptr %va)
+// ALL:   call void @llvm.va_start.p0(ptr %va)
 // ALL:   [[AP_CUR:%.+]] = load ptr, ptr %va, align [[$PTRALIGN]]
 //
 // Vectors are 16-byte aligned, however the O32 ABI has a maximum alignment of
@@ -152,7 +152,7 @@ int test_v4i32(char *fmt, ...) {
 // N32:   [[ARG:%.+]] = load <4 x i32>, ptr [[AP_CUR]], align 16
 // ALL:   store <4 x i32> [[ARG]], ptr [[V]], align 16
 //
-// ALL:   call void @llvm.va_end(ptr %va)
+// ALL:   call void @llvm.va_end.p0(ptr %va)
 // ALL:   [[VECEXT:%.+]] = extractelement <4 x i32> {{.*}}, i32 0
 // ALL:   ret i32 [[VECEXT]]
 // ALL: }
diff --git a/clang/test/CodeGen/pr53127.cpp b/clang/test/CodeGen/pr53127.cpp
index 97fe1291352d..5a52b4860eec 100644
--- a/clang/test/CodeGen/pr53127.cpp
+++ b/clang/test/CodeGen/pr53127.cpp
@@ -34,7 +34,7 @@ void operator delete(void*);
 // CHECK-NEXT:    br i1 [[CALL6]], label [[COND_TRUE7:%.*]], label [[COND_FALSE8:%.*]]
 // CHECK:       cond.true7:
 // CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[L]], i64 0, i64 0
-// CHECK-NEXT:    call void @llvm.va_start(ptr [[ARRAYDECAY]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[ARRAYDECAY]])
 // CHECK-NEXT:    br label [[COND_END9:%.*]]
 // CHECK:       cond.false8:
 // CHECK-NEXT:    br label [[COND_END9]]
@@ -44,7 +44,7 @@ void operator delete(void*);
 // CHECK:       cond.true11:
 // CHECK-NEXT:    [[ARRAYDECAY12:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[L]], i64 0, i64 0
 // CHECK-NEXT:    [[ARRAYDECAY13:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[L2]], i64 0, i64 0
-// CHECK-NEXT:    call void @llvm.va_copy(ptr [[ARRAYDECAY12]], ptr [[ARRAYDECAY13]])
+// CHECK-NEXT:    call void @llvm.va_copy.p0(ptr [[ARRAYDECAY12]], ptr [[ARRAYDECAY13]])
 // CHECK-NEXT:    br label [[COND_END15:%.*]]
 // CHECK:       cond.false14:
 // CHECK-NEXT:    br label [[COND_END15]]
diff --git a/clang/test/CodeGen/varargs-with-nonzero-default-address-space.c b/clang/test/CodeGen/varargs-with-nonzero-default-address-space.c
new file mode 100644
index 000000000000..b087da34c3df
--- /dev/null
+++ b/clang/test/CodeGen/varargs-with-nonzero-default-address-space.c
@@ -0,0 +1,46 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4
+// RUN: %clang_cc1 -triple spirv64-unknown-unknown -fcuda-is-device -emit-llvm -o - %s | FileCheck %s
+
+struct x {
+  double b;
+  long a;
+};
+
+// CHECK-LABEL: define spir_func void @testva(
+// CHECK-SAME: i32 noundef [[N:%.*]], ...) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[AP:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[T:%.*]] = alloca [[STRUCT_X:%.*]], align 8
+// CHECK-NEXT:    [[AP2:%.*]] = alloca ptr addrspace(4), align 8
+// CHECK-NEXT:    [[V:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[VARET:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[N_ADDR_ASCAST:%.*]] = addrspacecast ptr [[N_ADDR]] to ptr addrspace(4)
+// CHECK-NEXT:    [[AP_ASCAST:%.*]] = addrspacecast ptr [[AP]] to ptr addrspace(4)
+// CHECK-NEXT:    [[T_ASCAST:%.*]] = addrspacecast ptr [[T]] to ptr addrspace(4)
+// CHECK-NEXT:    [[AP2_ASCAST:%.*]] = addrspacecast ptr [[AP2]] to ptr addrspace(4)
+// CHECK-NEXT:    [[V_ASCAST:%.*]] = addrspacecast ptr [[V]] to ptr addrspace(4)
+// CHECK-NEXT:    [[VARET_ASCAST:%.*]] = addrspacecast ptr [[VARET]] to ptr addrspace(4)
+// CHECK-NEXT:    store i32 [[N]], ptr addrspace(4) [[N_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    call void @llvm.va_start.p4(ptr addrspace(4) [[AP_ASCAST]])
+// CHECK-NEXT:    [[TMP0:%.*]] = va_arg ptr addrspace(4) [[AP_ASCAST]], ptr
+// CHECK-NEXT:    call void @llvm.memcpy.p4.p0.i64(ptr addrspace(4) align 8 [[T_ASCAST]], ptr align 8 [[TMP0]], i64 16, i1 false)
+// CHECK-NEXT:    call void @llvm.va_copy.p4(ptr addrspace(4) [[AP2_ASCAST]], ptr addrspace(4) [[AP_ASCAST]])
+// CHECK-NEXT:    [[TMP1:%.*]] = va_arg ptr addrspace(4) [[AP2_ASCAST]], i32
+// CHECK-NEXT:    store i32 [[TMP1]], ptr addrspace(4) [[VARET_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(4) [[VARET_ASCAST]], align 4
+// CHECK-NEXT:    store i32 [[TMP2]], ptr addrspace(4) [[V_ASCAST]], align 4
+// CHECK-NEXT:    call void @llvm.va_end.p4(ptr addrspace(4) [[AP2_ASCAST]])
+// CHECK-NEXT:    call void @llvm.va_end.p4(ptr addrspace(4) [[AP_ASCAST]])
+// CHECK-NEXT:    ret void
+
+void testva(int n, ...) {
+  __builtin_va_list ap;
+  __builtin_va_start(ap, n);
+  struct x t = __builtin_va_arg(ap, struct x);
+  __builtin_va_list ap2;
+  __builtin_va_copy(ap2, ap);
+  int v = __builtin_va_arg(ap2, int);
+  __builtin_va_end(ap2);
+  __builtin_va_end(ap);
+}
diff --git a/clang/test/CodeGen/xcore-abi.c b/clang/test/CodeGen/xcore-abi.c
index 4dd0f221533b..bb8d2fec46bd 100644
--- a/clang/test/CodeGen/xcore-abi.c
+++ b/clang/test/CodeGen/xcore-abi.c
@@ -28,7 +28,7 @@ void testva (int n, ...) {
   // CHECK: [[AP:%[a-z0-9]+]] = alloca ptr, align 4
   // CHECK: [[V5:%[a-z0-9]+]] = alloca %struct.x, align 4
   // CHECK: [[TMP:%[a-z0-9]+]] = alloca [4 x i32], align 4
-  // CHECK: call void @llvm.va_start(ptr [[AP]])
+  // CHECK: call void @llvm.va_start.p0(ptr [[AP]])
 
   char* v1 = va_arg (ap, char*);
   f(v1);
diff --git a/clang/test/CodeGenCXX/ext-int.cpp b/clang/test/CodeGenCXX/ext-int.cpp
index 5a4270aef285..a1d17c840ee4 100644
--- a/clang/test/CodeGenCXX/ext-int.cpp
+++ b/clang/test/CodeGenCXX/ext-int.cpp
@@ -159,9 +159,9 @@ void TakesVarargs(int i, ...) {
   // WIN: %[[ARGS:.+]] = alloca ptr
   __builtin_va_start(args, i);
   // LIN64: %[[STARTAD:.+]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr %[[ARGS]]
-  // LIN64: call void @llvm.va_start(ptr %[[STARTAD]])
-  // LIN32: call void @llvm.va_start(ptr %[[ARGS]])
-  // WIN: call void @llvm.va_start(ptr %[[ARGS]])
+  // LIN64: call void @llvm.va_start.p0(ptr %[[STARTAD]])
+  // LIN32: call void @llvm.va_start.p0(ptr %[[ARGS]])
+  // WIN: call void @llvm.va_start.p0(ptr %[[ARGS]])
 
   _BitInt(92) A = __builtin_va_arg(args, _BitInt(92));
   // LIN64: %[[AD1:.+]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr %[[ARGS]]
@@ -302,9 +302,9 @@ void TakesVarargs(int i, ...) {
 
   __builtin_va_end(args);
   // LIN64: %[[ENDAD:.+]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr %[[ARGS]]
-  // LIN64: call void @llvm.va_end(ptr %[[ENDAD]])
-  // LIN32: call void @llvm.va_end(ptr %[[ARGS]])
-  // WIN: call void @llvm.va_end(ptr %[[ARGS]])
+  // LIN64: call void @llvm.va_end.p0(ptr %[[ENDAD]])
+  // LIN32: call void @llvm.va_end.p0(ptr %[[ARGS]])
+  // WIN: call void @llvm.va_end.p0(ptr %[[ARGS]])
 }
 void typeid_tests() {
   // LIN: define{{.*}} void @_Z12typeid_testsv()
diff --git a/clang/test/CodeGenCXX/ibm128-declarations.cpp b/clang/test/CodeGenCXX/ibm128-declarations.cpp
index 5ee4f354d379..e0187e20cde4 100644
--- a/clang/test/CodeGenCXX/ibm128-declarations.cpp
+++ b/clang/test/CodeGenCXX/ibm128-declarations.cpp
@@ -107,13 +107,13 @@ int main(void) {
 // CHECK: define dso_local noundef ppc_fp128 @_Z10func_vaargiz(i32 noundef signext %n, ...)
 // CHECK: entry:
 // CHECK:   store i32 %n, ptr %n.addr, align 4
-// CHECK:   call void @llvm.va_start(ptr %ap)
+// CHECK:   call void @llvm.va_start.p0(ptr %ap)
 // CHECK:   %argp.cur = load ptr, ptr %ap, align 8
 // CHECK:   %argp.next = getelementptr inbounds i8, ptr %argp.cur, i64 16
 // CHECK:   store ptr %argp.next, ptr %ap, align 8
 // CHECK:   %0 = load ppc_fp128, ptr %argp.cur, align 8
 // CHECK:   store ppc_fp128 %0, ptr %r, align 16
-// CHECK:   call void @llvm.va_end(ptr %ap)
+// CHECK:   call void @llvm.va_end.p0(ptr %ap)
 // CHECK:   %1 = load ppc_fp128, ptr %r, align 16
 // CHECK:   ret ppc_fp128 %1
 // CHECK: }
diff --git a/clang/test/CodeGenCXX/x86_64-vaarg.cpp b/clang/test/CodeGenCXX/x86_64-vaarg.cpp
index d221c1881d36..985a0cc41a14 100644
--- a/clang/test/CodeGenCXX/x86_64-vaarg.cpp
+++ b/clang/test/CodeGenCXX/x86_64-vaarg.cpp
@@ -11,7 +11,7 @@ typedef struct { struct {} a; } empty;
 // CHECK-NEXT:    [[TMP:%.*]] = alloca [[STRUCT_EMPTY]], align 1
 // CHECK-NEXT:    store i32 [[Z:%.*]], ptr [[Z_ADDR]], align 4
 // CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[LIST]], i64 0, i64 0
-// CHECK-NEXT:    call void @llvm.va_start(ptr [[ARRAYDECAY]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[ARRAYDECAY]])
 // CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[LIST]], i64 0, i64 0
 // CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[RETVAL]], ptr align 1 [[TMP]], i64 1, i1 false)
 // CHECK-NEXT:    ret void
@@ -34,7 +34,7 @@ typedef struct {
 // CHECK-NEXT:    [[LIST:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
 // CHECK-NEXT:    store i32 [[Z:%.*]], ptr [[Z_ADDR]], align 4
 // CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[LIST]], i64 0, i64 0
-// CHECK-NEXT:    call void @llvm.va_start(ptr [[ARRAYDECAY]])
+// CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[ARRAYDECAY]])
 // CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[LIST]], i64 0, i64 0
 // CHECK-NEXT:    [[FP_OFFSET_P:%.*]] = getelementptr inbounds [[STRUCT___VA_LIST_TAG:%.*]], ptr [[ARRAYDECAY1]], i32 0, i32 1
 // CHECK-NEXT:    [[FP_OFFSET:%.*]] = load i32, ptr [[FP_OFFSET_P]], align 4
diff --git a/clang/test/Modules/codegen.test b/clang/test/Modules/codegen.test
index 77602056defd..0af630a75480 100644
--- a/clang/test/Modules/codegen.test
+++ b/clang/test/Modules/codegen.test
@@ -26,7 +26,7 @@ USE: $_Z4instIiEvv = comdat any
 USE: $_Z10always_inlv = comdat any
 FOO: $_ZN13implicit_dtorD2Ev = comdat any
 FOO: define weak_odr void @_Z2f1PKcz(ptr noundef %fmt, ...) #{{[0-9]+}} comdat
-FOO:   call void @llvm.va_start(ptr %{{[a-zA-Z0-9]*}})
+FOO:   call void @llvm.va_start.p0(ptr %{{[a-zA-Z0-9]*}})
 
 Test that implicit special members are emitted into the FOO module if they're
 ODR used there, otherwise emit them linkonce_odr as usual in the use.
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 030b76935c7b..a4be31576931 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -12815,10 +12815,11 @@ Variable argument support is defined in LLVM with the
 functions. These functions are related to the similarly named macros
 defined in the ``<stdarg.h>`` header file.
 
-All of these functions operate on arguments that use a target-specific
+All of these functions take as arguments pointers to a target-specific
 value type "``va_list``". The LLVM assembly language reference manual
 does not define what this type is, so all transformations should be
-prepared to handle these functions regardless of the type used.
+prepared to handle these functions regardless of the type used. The intrinsics
+are overloaded, and can be used for pointers to different address spaces.
 
 This example shows how the :ref:`va_arg <i_va_arg>` instruction and the
 variable argument handling intrinsic functions are used.
@@ -12835,24 +12836,24 @@ variable argument handling intrinsic functions are used.
     define i32 @test(i32 %X, ...) {
       ; Initialize variable argument processing
       %ap = alloca %struct.va_list
-      call void @llvm.va_start(ptr %ap)
+      call void @llvm.va_start.p0(ptr %ap)
 
       ; Read a single integer argument
       %tmp = va_arg ptr %ap, i32
 
       ; Demonstrate usage of llvm.va_copy and llvm.va_end
       %aq = alloca ptr
-      call void @llvm.va_copy(ptr %aq, ptr %ap)
-      call void @llvm.va_end(ptr %aq)
+      call void @llvm.va_copy.p0(ptr %aq, ptr %ap)
+      call void @llvm.va_end.p0(ptr %aq)
 
       ; Stop processing of arguments.
-      call void @llvm.va_end(ptr %ap)
+      call void @llvm.va_end.p0(ptr %ap)
       ret i32 %tmp
     }
 
-    declare void @llvm.va_start(ptr)
-    declare void @llvm.va_copy(ptr, ptr)
-    declare void @llvm.va_end(ptr)
+    declare void @llvm.va_start.p0(ptr)
+    declare void @llvm.va_copy.p0(ptr, ptr)
+    declare void @llvm.va_end.p0(ptr)
 
 .. _int_va_start:
 
@@ -12864,7 +12865,8 @@ Syntax:
 
 ::
 
-      declare void @llvm.va_start(ptr <arglist>)
+      declare void @llvm.va_start.p0(ptr <arglist>)
+      declare void @llvm.va_start.p5(ptr addrspace(5) <arglist>)
 
 Overview:
 """""""""
@@ -12896,7 +12898,8 @@ Syntax:
 
 ::
 
-      declare void @llvm.va_end(ptr <arglist>)
+      declare void @llvm.va_end.p0(ptr <arglist>)
+      declare void @llvm.va_end.p5(ptr addrspace(5) <arglist>)
 
 Overview:
 """""""""
@@ -12929,7 +12932,8 @@ Syntax:
 
 ::
 
-      declare void @llvm.va_copy(ptr <destarglist>, ptr <srcarglist>)
+      declare void @llvm.va_copy.p0(ptr <destarglist>, ptr <srcarglist>)
+      declare void @llvm.va_copy.p5(ptr addrspace(5) <destarglist>, ptr addrspace(5) <srcarglist>)
 
 Overview:
 """""""""
@@ -12942,6 +12946,7 @@ Arguments:
 
 The first argument is a pointer to a ``va_list`` element to initialize.
 The second argument is a pointer to a ``va_list`` element to copy from.
+The address spaces of the two arguments must match.
 
 Semantics:
 """"""""""
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index d0ef9c25f39e..764902426f0b 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -700,10 +700,13 @@ class MSBuiltin<string name> {
 //===--------------- Variable Argument Handling Intrinsics ----------------===//
 //
 
-def int_vastart : DefaultAttrsIntrinsic<[], [llvm_ptr_ty], [], "llvm.va_start">;
-def int_vacopy  : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_ptr_ty], [],
-                            "llvm.va_copy">;
-def int_vaend   : DefaultAttrsIntrinsic<[], [llvm_ptr_ty], [], "llvm.va_end">;
+def int_vastart : DefaultAttrsIntrinsic<[],
+                                        [llvm_anyptr_ty], [], "llvm.va_start">;
+def int_vacopy  : DefaultAttrsIntrinsic<[],
+                                        [llvm_anyptr_ty, LLVMMatchType<0>], [],
+                                        "llvm.va_copy">;
+def int_vaend   : DefaultAttrsIntrinsic<[],
+                                        [llvm_anyptr_ty], [], "llvm.va_end">;
 
 //===------------------- Garbage Collection Intrinsics --------------------===//
 //
diff --git a/llvm/test/Bitcode/compatibility-3.6.ll b/llvm/test/Bitcode/compatibility-3.6.ll
index b1f4abf7b8c5..2190e2fbccf2 100644
--- a/llvm/test/Bitcode/compatibility-3.6.ll
+++ b/llvm/test/Bitcode/compatibility-3.6.ll
@@ -1061,16 +1061,16 @@ define void @instructions.va_arg(i8* %v, ...) {
   %ap2 = bitcast i8** %ap to i8*
 
   call void @llvm.va_start(i8* %ap2)
-  ; CHECK: call void @llvm.va_start(ptr %ap2)
+  ; CHECK: call void @llvm.va_start.p0(ptr %ap2)
 
   va_arg i8* %ap2, i32
   ; CHECK: va_arg ptr %ap2, i32
 
   call void @llvm.va_copy(i8* %v, i8* %ap2)
-  ; CHECK: call void @llvm.va_copy(ptr %v, ptr %ap2)
+  ; CHECK: call void @llvm.va_copy.p0(ptr %v, ptr %ap2)
 
   call void @llvm.va_end(i8* %ap2)
-  ; CHECK: call void @llvm.va_end(ptr %ap2)
+  ; CHECK: call void @llvm.va_end.p0(ptr %ap2)
 
   ret void
 }
@@ -1178,11 +1178,11 @@ define void @intrinsics.codegen() {
 ; CHECK: attributes #27 = { uwtable }
 ; CHECK: attributes #28 = { "cpu"="cortex-a8" }
 ; CHECK: attributes #29 = { nocallback nofree nosync nounwind willreturn memory(none) }
-; CHECK: attributes #30 = { nocallback nofree nosync nounwind willreturn }
-; CHECK: attributes #31 = { nounwind memory(argmem: read) }
-; CHECK: attributes #32 = { nounwind memory(argmem: readwrite) }
-; CHECK: attributes #33 = { nocallback nofree nosync nounwind willreturn memory(read) }
-; CHECK: attributes #34 = { nocallback nounwind }
+; CHECK: attributes #30 = { nounwind memory(argmem: read) }
+; CHECK: attributes #31 = { nounwind memory(argmem: readwrite) }
+; CHECK: attributes #32 = { nocallback nofree nosync nounwind willreturn memory(read) }
+; CHECK: attributes #33 = { nocallback nounwind }
+; CHECK: attributes #34 = { nocallback nofree nosync nounwind willreturn }
 ; CHECK: attributes #35 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite) }
 ; CHECK: attributes #36 = { builtin }
 
diff --git a/llvm/test/Bitcode/compatibility-3.7.ll b/llvm/test/Bitcode/compatibility-3.7.ll
index 91e55f6eda59..7e59b5c1be6e 100644
--- a/llvm/test/Bitcode/compatibility-3.7.ll
+++ b/llvm/test/Bitcode/compatibility-3.7.ll
@@ -1092,16 +1092,16 @@ define void @instructions.va_arg(i8* %v, ...) {
   %ap2 = bitcast i8** %ap to i8*
 
   call void @llvm.va_start(i8* %ap2)
-  ; CHECK: call void @llvm.va_start(ptr %ap2)
+  ; CHECK: call void @llvm.va_start.p0(ptr %ap2)
 
   va_arg i8* %ap2, i32
   ; CHECK: va_arg ptr %ap2, i32
 
   call void @llvm.va_copy(i8* %v, i8* %ap2)
-  ; CHECK: call void @llvm.va_copy(ptr %v, ptr %ap2)
+  ; CHECK: call void @llvm.va_copy.p0(ptr %v, ptr %ap2)
 
   call void @llvm.va_end(i8* %ap2)
-  ; CHECK: call void @llvm.va_end(ptr %ap2)
+  ; CHECK: call void @llvm.va_end.p0(ptr %ap2)
 
   ret void
 }
@@ -1241,11 +1241,11 @@ define void @misc.metadata() {
 ; CHECK: attributes #30 = { uwtable }
 ; CHECK: attributes #31 = { "cpu"="cortex-a8" }
 ; CHECK: attributes #32 = { nocallback nofree nosync nounwind willreturn memory(none) }
-; CHECK: attributes #33 = { nocallback nofree nosync nounwind willreturn }
-; CHECK: attributes #34 = { nounwind memory(argmem: read) }
-; CHECK: attributes #35 = { nounwind memory(argmem: readwrite) }
-; CHECK: attributes #36 = { nocallback nofree nosync nounwind willreturn memory(read) }
-; CHECK: attributes #37 = { nocallback nounwind }
+; CHECK: attributes #33 = { nounwind memory(argmem: read) }
+; CHECK: attributes #34 = { nounwind memory(argmem: readwrite) }
+; CHECK: attributes #35 = { nocallback nofree nosync nounwind willreturn memory(read) }
+; CHECK: attributes #36 = { nocallback nounwind }
+; CHECK: attributes #37 = { nocallback nofree nosync nounwind willreturn }
 ; CHECK: attributes #38 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite) }
 ; CHECK: attributes #39 = { builtin }
 
diff --git a/llvm/test/Bitcode/compatibility-3.8.ll b/llvm/test/Bitcode/compatibility-3.8.ll
index aa4d8b14968c..ebd1f2fff8c9 100644
--- a/llvm/test/Bitcode/compatibility-3.8.ll
+++ b/llvm/test/Bitcode/compatibility-3.8.ll
@@ -1247,16 +1247,16 @@ define void @instructions.va_arg(i8* %v, ...) {
   %ap2 = bitcast i8** %ap to i8*
 
   call void @llvm.va_start(i8* %ap2)
-  ; CHECK: call void @llvm.va_start(ptr %ap2)
+  ; CHECK: call void @llvm.va_start.p0(ptr %ap2)
 
   va_arg i8* %ap2, i32
   ; CHECK: va_arg ptr %ap2, i32
 
   call void @llvm.va_copy(i8* %v, i8* %ap2)
-  ; CHECK: call void @llvm.va_copy(ptr %v, ptr %ap2)
+  ; CHECK: call void @llvm.va_copy.p0(ptr %v, ptr %ap2)
 
   call void @llvm.va_end(i8* %ap2)
-  ; CHECK: call void @llvm.va_end(ptr %ap2)
+  ; CHECK: call void @llvm.va_end.p0(ptr %ap2)
 
   ret void
 }
@@ -1551,11 +1551,11 @@ normal:
 ; CHECK: attributes #33 = { memory(inaccessiblemem: readwrite) }
 ; CHECK: attributes #34 = { memory(argmem: readwrite, inaccessiblemem: readwrite) }
 ; CHECK: attributes #35 = { nocallback nofree nosync nounwind willreturn memory(none) }
-; CHECK: attributes #36 = { nocallback nofree nosync nounwind willreturn }
-; CHECK: attributes #37 = { nounwind memory(argmem: read) }
-; CHECK: attributes #38 = { nounwind memory(argmem: readwrite) }
-; CHECK: attributes #39 = { nocallback nofree nosync nounwind willreturn memory(read) }
-; CHECK: attributes #40 = { nocallback nounwind }
+; CHECK: attributes #36 = { nounwind memory(argmem: read) }
+; CHECK: attributes #37 = { nounwind memory(argmem: readwrite) }
+; CHECK: attributes #38 = { nocallback nofree nosync nounwind willreturn memory(read) }
+; CHECK: attributes #39 = { nocallback nounwind }
+; CHECK: attributes #40 = { nocallback nofree nosync nounwind willreturn }
 ; CHECK: attributes #41 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite) }
 ; CHECK: attributes #42 = { builtin }
 
diff --git a/llvm/test/Bitcode/compatibility-3.9.ll b/llvm/test/Bitcode/compatibility-3.9.ll
index e3c84f6e6007..c34f04ceb0de 100644
--- a/llvm/test/Bitcode/compatibility-3.9.ll
+++ b/llvm/test/Bitcode/compatibility-3.9.ll
@@ -1318,16 +1318,16 @@ define void @instructions.va_arg(i8* %v, ...) {
   %ap2 = bitcast i8** %ap to i8*
 
   call void @llvm.va_start(i8* %ap2)
-  ; CHECK: call void @llvm.va_start(ptr %ap2)
+  ; CHECK: call void @llvm.va_start.p0(ptr %ap2)
 
   va_arg i8* %ap2, i32
   ; CHECK: va_arg ptr %ap2, i32
 
   call void @llvm.va_copy(i8* %v, i8* %ap2)
-  ; CHECK: call void @llvm.va_copy(ptr %v, ptr %ap2)
+  ; CHECK: call void @llvm.va_copy.p0(ptr %v, ptr %ap2)
 
   call void @llvm.va_end(i8* %ap2)
-  ; CHECK: call void @llvm.va_end(ptr %ap2)
+  ; CHECK: call void @llvm.va_end.p0(ptr %ap2)
 
   ret void
 }
@@ -1624,11 +1624,11 @@ declare void @f.writeonly() writeonly
 ; CHECK: attributes #33 = { memory(inaccessiblemem: readwrite) }
 ; CHECK: attributes #34 = { memory(argmem: readwrite, inaccessiblemem: readwrite) }
 ; CHECK: attributes #35 = { nocallback nofree nosync nounwind willreturn memory(none) }
-; CHECK: attributes #36 = { nocallback nofree nosync nounwind willreturn }
-; CHECK: attributes #37 = { nounwind memory(argmem: read) }
-; CHECK: attributes #38 = { nounwind memory(argmem: readwrite) }
-; CHECK: attributes #39 = { nocallback nofree nosync nounwind willreturn memory(read) }
-; CHECK: attributes #40 = { nocallback nounwind }
+; CHECK: attributes #36 = { nounwind memory(argmem: read) }
+; CHECK: attributes #37 = { nounwind memory(argmem: readwrite) }
+; CHECK: attributes #38 = { nocallback nofree nosync nounwind willreturn memory(read) }
+; CHECK: attributes #39 = { nocallback nounwind }
+; CHECK: attributes #40 = { nocallback nofree nosync nounwind willreturn }
 ; CHECK: attributes #41 = { memory(write) }
 ; CHECK: attributes #42 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite) }
 ; CHECK: attributes #43 = { builtin }
diff --git a/llvm/test/Bitcode/compatibility-4.0.ll b/llvm/test/Bitcode/compatibility-4.0.ll
index 06cb842059a4..05bffda1d117 100644
--- a/llvm/test/Bitcode/compatibility-4.0.ll
+++ b/llvm/test/Bitcode/compatibility-4.0.ll
@@ -1318,16 +1318,16 @@ define void @instructions.va_arg(i8* %v, ...) {
   %ap2 = bitcast i8** %ap to i8*
 
   call void @llvm.va_start(i8* %ap2)
-  ; CHECK: call void @llvm.va_start(ptr %ap2)
+  ; CHECK: call void @llvm.va_start.p0(ptr %ap2)
 
   va_arg i8* %ap2, i32
   ; CHECK: va_arg ptr %ap2, i32
 
   call void @llvm.va_copy(i8* %v, i8* %ap2)
-  ; CHECK: call void @llvm.va_copy(ptr %v, ptr %ap2)
+  ; CHECK: call void @llvm.va_copy.p0(ptr %v, ptr %ap2)
 
   call void @llvm.va_end(i8* %ap2)
-  ; CHECK: call void @llvm.va_end(ptr %ap2)
+  ; CHECK: call void @llvm.va_end.p0(ptr %ap2)
 
   ret void
 }
@@ -1649,11 +1649,11 @@ define i8** @constexpr() {
 ; CHECK: attributes #33 = { memory(inaccessiblemem: readwrite) }
 ; CHECK: attributes #34 = { memory(argmem: readwrite, inaccessiblemem: readwrite) }
 ; CHECK: attributes #35 = { nocallback nofree nosync nounwind willreturn memory(none) }
-; CHECK: attributes #36 = { nocallback nofree nosync nounwind willreturn }
-; CHECK: attributes #37 = { nounwind memory(argmem: read) }
-; CHECK: attributes #38 = { nounwind memory(argmem: readwrite) }
-; CHECK: attributes #39 = { nocallback nofree nosync nounwind willreturn memory(read) }
-; CHECK: attributes #40 = { nocallback nounwind }
+; CHECK: attributes #36 = { nounwind memory(argmem: read) }
+; CHECK: attributes #37 = { nounwind memory(argmem: readwrite) }
+; CHECK: attributes #38 = { nocallback nofree nosync nounwind willreturn memory(read) }
+; CHECK: attributes #39 = { nocallback nounwind }
+; CHECK: attributes #40 = { nocallback nofree nosync nounwind willreturn }
 ; CHECK: attributes #41 = { memory(write) }
 ; CHECK: attributes #42 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite) }
 ; CHECK: attributes #43 = { builtin }
diff --git a/llvm/test/Bitcode/compatibility-5.0.ll b/llvm/test/Bitcode/compatibility-5.0.ll
index f9ae558917cd..0c872289c62b 100644
--- a/llvm/test/Bitcode/compatibility-5.0.ll
+++ b/llvm/test/Bitcode/compatibility-5.0.ll
@@ -1330,16 +1330,16 @@ define void @instructions.va_arg(i8* %v, ...) {
   %ap2 = bitcast i8** %ap to i8*
 
   call void @llvm.va_start(i8* %ap2)
-  ; CHECK: call void @llvm.va_start(ptr %ap2)
+  ; CHECK: call void @llvm.va_start.p0(ptr %ap2)
 
   va_arg i8* %ap2, i32
   ; CHECK: va_arg ptr %ap2, i32
 
   call void @llvm.va_copy(i8* %v, i8* %ap2)
-  ; CHECK: call void @llvm.va_copy(ptr %v, ptr %ap2)
+  ; CHECK: call void @llvm.va_copy.p0(ptr %v, ptr %ap2)
 
   call void @llvm.va_end(i8* %ap2)
-  ; CHECK: call void @llvm.va_end(ptr %ap2)
+  ; CHECK: call void @llvm.va_end.p0(ptr %ap2)
 
   ret void
 }
@@ -1664,11 +1664,11 @@ define i8** @constexpr() {
 ; CHECK: attributes #33 = { memory(inaccessiblemem: readwrite) }
 ; CHECK: attributes #34 = { memory(argmem: readwrite, inaccessiblemem: readwrite) }
 ; CHECK: attributes #35 = { nocallback nofree nosync nounwind willreturn memory(none) }
-; CHECK: attributes #36 = { nocallback nofree nosync nounwind willreturn }
-; CHECK: attributes #37 = { nounwind memory(argmem: read) }
-; CHECK: attributes #38 = { nounwind memory(argmem: readwrite) }
-; CHECK: attributes #39 = { nocallback nofree nosync nounwind willreturn memory(read) }
-; CHECK: attributes #40 = { nocallback nounwind }
+; CHECK: attributes #36 = { nounwind memory(argmem: read) }
+; CHECK: attributes #37 = { nounwind memory(argmem: readwrite) }
+; CHECK: attributes #38 = { nocallback nofree nosync nounwind willreturn memory(read) }
+; CHECK: attributes #39 = { nocallback nounwind }
+; CHECK: attributes #40 = { nocallback nofree nosync nounwind willreturn }
 ; CHECK: attributes #41 = { memory(write) }
 ; CHECK: attributes #42 = { speculatable }
 ; CHECK: attributes #43 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite) }
diff --git a/llvm/test/Bitcode/compatibility-6.0.ll b/llvm/test/Bitcode/compatibility-6.0.ll
index 1458e1b639ad..44c680885be3 100644
--- a/llvm/test/Bitcode/compatibility-6.0.ll
+++ b/llvm/test/Bitcode/compatibility-6.0.ll
@@ -1340,16 +1340,16 @@ define void @instructions.va_arg(i8* %v, ...) {
   %ap2 = bitcast i8** %ap to i8*
 
   call void @llvm.va_start(i8* %ap2)
-  ; CHECK: call void @llvm.va_start(ptr %ap2)
+  ; CHECK: call void @llvm.va_start.p0(ptr %ap2)
 
   va_arg i8* %ap2, i32
   ; CHECK: va_arg ptr %ap2, i32
 
   call void @llvm.va_copy(i8* %v, i8* %ap2)
-  ; CHECK: call void @llvm.va_copy(ptr %v, ptr %ap2)
+  ; CHECK: call void @llvm.va_copy.p0(ptr %v, ptr %ap2)
 
   call void @llvm.va_end(i8* %ap2)
-  ; CHECK: call void @llvm.va_end(ptr %ap2)
+  ; CHECK: call void @llvm.va_end.p0(ptr %ap2)
 
   ret void
 }
@@ -1674,11 +1674,11 @@ define i8** @constexpr() {
 ; CHECK: attributes #33 = { memory(inaccessiblemem: readwrite) }
 ; CHECK: attributes #34 = { memory(argmem: readwrite, inaccessiblemem: readwrite) }
 ; CHECK: attributes #35 = { nocallback nofree nosync nounwind willreturn memory(none) }
-; CHECK: attributes #36 = { nocallback nofree nosync nounwind willreturn }
-; CHECK: attributes #37 = { nounwind memory(argmem: read) }
-; CHECK: attributes #38 = { nounwind memory(argmem: readwrite) }
-; CHECK: attributes #39 = { nocallback nofree nosync nounwind willreturn memory(read) }
-; CHECK: attributes #40 = { nocallback nounwind }
+; CHECK: attributes #36 = { nounwind memory(argmem: read) }
+; CHECK: attributes #37 = { nounwind memory(argmem: readwrite) }
+; CHECK: attributes #38 = { nocallback nofree nosync nounwind willreturn memory(read) }
+; CHECK: attributes #39 = { nocallback nounwind }
+; CHECK: attributes #40 = { nocallback nofree nosync nounwind willreturn }
 ; CHECK: attributes #41 = { memory(write) }
 ; CHECK: attributes #42 = { speculatable }
 ; CHECK: attributes #43 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite) }
diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll
index fa8b0520567a..b374924516d6 100644
--- a/llvm/test/Bitcode/compatibility.ll
+++ b/llvm/test/Bitcode/compatibility.ll
@@ -1648,16 +1648,16 @@ define void @instructions.va_arg(ptr %v, ...) {
   %ap = alloca ptr
 
   call void @llvm.va_start(ptr %ap)
-  ; CHECK: call void @llvm.va_start(ptr %ap)
+  ; CHECK: call void @llvm.va_start.p0(ptr %ap)
 
   va_arg ptr %ap, i32
   ; CHECK: va_arg ptr %ap, i32
 
   call void @llvm.va_copy(ptr %v, ptr %ap)
-  ; CHECK: call void @llvm.va_copy(ptr %v, ptr %ap)
+  ; CHECK: call void @llvm.va_copy.p0(ptr %v, ptr %ap)
 
   call void @llvm.va_end(ptr %ap)
-  ; CHECK: call void @llvm.va_end(ptr %ap)
+  ; CHECK: call void @llvm.va_end.p0(ptr %ap)
 
   ret void
 }
@@ -2091,12 +2091,12 @@ define float @nofpclass_callsites(float %arg) {
 ; CHECK: attributes #33 = { memory(inaccessiblemem: readwrite) }
 ; CHECK: attributes #34 = { memory(argmem: readwrite, inaccessiblemem: readwrite) }
 ; CHECK: attributes #35 = { nocallback nofree nosync nounwind willreturn memory(none) }
-; CHECK: attributes #36 = { nocallback nofree nosync nounwind willreturn }
-; CHECK: attributes #37 = { nounwind memory(argmem: read) }
-; CHECK: attributes #38 = { nounwind memory(argmem: readwrite) }
-; CHECK: attributes #39 = { nocallback nofree nosync nounwind willreturn memory(read) }
-; CHECK: attributes #40 = { nocallback nounwind }
-; CHECK: attributes #41 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite) }
+; CHECK: attributes #36 = { nounwind memory(argmem: read) }
+; CHECK: attributes #37 = { nounwind memory(argmem: readwrite) }
+; CHECK: attributes #38 = { nocallback nofree nosync nounwind willreturn memory(read) }
+; CHECK: attributes #39 = { nocallback nounwind }
+; CHECK: attributes #40 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite) }
+; CHECK: attributes #41 = { nocallback nofree nosync nounwind willreturn }
 ; CHECK: attributes #42 = { memory(write) }
 ; CHECK: attributes #43 = { speculatable }
 ; CHECK: attributes #44 = { strictfp }
diff --git a/llvm/test/Bitcode/thinlto-function-summary.ll b/llvm/test/Bitcode/thinlto-function-summary.ll
index 799759ebcac1..13c6611843d6 100644
--- a/llvm/test/Bitcode/thinlto-function-summary.ll
+++ b/llvm/test/Bitcode/thinlto-function-summary.ll
@@ -13,9 +13,9 @@
 ; "variadic"
 ; BC-NEXT: <FUNCTION op0=46 op1=8
 ; "llvm.va_start"
-; BC-NEXT: <FUNCTION op0=54 op1=13
+; BC-NEXT: <FUNCTION op0=54 op1=16
 ; "f"
-; BC-NEXT: <ALIAS op0=67 op1=1
+; BC-NEXT: <ALIAS op0=70 op1=1
 ; BC: <GLOBALVAL_SUMMARY_BLOCK
 ; BC-NEXT: <VERSION
 ; BC-NEXT: <FLAGS
@@ -26,7 +26,7 @@
 ; BC-NEXT: <ALIAS {{.*}} op0=6 op1=0 op2=3
 ; BC-NEXT: </GLOBALVAL_SUMMARY_BLOCK
 ; BC: <STRTAB_BLOCK
-; BC-NEXT: blob data = 'hfoobaranon.{{................................}}.0variadicllvm.va_startf{{.*}}'
+; BC-NEXT: blob data = 'hfoobaranon.{{................................}}.0variadicllvm.va_start.p{{[0-9]+}}f{{.*}}'
 
 
 ; RUN: opt -passes=name-anon-globals -module-summary < %s | llvm-dis | FileCheck %s
diff --git a/llvm/test/Bitcode/variableArgumentIntrinsic.3.2.ll b/llvm/test/Bitcode/variableArgumentIntrinsic.3.2.ll
index fad7b8ea6a58..fd3f500de791 100644
--- a/llvm/test/Bitcode/variableArgumentIntrinsic.3.2.ll
+++ b/llvm/test/Bitcode/variableArgumentIntrinsic.3.2.ll
@@ -10,7 +10,7 @@ define i32 @varArgIntrinsic(i32 %X, ...) {
   %ap = alloca i8*
   %ap2 = bitcast i8** %ap to i8*
 
-; CHECK: call void @llvm.va_start(ptr %ap2)
+; CHECK: call void @llvm.va_start.p0(ptr %ap2)
   call void @llvm.va_start(i8* %ap2)
 
 ; CHECK-NEXT: %tmp = va_arg ptr %ap, i32
@@ -19,12 +19,12 @@ define i32 @varArgIntrinsic(i32 %X, ...) {
   %aq = alloca i8*
   %aq2 = bitcast i8** %aq to i8*
 
-; CHECK: call void @llvm.va_copy(ptr %aq2, ptr %ap2)
+; CHECK: call void @llvm.va_copy.p0(ptr %aq2, ptr %ap2)
   call void @llvm.va_copy(i8* %aq2, i8* %ap2)
-; CHECK-NEXT: call void @llvm.va_end(ptr %aq2)
+; CHECK-NEXT: call void @llvm.va_end.p0(ptr %aq2)
   call void @llvm.va_end(i8* %aq2)
 
-; CHECK-NEXT:  call void @llvm.va_end(ptr %ap2)
+; CHECK-NEXT:  call void @llvm.va_end.p0(ptr %ap2)
   call void @llvm.va_end(i8* %ap2)
   ret i32 %tmp
 }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/vararg_shadow.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/vararg_shadow.ll
index 96ac4b6088c3..9133b329deb2 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/AArch64/vararg_shadow.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/vararg_shadow.ll
@@ -758,7 +758,7 @@ define linkonce_odr dso_local void @_Z5test2IcEvT_iz(i8 noundef %t, i32 noundef
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 193514046488576
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 32, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -808,7 +808,7 @@ define linkonce_odr dso_local void @_Z5test2IcEvT_iz(i8 noundef %t, i32 noundef
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP52]], ptr align 16 [[TMP53]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -851,7 +851,7 @@ define linkonce_odr dso_local void @_Z5test2IiEvT_iz(i32 noundef %t, i32 noundef
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 193514046488576
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 32, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -901,7 +901,7 @@ define linkonce_odr dso_local void @_Z5test2IiEvT_iz(i32 noundef %t, i32 noundef
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP52]], ptr align 16 [[TMP53]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -936,7 +936,7 @@ define linkonce_odr dso_local void @_Z5test2IfEvT_iz(float noundef %t, i32 nound
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 193514046488576
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 32, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -986,7 +986,7 @@ define linkonce_odr dso_local void @_Z5test2IfEvT_iz(float noundef %t, i32 nound
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP52]], ptr align 16 [[TMP53]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -1021,7 +1021,7 @@ define linkonce_odr dso_local void @_Z5test2IdEvT_iz(double noundef %t, i32 noun
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 193514046488576
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 32, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -1071,7 +1071,7 @@ define linkonce_odr dso_local void @_Z5test2IdEvT_iz(double noundef %t, i32 noun
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP52]], ptr align 16 [[TMP53]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -1106,7 +1106,7 @@ define linkonce_odr dso_local void @_Z5test2IeEvT_iz(fp128 noundef %t, i32 nound
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 193514046488576
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 32, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -1156,7 +1156,7 @@ define linkonce_odr dso_local void @_Z5test2IeEvT_iz(fp128 noundef %t, i32 nound
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP52]], ptr align 16 [[TMP53]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -1191,7 +1191,7 @@ define linkonce_odr dso_local void @_Z5test2I6IntIntEvT_iz(i64 %t.coerce, i32 no
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 193514046488576
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 32, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -1241,7 +1241,7 @@ define linkonce_odr dso_local void @_Z5test2I6IntIntEvT_iz(i64 %t.coerce, i32 no
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP52]], ptr align 16 [[TMP53]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -1276,7 +1276,7 @@ define linkonce_odr dso_local void @_Z5test2I10Int64Int64EvT_iz([2 x i64] %t.coe
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 193514046488576
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 32, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -1326,7 +1326,7 @@ define linkonce_odr dso_local void @_Z5test2I10Int64Int64EvT_iz([2 x i64] %t.coe
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP52]], ptr align 16 [[TMP53]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -1361,7 +1361,7 @@ define linkonce_odr dso_local void @_Z5test2I12DoubleDoubleEvT_iz([2 x double] a
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 193514046488576
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 32, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -1411,7 +1411,7 @@ define linkonce_odr dso_local void @_Z5test2I12DoubleDoubleEvT_iz([2 x double] a
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP52]], ptr align 16 [[TMP53]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -1446,7 +1446,7 @@ define linkonce_odr dso_local void @_Z5test2I7Double4EvT_iz([4 x double] alignst
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 193514046488576
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 32, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -1496,7 +1496,7 @@ define linkonce_odr dso_local void @_Z5test2I7Double4EvT_iz([4 x double] alignst
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP52]], ptr align 16 [[TMP53]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -1531,7 +1531,7 @@ define linkonce_odr dso_local void @_Z5test2I11DoubleFloatEvT_iz([2 x i64] %t.co
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 193514046488576
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 32, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -1581,7 +1581,7 @@ define linkonce_odr dso_local void @_Z5test2I11DoubleFloatEvT_iz([2 x i64] %t.co
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP52]], ptr align 16 [[TMP53]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -1616,7 +1616,7 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble2EvT_iz([2 x fp128] ali
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 193514046488576
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 32, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -1666,7 +1666,7 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble2EvT_iz([2 x fp128] ali
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP52]], ptr align 16 [[TMP53]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -1701,7 +1701,7 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble4EvT_iz([4 x fp128] ali
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 193514046488576
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 32, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -1751,7 +1751,7 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble4EvT_iz([4 x fp128] ali
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP52]], ptr align 16 [[TMP53]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg-kernel.ll b/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg-kernel.ll
index 1535fccfc211..e0b5907719af 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg-kernel.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg-kernel.ll
@@ -39,7 +39,7 @@ define i64 @foo(i64 %guard, ...) #1 {
 ; Only 56 bytes of the register save area is copied, because of
 ; "use-soft-float".
 
-; CHECK: call void @llvm.va_start(ptr %vl)
+; CHECK: call void @llvm.va_start.p0(ptr %vl)
 ; CHECK: [[VlAddr:%.*]] = ptrtoint ptr %vl to i64
 ; CHECK: [[RegSaveAreaAddrAddr:%.*]] = add i64 [[VlAddr]], 24
 ; CHECK: [[RegSaveAreaAddr:%.*]] = inttoptr i64 [[RegSaveAreaAddrAddr]] to ptr
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/vararg_shadow.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/vararg_shadow.ll
index aff4d2c55ad6..205101564dfe 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/vararg_shadow.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/vararg_shadow.ll
@@ -560,7 +560,7 @@ define linkonce_odr dso_local void @_Z5test2IcEvT_iz(i8 noundef signext %t, i32
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -580,7 +580,7 @@ define linkonce_odr dso_local void @_Z5test2IcEvT_iz(i8 noundef signext %t, i32
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -623,7 +623,7 @@ define linkonce_odr dso_local void @_Z5test2IiEvT_iz(i32 noundef %t, i32 noundef
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -643,7 +643,7 @@ define linkonce_odr dso_local void @_Z5test2IiEvT_iz(i32 noundef %t, i32 noundef
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -678,7 +678,7 @@ define linkonce_odr dso_local void @_Z5test2IfEvT_iz(float noundef %t, i32 nound
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -698,7 +698,7 @@ define linkonce_odr dso_local void @_Z5test2IfEvT_iz(float noundef %t, i32 nound
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -733,7 +733,7 @@ define linkonce_odr dso_local void @_Z5test2IdEvT_iz(double noundef %t, i32 noun
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -753,7 +753,7 @@ define linkonce_odr dso_local void @_Z5test2IdEvT_iz(double noundef %t, i32 noun
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -788,7 +788,7 @@ define linkonce_odr dso_local void @_Z5test2IeEvT_iz(x86_fp80 noundef %t, i32 no
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -808,7 +808,7 @@ define linkonce_odr dso_local void @_Z5test2IeEvT_iz(x86_fp80 noundef %t, i32 no
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -843,7 +843,7 @@ define linkonce_odr dso_local void @_Z5test2I6IntIntEvT_iz(i64 %t.coerce, i32 no
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -863,7 +863,7 @@ define linkonce_odr dso_local void @_Z5test2I6IntIntEvT_iz(i64 %t.coerce, i32 no
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -898,7 +898,7 @@ define linkonce_odr dso_local void @_Z5test2I10Int64Int64EvT_iz(i64 %t.coerce0,
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -918,7 +918,7 @@ define linkonce_odr dso_local void @_Z5test2I10Int64Int64EvT_iz(i64 %t.coerce0,
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -953,7 +953,7 @@ define linkonce_odr dso_local void @_Z5test2I12DoubleDoubleEvT_iz(double %t.coer
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -973,7 +973,7 @@ define linkonce_odr dso_local void @_Z5test2I12DoubleDoubleEvT_iz(double %t.coer
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -1008,7 +1008,7 @@ define linkonce_odr dso_local void @_Z5test2I7Double4EvT_iz(ptr noundef byval(%s
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -1028,7 +1028,7 @@ define linkonce_odr dso_local void @_Z5test2I7Double4EvT_iz(ptr noundef byval(%s
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -1063,7 +1063,7 @@ define linkonce_odr dso_local void @_Z5test2I11DoubleFloatEvT_iz(double %t.coerc
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -1083,7 +1083,7 @@ define linkonce_odr dso_local void @_Z5test2I11DoubleFloatEvT_iz(double %t.coerc
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -1118,7 +1118,7 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble2EvT_iz(ptr noundef byv
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -1138,7 +1138,7 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble2EvT_iz(ptr noundef byv
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
@@ -1173,7 +1173,7 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble4EvT_iz(ptr noundef byv
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 24, i1 false)
-; CHECK-NEXT:    call void @llvm.va_start(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 16
 ; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -1193,7 +1193,7 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble4EvT_iz(ptr noundef byv
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[TMP23]], ptr align 16 [[TMP24]], i64 [[TMP0]], i1 false)
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Instrumentation/MemorySanitizer/msan_debug_info.ll b/llvm/test/Instrumentation/MemorySanitizer/msan_debug_info.ll
index 21f3311a57ef..f07f3ad06e60 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/msan_debug_info.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/msan_debug_info.ll
@@ -542,7 +542,7 @@ define void @VAStart(i32 %x, ...) sanitize_memory {
 ; CHECK-NEXT:    [[TMP29:%.*]] = add i64 [[TMP27]], 17592186044416, !dbg [[DBG11]]
 ; CHECK-NEXT:    [[TMP30:%.*]] = inttoptr i64 [[TMP29]] to ptr, !dbg [[DBG11]]
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP28]], i8 0, i64 24, i1 false), !dbg [[DBG11]]
-; CHECK-NEXT:    call void @llvm.va_start(ptr [[VA]]), !dbg [[DBG11]]
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[VA]]), !dbg [[DBG11]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = ptrtoint ptr [[VA]] to i64, !dbg [[DBG11]]
 ; CHECK-NEXT:    [[TMP32:%.*]] = add i64 [[TMP31]], 16, !dbg [[DBG11]]
 ; CHECK-NEXT:    [[TMP33:%.*]] = inttoptr i64 [[TMP32]] to ptr, !dbg [[DBG11]]
diff --git a/llvm/test/Transforms/GlobalOpt/inalloca-varargs.ll b/llvm/test/Transforms/GlobalOpt/inalloca-varargs.ll
index 188210782edd..4c5a448d12c4 100644
--- a/llvm/test/Transforms/GlobalOpt/inalloca-varargs.ll
+++ b/llvm/test/Transforms/GlobalOpt/inalloca-varargs.ll
@@ -23,7 +23,7 @@ define internal i32 @i(ptr inalloca(ptr) %a, ...) {
 ; CHECK-LABEL: define {{[^@]+}}@i
 ; CHECK-SAME: (ptr inalloca(ptr) [[A:%.*]], ...) unnamed_addr {
 ; CHECK-NEXT:    [[AP:%.*]] = alloca ptr, align 4
-; CHECK-NEXT:    call void @llvm.va_start(ptr [[AP]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[AP]])
 ; CHECK-NEXT:    [[ARGP_CUR:%.*]] = load ptr, ptr [[AP]], align 4
 ; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[ARGP_CUR]], align 4
 ; CHECK-NEXT:    ret i32 [[L]]
diff --git a/llvm/test/Transforms/IROutliner/illegal-vaarg.ll b/llvm/test/Transforms/IROutliner/illegal-vaarg.ll
index ef365d6eaddb..38dfd25e039e 100644
--- a/llvm/test/Transforms/IROutliner/illegal-vaarg.ll
+++ b/llvm/test/Transforms/IROutliner/illegal-vaarg.ll
@@ -17,10 +17,10 @@ define i32 @func1(i32 %a, double %b, ptr %v, ...) nounwind {
 ; CHECK-NEXT:    [[AP:%.*]] = alloca ptr, align 4
 ; CHECK-NEXT:    [[C:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    call void @outlined_ir_func_0(i32 [[A:%.*]], ptr [[A_ADDR]], double [[B:%.*]], ptr [[B_ADDR]])
-; CHECK-NEXT:    call void @llvm.va_start(ptr [[AP]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[AP]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = va_arg ptr [[AP]], i32
-; CHECK-NEXT:    call void @llvm.va_copy(ptr [[V:%.*]], ptr [[AP]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr [[AP]])
+; CHECK-NEXT:    call void @llvm.va_copy.p0(ptr [[V:%.*]], ptr [[AP]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[AP]])
 ; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[TMP_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_1(i32 [[TMP0]], ptr [[C]], ptr [[TMP_LOC]])
 ; CHECK-NEXT:    [[TMP_RELOAD:%.*]] = load i32, ptr [[TMP_LOC]], align 4
@@ -52,10 +52,10 @@ define i32 @func2(i32 %a, double %b, ptr %v, ...) nounwind {
 ; CHECK-NEXT:    [[AP:%.*]] = alloca ptr, align 4
 ; CHECK-NEXT:    [[C:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    call void @outlined_ir_func_0(i32 [[A:%.*]], ptr [[A_ADDR]], double [[B:%.*]], ptr [[B_ADDR]])
-; CHECK-NEXT:    call void @llvm.va_start(ptr [[AP]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[AP]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = va_arg ptr [[AP]], i32
-; CHECK-NEXT:    call void @llvm.va_copy(ptr [[V:%.*]], ptr [[AP]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr [[AP]])
+; CHECK-NEXT:    call void @llvm.va_copy.p0(ptr [[V:%.*]], ptr [[AP]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[AP]])
 ; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[TMP_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_1(i32 [[TMP0]], ptr [[C]], ptr [[TMP_LOC]])
 ; CHECK-NEXT:    [[TMP_RELOAD:%.*]] = load i32, ptr [[TMP_LOC]], align 4
diff --git a/llvm/test/Transforms/IROutliner/outline-vaarg-intrinsic.ll b/llvm/test/Transforms/IROutliner/outline-vaarg-intrinsic.ll
index 9f565de96057..2d526086fae4 100644
--- a/llvm/test/Transforms/IROutliner/outline-vaarg-intrinsic.ll
+++ b/llvm/test/Transforms/IROutliner/outline-vaarg-intrinsic.ll
@@ -51,7 +51,7 @@ entry:
 ; CHECK-NEXT:    [[C:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 ; CHECK-NEXT:    store double [[B]], ptr [[B_ADDR]], align 8
-; CHECK-NEXT:    call void @llvm.va_start(ptr [[AP]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[AP]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = va_arg ptr [[AP]], i32
 ; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[TMP_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[V]], ptr [[AP]], i32 [[TMP0]], ptr [[C]], ptr [[TMP_LOC]])
@@ -70,7 +70,7 @@ entry:
 ; CHECK-NEXT:    [[C:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 ; CHECK-NEXT:    store double [[B]], ptr [[B_ADDR]], align 8
-; CHECK-NEXT:    call void @llvm.va_start(ptr [[AP]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[AP]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = va_arg ptr [[AP]], i32
 ; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[TMP_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[V]], ptr [[AP]], i32 [[TMP0]], ptr [[C]], ptr [[TMP_LOC]])
@@ -84,8 +84,8 @@ entry:
 ; CHECK-NEXT:  newFuncRoot:
 ; CHECK-NEXT:    br label [[ENTRY_TO_OUTLINE:%.*]]
 ; CHECK:       entry_to_outline:
-; CHECK-NEXT:    call void @llvm.va_copy(ptr [[TMP0]], ptr [[TMP1]])
-; CHECK-NEXT:    call void @llvm.va_end(ptr [[TMP1]])
+; CHECK-NEXT:    call void @llvm.va_copy.p0(ptr [[TMP0]], ptr [[TMP1]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[TMP1]])
 ; CHECK-NEXT:    store i32 [[TMP2]], ptr [[TMP3]], align 4
 ; CHECK-NEXT:    [[TMP:%.*]] = load i32, ptr [[TMP3]], align 4
 ; CHECK-NEXT:    br label [[ENTRY_AFTER_OUTLINE_EXITSTUB:%.*]]
diff --git a/llvm/test/Transforms/NewGVN/pr31483.ll b/llvm/test/Transforms/NewGVN/pr31483.ll
index 0e7461c2612b..82e9a2ab286e 100644
--- a/llvm/test/Transforms/NewGVN/pr31483.ll
+++ b/llvm/test/Transforms/NewGVN/pr31483.ll
@@ -41,7 +41,7 @@ define signext i32 @ham(ptr %arg, ptr %arg1) #0 {
 ; CHECK:       bb22:
 ; CHECK-NEXT:    br label [[BB2]]
 ; CHECK:       bb23:
-; CHECK-NEXT:    call void @llvm.va_end(ptr [[TMP]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[TMP]])
 ; CHECK-NEXT:    ret i32 undef
 ;
 bb:
diff --git a/llvm/test/Transforms/Reassociate/vaarg_movable.ll b/llvm/test/Transforms/Reassociate/vaarg_movable.ll
index 337877a54a90..4e45b219fccd 100644
--- a/llvm/test/Transforms/Reassociate/vaarg_movable.ll
+++ b/llvm/test/Transforms/Reassociate/vaarg_movable.ll
@@ -10,13 +10,13 @@ define i32 @func(i32 %dummy, ...) {
 ;
 ; CHECK-LABEL: @func(
 ; CHECK-NEXT:    [[VARARGS:%.*]] = alloca ptr, align 8
-; CHECK-NEXT:    call void @llvm.va_start(ptr [[VARARGS]])
+; CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[VARARGS]])
 ; CHECK-NEXT:    [[V0:%.*]] = va_arg ptr [[VARARGS]], i32
 ; CHECK-NEXT:    [[V1:%.*]] = va_arg ptr [[VARARGS]], i32
 ; CHECK-NEXT:    [[V0_NEG:%.*]] = sub i32 0, [[V0]]
 ; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[V0_NEG]], 1
 ; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[SUB]], [[V1]]
-; CHECK-NEXT:    call void @llvm.va_end(ptr [[VARARGS]])
+; CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VARARGS]])
 ; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %varargs = alloca ptr, align 8
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index f4bac9376f2e..28526f1a1560 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -611,19 +611,19 @@ def LLVM_DbgLabelOp : LLVM_IntrOp<"dbg.label", [], [], [], 0> {
 // Variadic function intrinsics.
 //
 
-def LLVM_VaStartOp : LLVM_ZeroResultIntrOp<"vastart">,
+def LLVM_VaStartOp : LLVM_ZeroResultIntrOp<"vastart", [0]>,
                      Arguments<(ins LLVM_AnyPointer:$arg_list)> {
   let assemblyFormat = "$arg_list attr-dict `:` qualified(type($arg_list))";
   let summary = "Initializes `arg_list` for subsequent variadic argument extractions.";
 }
 
-def LLVM_VaCopyOp : LLVM_ZeroResultIntrOp<"vacopy">,
+def LLVM_VaCopyOp : LLVM_ZeroResultIntrOp<"vacopy", [0]>,
                     Arguments<(ins LLVM_AnyPointer:$dest_list, LLVM_AnyPointer:$src_list)> {
   let assemblyFormat = "$src_list `to` $dest_list attr-dict `:` type(operands)";
   let summary = "Copies the current argument position from `src_list` to `dest_list`.";
 }
 
-def LLVM_VaEndOp : LLVM_ZeroResultIntrOp<"vaend">,
+def LLVM_VaEndOp : LLVM_ZeroResultIntrOp<"vaend", [0]>,
                    Arguments<(ins LLVM_AnyPointer:$arg_list)> {
   let assemblyFormat = "$arg_list attr-dict `:` qualified(type($arg_list))";
   let summary = "Destroys `arg_list`, which has been initialized by `intr.vastart` or `intr.vacopy`.";
diff --git a/mlir/test/Target/LLVMIR/Import/basic.ll b/mlir/test/Target/LLVMIR/Import/basic.ll
index a059425d9780..448b0ebe2574 100644
--- a/mlir/test/Target/LLVMIR/Import/basic.ll
+++ b/mlir/test/Target/LLVMIR/Import/basic.ll
@@ -72,26 +72,26 @@ define i32 @useFreezeOp(i32 %x) {
 ; Varadic function definition
 %struct.va_list = type { ptr }
 
-declare void @llvm.va_start(ptr)
-declare void @llvm.va_copy(ptr, ptr)
-declare void @llvm.va_end(ptr)
+declare void @llvm.va_start.p0(ptr)
+declare void @llvm.va_copy.p0(ptr, ptr)
+declare void @llvm.va_end.p0(ptr)
 
 ; CHECK-LABEL: llvm.func @variadic_function
 define void @variadic_function(i32 %X, ...) {
   ; CHECK: %[[ALLOCA0:.+]] = llvm.alloca %{{.*}} x !llvm.struct<"struct.va_list", (ptr)> {{.*}} : (i32) -> !llvm.ptr
   %ap = alloca %struct.va_list
   ; CHECK: llvm.intr.vastart %[[ALLOCA0]]
-  call void @llvm.va_start(ptr %ap)
+  call void @llvm.va_start.p0(ptr %ap)
 
   ; CHECK: %[[ALLOCA1:.+]] = llvm.alloca %{{.*}} x !llvm.ptr {{.*}} : (i32) -> !llvm.ptr
   %aq = alloca ptr
   ; CHECK: llvm.intr.vacopy %[[ALLOCA0]] to %[[ALLOCA1]]
-  call void @llvm.va_copy(ptr %aq, ptr %ap)
+  call void @llvm.va_copy.p0(ptr %aq, ptr %ap)
   ; CHECK: llvm.intr.vaend %[[ALLOCA1]]
-  call void @llvm.va_end(ptr %aq)
+  call void @llvm.va_end.p0(ptr %aq)
 
   ; CHECK: llvm.intr.vaend %[[ALLOCA0]]
-  call void @llvm.va_end(ptr %ap)
+  call void @llvm.va_end.p0(ptr %ap)
   ; CHECK: llvm.return
   ret void
 }
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index 85561839f31a..0cefb4f8983a 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -599,11 +599,11 @@ define void @ushl_sat_test(i32 %0, i32 %1, <8 x i32> %2, <8 x i32> %3) {
 ; CHECK-LABEL: llvm.func @va_intrinsics_test
 define void @va_intrinsics_test(ptr %0, ptr %1) {
 ; CHECK: llvm.intr.vastart %{{.*}}
-  call void @llvm.va_start(ptr %0)
+  call void @llvm.va_start.p0(ptr %0)
 ; CHECK: llvm.intr.vacopy %{{.*}} to %{{.*}}
-  call void @llvm.va_copy(ptr %1, ptr %0)
+  call void @llvm.va_copy.p0(ptr %1, ptr %0)
 ; CHECK: llvm.intr.vaend %{{.*}}
-  call void @llvm.va_end(ptr %0)
+  call void @llvm.va_end.p0(ptr %0)
   ret void
 }
 
@@ -1076,9 +1076,9 @@ declare ptr @llvm.stacksave.p0()
 declare ptr addrspace(1) @llvm.stacksave.p1()
 declare void @llvm.stackrestore.p0(ptr)
 declare void @llvm.stackrestore.p1(ptr addrspace(1))
-declare void @llvm.va_start(ptr)
-declare void @llvm.va_copy(ptr, ptr)
-declare void @llvm.va_end(ptr)
+declare void @llvm.va_start.p0(ptr)
+declare void @llvm.va_copy.p0(ptr, ptr)
+declare void @llvm.va_end.p0(ptr)
 declare <8 x i32> @llvm.vp.add.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32)
 declare <8 x i32> @llvm.vp.sub.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32)
 declare <8 x i32> @llvm.vp.mul.v8i32(<8 x i32>, <8 x i32>, <8 x i1>, i32)
diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir
index c38c7ea587d2..97f37939551d 100644
--- a/mlir/test/Target/LLVMIR/llvmir.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir.mlir
@@ -2251,14 +2251,14 @@ llvm.func @vararg_function(%arg0: i32, ...) {
   %1 = llvm.mlir.constant(1 : i32) : i32
   // CHECK: %[[ALLOCA0:.+]] = alloca %struct.va_list, align 8
   %2 = llvm.alloca %1 x !llvm.struct<"struct.va_list", (ptr)> {alignment = 8 : i64} : (i32) -> !llvm.ptr
-  // CHECK: call void @llvm.va_start(ptr %[[ALLOCA0]])
+  // CHECK: call void @llvm.va_start.p0(ptr %[[ALLOCA0]])
   llvm.intr.vastart %2 : !llvm.ptr
   // CHECK: %[[ALLOCA1:.+]] = alloca ptr, align 8
   %4 = llvm.alloca %0 x !llvm.ptr {alignment = 8 : i64} : (i32) -> !llvm.ptr
-  // CHECK: call void @llvm.va_copy(ptr %[[ALLOCA1]], ptr %[[ALLOCA0]])
+  // CHECK: call void @llvm.va_copy.p0(ptr %[[ALLOCA1]], ptr %[[ALLOCA0]])
   llvm.intr.vacopy %2 to %4 : !llvm.ptr, !llvm.ptr
-  // CHECK: call void @llvm.va_end(ptr %[[ALLOCA1]])
-  // CHECK: call void @llvm.va_end(ptr %[[ALLOCA0]])
+  // CHECK: call void @llvm.va_end.p0(ptr %[[ALLOCA1]])
+  // CHECK: call void @llvm.va_end.p0(ptr %[[ALLOCA0]])
   llvm.intr.vaend %4 : !llvm.ptr
   llvm.intr.vaend %2 : !llvm.ptr
   // CHECK: ret void
-- 
GitLab


From ef316da4a2c5954a02c92707b5cb621402b76910 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Wed, 27 Mar 2024 00:56:47 +0530
Subject: [PATCH 393/404] AMDGPU: Fix dead check prefixes in test

---
 .../AMDGPU/global_atomics_i32_system.ll       | 564 ------------------
 .../AMDGPU/global_atomics_i64_system.ll       | 564 ------------------
 2 files changed, 1128 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
index 76ec1cc84f55..99d02ffaa523 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
@@ -358,65 +358,6 @@ define amdgpu_gfx i32 @global_atomic_xchg_i32_ret_offset_scalar(ptr addrspace(1)
 ; ---------------------------------------------------------------------
 
 define void @global_atomic_xchg_f32_noret(ptr addrspace(1) %ptr, float %in) {
-; GCN1-LABEL: global_atomic_xchg_f32_noret:
-; GCN1:       ; %bb.0:
-; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_load_dword v3, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[4:5], 0
-; GCN1-NEXT:  .LBB0_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] glc
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT:    v_mov_b32_e32 v3, v4
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB0_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN2-LABEL: global_atomic_xchg_f32_noret:
-; GCN2:       ; %bb.0:
-; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_load_dword v3, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[4:5], 0
-; GCN2-NEXT:  .LBB0_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] glc
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT:    v_mov_b32_e32 v3, v4
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB0_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN3-LABEL: global_atomic_xchg_f32_noret:
-; GCN3:       ; %bb.0:
-; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_load_dword v3, v[0:1]
-; GCN3-NEXT:    s_mov_b64 s[4:5], 0
-; GCN3-NEXT:  .LBB0_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] glc
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT:    v_mov_b32_e32 v3, v4
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB0_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; SI-LABEL: global_atomic_xchg_f32_noret:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -450,69 +391,6 @@ define void @global_atomic_xchg_f32_noret(ptr addrspace(1) %ptr, float %in) {
 }
 
 define void @global_atomic_xchg_f32_noret_offset(ptr addrspace(1) %out, float %in) {
-; GCN1-LABEL: global_atomic_xchg_f32_noret_offset:
-; GCN1:       ; %bb.0:
-; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_add_f32_e32 v0, vcc, 16, v0
-; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN1-NEXT:    global_load_dword v3, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[4:5], 0
-; GCN1-NEXT:  .LBB1_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] glc
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT:    v_mov_b32_e32 v3, v4
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB1_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN2-LABEL: global_atomic_xchg_f32_noret_offset:
-; GCN2:       ; %bb.0:
-; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
-; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN2-NEXT:    global_load_dword v3, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[4:5], 0
-; GCN2-NEXT:  .LBB1_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] glc
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT:    v_mov_b32_e32 v3, v4
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB1_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN3-LABEL: global_atomic_xchg_f32_noret_offset:
-; GCN3:       ; %bb.0:
-; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_load_dword v3, v[0:1] offset:16
-; GCN3-NEXT:    s_mov_b64 s[4:5], 0
-; GCN3-NEXT:  .LBB1_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] offset:16 glc
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT:    v_mov_b32_e32 v3, v4
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB1_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; SI-LABEL: global_atomic_xchg_f32_noret_offset:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -549,71 +427,6 @@ define void @global_atomic_xchg_f32_noret_offset(ptr addrspace(1) %out, float %i
 }
 
 define float @global_atomic_xchg_f32_ret(ptr addrspace(1) %ptr, float %in) {
-; GCN1-LABEL: global_atomic_xchg_f32_ret:
-; GCN1:       ; %bb.0:
-; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_load_dword v4, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[4:5], 0
-; GCN1-NEXT:  .LBB2_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v3, v4
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] glc
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB2_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    v_mov_b32_e32 v0, v4
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN2-LABEL: global_atomic_xchg_f32_ret:
-; GCN2:       ; %bb.0:
-; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_load_dword v4, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[4:5], 0
-; GCN2-NEXT:  .LBB2_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v3, v4
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] glc
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB2_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    v_mov_b32_e32 v0, v4
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN3-LABEL: global_atomic_xchg_f32_ret:
-; GCN3:       ; %bb.0:
-; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_load_dword v4, v[0:1]
-; GCN3-NEXT:    s_mov_b64 s[4:5], 0
-; GCN3-NEXT:  .LBB2_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v3, v4
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] glc
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB2_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    v_mov_b32_e32 v0, v4
-; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; SI-LABEL: global_atomic_xchg_f32_ret:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -648,73 +461,6 @@ define float @global_atomic_xchg_f32_ret(ptr addrspace(1) %ptr, float %in) {
 }
 
 define float @global_atomic_xchg_f32_ret_offset(ptr addrspace(1) %out, float %in) {
-; GCN1-LABEL: global_atomic_xchg_f32_ret_offset:
-; GCN1:       ; %bb.0:
-; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_add_f32_e32 v4, vcc, 16, v0
-; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN1-NEXT:    global_load_dword v0, v[4:5]
-; GCN1-NEXT:    s_mov_b64 s[4:5], 0
-; GCN1-NEXT:  .LBB3_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v3, v0
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_atomic_cmpswap v0, v[4:5], v[2:3] glc
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
-; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB3_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN2-LABEL: global_atomic_xchg_f32_ret_offset:
-; GCN2:       ; %bb.0:
-; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 16, v0
-; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN2-NEXT:    global_load_dword v0, v[4:5]
-; GCN2-NEXT:    s_mov_b64 s[4:5], 0
-; GCN2-NEXT:  .LBB3_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v3, v0
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_atomic_cmpswap v0, v[4:5], v[2:3] glc
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
-; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB3_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN3-LABEL: global_atomic_xchg_f32_ret_offset:
-; GCN3:       ; %bb.0:
-; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_load_dword v4, v[0:1] offset:16
-; GCN3-NEXT:    s_mov_b64 s[4:5], 0
-; GCN3-NEXT:  .LBB3_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v3, v4
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] offset:16 glc
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB3_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    v_mov_b32_e32 v0, v4
-; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; SI-LABEL: global_atomic_xchg_f32_ret_offset:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -752,80 +498,6 @@ define float @global_atomic_xchg_f32_ret_offset(ptr addrspace(1) %out, float %in
 }
 
 define amdgpu_gfx void @global_atomic_xchg_f32_noret_scalar(ptr addrspace(1) inreg %ptr, float inreg %in) {
-; GCN1-LABEL: global_atomic_xchg_f32_noret_scalar:
-; GCN1:       ; %bb.0:
-; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v0, s4
-; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    global_load_dword v1, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[34:35], 0
-; GCN1-NEXT:  .LBB4_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    v_mov_b32_e32 v2, s4
-; GCN1-NEXT:    v_mov_b32_e32 v0, s6
-; GCN1-NEXT:    v_mov_b32_e32 v3, s5
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_atomic_cmpswap v0, v[2:3], v[0:1] glc
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
-; GCN1-NEXT:    v_mov_b32_e32 v1, v0
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GCN1-NEXT:    s_cbranch_execnz .LBB4_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN2-LABEL: global_atomic_xchg_f32_noret_scalar:
-; GCN2:       ; %bb.0:
-; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v0, s4
-; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    global_load_dword v1, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[34:35], 0
-; GCN2-NEXT:  .LBB4_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    v_mov_b32_e32 v2, s4
-; GCN2-NEXT:    v_mov_b32_e32 v0, s6
-; GCN2-NEXT:    v_mov_b32_e32 v3, s5
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_atomic_cmpswap v0, v[2:3], v[0:1] glc
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
-; GCN2-NEXT:    v_mov_b32_e32 v1, v0
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GCN2-NEXT:    s_cbranch_execnz .LBB4_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN3-LABEL: global_atomic_xchg_f32_noret_scalar:
-; GCN3:       ; %bb.0:
-; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    global_load_dword v1, v[0:1]
-; GCN3-NEXT:    s_mov_b64 s[34:35], 0
-; GCN3-NEXT:  .LBB4_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    v_mov_b32_e32 v2, s4
-; GCN3-NEXT:    v_mov_b32_e32 v0, s6
-; GCN3-NEXT:    v_mov_b32_e32 v3, s5
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_atomic_cmpswap v0, v[2:3], v[0:1] glc
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT:    v_mov_b32_e32 v1, v0
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_execnz .LBB4_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
-; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; SI-LABEL: global_atomic_xchg_f32_noret_scalar:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -876,84 +548,6 @@ define amdgpu_gfx void @global_atomic_xchg_f32_noret_scalar(ptr addrspace(1) inr
 }
 
 define amdgpu_gfx void @global_atomic_xchg_f32_noret_offset_scalar(ptr addrspace(1) inreg %out, float inreg %in) {
-; GCN1-LABEL: global_atomic_xchg_f32_noret_offset_scalar:
-; GCN1:       ; %bb.0:
-; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    s_add_u32 s34, s4, 16
-; GCN1-NEXT:    s_addc_u32 s35, s5, 0
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    global_load_dword v1, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[36:37], 0
-; GCN1-NEXT:  .LBB5_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    v_mov_b32_e32 v2, s34
-; GCN1-NEXT:    v_mov_b32_e32 v0, s6
-; GCN1-NEXT:    v_mov_b32_e32 v3, s35
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_atomic_cmpswap v0, v[2:3], v[0:1] glc
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT:    v_mov_b32_e32 v1, v0
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; GCN1-NEXT:    s_cbranch_execnz .LBB5_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN2-LABEL: global_atomic_xchg_f32_noret_offset_scalar:
-; GCN2:       ; %bb.0:
-; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    s_add_u32 s34, s4, 16
-; GCN2-NEXT:    s_addc_u32 s35, s5, 0
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    global_load_dword v1, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[36:37], 0
-; GCN2-NEXT:  .LBB5_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    v_mov_b32_e32 v2, s34
-; GCN2-NEXT:    v_mov_b32_e32 v0, s6
-; GCN2-NEXT:    v_mov_b32_e32 v3, s35
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_atomic_cmpswap v0, v[2:3], v[0:1] glc
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT:    v_mov_b32_e32 v1, v0
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; GCN2-NEXT:    s_cbranch_execnz .LBB5_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN3-LABEL: global_atomic_xchg_f32_noret_offset_scalar:
-; GCN3:       ; %bb.0:
-; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    global_load_dword v1, v[0:1] offset:16
-; GCN3-NEXT:    s_mov_b64 s[34:35], 0
-; GCN3-NEXT:  .LBB5_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    v_mov_b32_e32 v2, s4
-; GCN3-NEXT:    v_mov_b32_e32 v0, s6
-; GCN3-NEXT:    v_mov_b32_e32 v3, s5
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT:    v_mov_b32_e32 v1, v0
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_execnz .LBB5_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
-; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; SI-LABEL: global_atomic_xchg_f32_noret_offset_scalar:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1007,83 +601,6 @@ define amdgpu_gfx void @global_atomic_xchg_f32_noret_offset_scalar(ptr addrspace
 }
 
 define amdgpu_gfx float @global_atomic_xchg_f32_ret_scalar(ptr addrspace(1) inreg %ptr, float inreg %in) {
-; GCN1-LABEL: global_atomic_xchg_f32_ret_scalar:
-; GCN1:       ; %bb.0:
-; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v0, s4
-; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    global_load_dword v0, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[34:35], 0
-; GCN1-NEXT:  .LBB6_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    v_mov_b32_e32 v3, s4
-; GCN1-NEXT:    v_mov_b32_e32 v1, s6
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v2, v0
-; GCN1-NEXT:    v_mov_b32_e32 v4, s5
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_atomic_cmpswap v0, v[3:4], v[1:2] glc
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
-; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GCN1-NEXT:    s_cbranch_execnz .LBB6_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN2-LABEL: global_atomic_xchg_f32_ret_scalar:
-; GCN2:       ; %bb.0:
-; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v0, s4
-; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    global_load_dword v0, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[34:35], 0
-; GCN2-NEXT:  .LBB6_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    v_mov_b32_e32 v3, s4
-; GCN2-NEXT:    v_mov_b32_e32 v1, s6
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v2, v0
-; GCN2-NEXT:    v_mov_b32_e32 v4, s5
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_atomic_cmpswap v0, v[3:4], v[1:2] glc
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
-; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GCN2-NEXT:    s_cbranch_execnz .LBB6_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN3-LABEL: global_atomic_xchg_f32_ret_scalar:
-; GCN3:       ; %bb.0:
-; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    global_load_dword v0, v[0:1]
-; GCN3-NEXT:    s_mov_b64 s[34:35], 0
-; GCN3-NEXT:  .LBB6_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    v_mov_b32_e32 v3, s4
-; GCN3-NEXT:    v_mov_b32_e32 v1, s6
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v2, v0
-; GCN3-NEXT:    v_mov_b32_e32 v4, s5
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_atomic_cmpswap v0, v[3:4], v[1:2] glc
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
-; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_execnz .LBB6_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
-; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; SI-LABEL: global_atomic_xchg_f32_ret_scalar:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1134,87 +651,6 @@ define amdgpu_gfx float @global_atomic_xchg_f32_ret_scalar(ptr addrspace(1) inre
 }
 
 define amdgpu_gfx float @global_atomic_xchg_f32_ret_offset_scalar(ptr addrspace(1) inreg %out, float inreg %in) {
-; GCN1-LABEL: global_atomic_xchg_f32_ret_offset_scalar:
-; GCN1:       ; %bb.0:
-; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    s_add_u32 s34, s4, 16
-; GCN1-NEXT:    s_addc_u32 s35, s5, 0
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    global_load_dword v0, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[36:37], 0
-; GCN1-NEXT:  .LBB7_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    v_mov_b32_e32 v3, s34
-; GCN1-NEXT:    v_mov_b32_e32 v1, s6
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v2, v0
-; GCN1-NEXT:    v_mov_b32_e32 v4, s35
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_atomic_cmpswap v0, v[3:4], v[1:2] glc
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
-; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; GCN1-NEXT:    s_cbranch_execnz .LBB7_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN2-LABEL: global_atomic_xchg_f32_ret_offset_scalar:
-; GCN2:       ; %bb.0:
-; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    s_add_u32 s34, s4, 16
-; GCN2-NEXT:    s_addc_u32 s35, s5, 0
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    global_load_dword v0, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[36:37], 0
-; GCN2-NEXT:  .LBB7_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    v_mov_b32_e32 v3, s34
-; GCN2-NEXT:    v_mov_b32_e32 v1, s6
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v2, v0
-; GCN2-NEXT:    v_mov_b32_e32 v4, s35
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_atomic_cmpswap v0, v[3:4], v[1:2] glc
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
-; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; GCN2-NEXT:    s_cbranch_execnz .LBB7_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN3-LABEL: global_atomic_xchg_f32_ret_offset_scalar:
-; GCN3:       ; %bb.0:
-; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    global_load_dword v0, v[0:1] offset:16
-; GCN3-NEXT:    s_mov_b64 s[34:35], 0
-; GCN3-NEXT:  .LBB7_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    v_mov_b32_e32 v3, s4
-; GCN3-NEXT:    v_mov_b32_e32 v1, s6
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v2, v0
-; GCN3-NEXT:    v_mov_b32_e32 v4, s5
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_atomic_cmpswap v0, v[3:4], v[1:2] offset:16 glc
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
-; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_execnz .LBB7_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
-; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; SI-LABEL: global_atomic_xchg_f32_ret_offset_scalar:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll
index d137f471910d..380ce7f3b939 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll
@@ -372,65 +372,6 @@ define amdgpu_gfx i64 @global_atomic_xchg_i64_ret_offset_scalar(ptr addrspace(1)
 ; ---------------------------------------------------------------------
 
 define void @global_atomic_xchg_f64_noret(ptr addrspace(1) %ptr, double %in) {
-; GCN1-LABEL: global_atomic_xchg_f64_noret:
-; GCN1:       ; %bb.0:
-; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_load_dword v3, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[4:5], 0
-; GCN1-NEXT:  .LBB0_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] glc
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT:    v_mov_b32_e32 v3, v4
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB0_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN2-LABEL: global_atomic_xchg_f64_noret:
-; GCN2:       ; %bb.0:
-; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_load_dword v3, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[4:5], 0
-; GCN2-NEXT:  .LBB0_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] glc
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT:    v_mov_b32_e32 v3, v4
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB0_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN3-LABEL: global_atomic_xchg_f64_noret:
-; GCN3:       ; %bb.0:
-; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_load_dword v3, v[0:1]
-; GCN3-NEXT:    s_mov_b64 s[4:5], 0
-; GCN3-NEXT:  .LBB0_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] glc
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT:    v_mov_b32_e32 v3, v4
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB0_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; SI-LABEL: global_atomic_xchg_f64_noret:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -464,69 +405,6 @@ define void @global_atomic_xchg_f64_noret(ptr addrspace(1) %ptr, double %in) {
 }
 
 define void @global_atomic_xchg_f64_noret_offset(ptr addrspace(1) %out, double %in) {
-; GCN1-LABEL: global_atomic_xchg_f64_noret_offset:
-; GCN1:       ; %bb.0:
-; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_add_f64_e32 v0, vcc, 16, v0
-; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN1-NEXT:    global_load_dword v3, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[4:5], 0
-; GCN1-NEXT:  .LBB1_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] glc
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT:    v_mov_b32_e32 v3, v4
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB1_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN2-LABEL: global_atomic_xchg_f64_noret_offset:
-; GCN2:       ; %bb.0:
-; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
-; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN2-NEXT:    global_load_dword v3, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[4:5], 0
-; GCN2-NEXT:  .LBB1_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] glc
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT:    v_mov_b32_e32 v3, v4
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB1_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN3-LABEL: global_atomic_xchg_f64_noret_offset:
-; GCN3:       ; %bb.0:
-; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_load_dword v3, v[0:1] offset:16
-; GCN3-NEXT:    s_mov_b64 s[4:5], 0
-; GCN3-NEXT:  .LBB1_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] offset:16 glc
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT:    v_mov_b32_e32 v3, v4
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB1_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; SI-LABEL: global_atomic_xchg_f64_noret_offset:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -563,71 +441,6 @@ define void @global_atomic_xchg_f64_noret_offset(ptr addrspace(1) %out, double %
 }
 
 define double @global_atomic_xchg_f64_ret(ptr addrspace(1) %ptr, double %in) {
-; GCN1-LABEL: global_atomic_xchg_f64_ret:
-; GCN1:       ; %bb.0:
-; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_load_dword v4, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[4:5], 0
-; GCN1-NEXT:  .LBB2_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v3, v4
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] glc
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB2_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    v_mov_b32_e32 v0, v4
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN2-LABEL: global_atomic_xchg_f64_ret:
-; GCN2:       ; %bb.0:
-; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_load_dword v4, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[4:5], 0
-; GCN2-NEXT:  .LBB2_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v3, v4
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] glc
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB2_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    v_mov_b32_e32 v0, v4
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN3-LABEL: global_atomic_xchg_f64_ret:
-; GCN3:       ; %bb.0:
-; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_load_dword v4, v[0:1]
-; GCN3-NEXT:    s_mov_b64 s[4:5], 0
-; GCN3-NEXT:  .LBB2_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v3, v4
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] glc
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB2_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    v_mov_b32_e32 v0, v4
-; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; SI-LABEL: global_atomic_xchg_f64_ret:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -663,73 +476,6 @@ define double @global_atomic_xchg_f64_ret(ptr addrspace(1) %ptr, double %in) {
 }
 
 define double @global_atomic_xchg_f64_ret_offset(ptr addrspace(1) %out, double %in) {
-; GCN1-LABEL: global_atomic_xchg_f64_ret_offset:
-; GCN1:       ; %bb.0:
-; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_add_f64_e32 v4, vcc, 16, v0
-; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN1-NEXT:    global_load_dword v0, v[4:5]
-; GCN1-NEXT:    s_mov_b64 s[4:5], 0
-; GCN1-NEXT:  .LBB3_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v3, v0
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_atomic_cmpswap v0, v[4:5], v[2:3] glc
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
-; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB3_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN2-LABEL: global_atomic_xchg_f64_ret_offset:
-; GCN2:       ; %bb.0:
-; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 16, v0
-; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN2-NEXT:    global_load_dword v0, v[4:5]
-; GCN2-NEXT:    s_mov_b64 s[4:5], 0
-; GCN2-NEXT:  .LBB3_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v3, v0
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_atomic_cmpswap v0, v[4:5], v[2:3] glc
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
-; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB3_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN3-LABEL: global_atomic_xchg_f64_ret_offset:
-; GCN3:       ; %bb.0:
-; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_load_dword v4, v[0:1] offset:16
-; GCN3-NEXT:    s_mov_b64 s[4:5], 0
-; GCN3-NEXT:  .LBB3_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v3, v4
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_atomic_cmpswap v4, v[0:1], v[2:3] offset:16 glc
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v4, v3
-; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB3_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    v_mov_b32_e32 v0, v4
-; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; SI-LABEL: global_atomic_xchg_f64_ret_offset:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -768,80 +514,6 @@ define double @global_atomic_xchg_f64_ret_offset(ptr addrspace(1) %out, double %
 }
 
 define amdgpu_gfx void @global_atomic_xchg_f64_noret_scalar(ptr addrspace(1) inreg %ptr, double inreg %in) {
-; GCN1-LABEL: global_atomic_xchg_f64_noret_scalar:
-; GCN1:       ; %bb.0:
-; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v0, s4
-; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    global_load_dword v1, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[34:35], 0
-; GCN1-NEXT:  .LBB4_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    v_mov_b32_e32 v2, s4
-; GCN1-NEXT:    v_mov_b32_e32 v0, s6
-; GCN1-NEXT:    v_mov_b32_e32 v3, s5
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_atomic_cmpswap v0, v[2:3], v[0:1] glc
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
-; GCN1-NEXT:    v_mov_b32_e32 v1, v0
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GCN1-NEXT:    s_cbranch_execnz .LBB4_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN2-LABEL: global_atomic_xchg_f64_noret_scalar:
-; GCN2:       ; %bb.0:
-; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v0, s4
-; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    global_load_dword v1, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[34:35], 0
-; GCN2-NEXT:  .LBB4_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    v_mov_b32_e32 v2, s4
-; GCN2-NEXT:    v_mov_b32_e32 v0, s6
-; GCN2-NEXT:    v_mov_b32_e32 v3, s5
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_atomic_cmpswap v0, v[2:3], v[0:1] glc
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
-; GCN2-NEXT:    v_mov_b32_e32 v1, v0
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GCN2-NEXT:    s_cbranch_execnz .LBB4_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN3-LABEL: global_atomic_xchg_f64_noret_scalar:
-; GCN3:       ; %bb.0:
-; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    global_load_dword v1, v[0:1]
-; GCN3-NEXT:    s_mov_b64 s[34:35], 0
-; GCN3-NEXT:  .LBB4_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    v_mov_b32_e32 v2, s4
-; GCN3-NEXT:    v_mov_b32_e32 v0, s6
-; GCN3-NEXT:    v_mov_b32_e32 v3, s5
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_atomic_cmpswap v0, v[2:3], v[0:1] glc
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT:    v_mov_b32_e32 v1, v0
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_execnz .LBB4_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
-; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; SI-LABEL: global_atomic_xchg_f64_noret_scalar:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -896,84 +568,6 @@ define amdgpu_gfx void @global_atomic_xchg_f64_noret_scalar(ptr addrspace(1) inr
 }
 
 define amdgpu_gfx void @global_atomic_xchg_f64_noret_offset_scalar(ptr addrspace(1) inreg %out, double inreg %in) {
-; GCN1-LABEL: global_atomic_xchg_f64_noret_offset_scalar:
-; GCN1:       ; %bb.0:
-; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    s_add_u32 s34, s4, 16
-; GCN1-NEXT:    s_addc_u32 s35, s5, 0
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    global_load_dword v1, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[36:37], 0
-; GCN1-NEXT:  .LBB5_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    v_mov_b32_e32 v2, s34
-; GCN1-NEXT:    v_mov_b32_e32 v0, s6
-; GCN1-NEXT:    v_mov_b32_e32 v3, s35
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_atomic_cmpswap v0, v[2:3], v[0:1] glc
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT:    v_mov_b32_e32 v1, v0
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; GCN1-NEXT:    s_cbranch_execnz .LBB5_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN2-LABEL: global_atomic_xchg_f64_noret_offset_scalar:
-; GCN2:       ; %bb.0:
-; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    s_add_u32 s34, s4, 16
-; GCN2-NEXT:    s_addc_u32 s35, s5, 0
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    global_load_dword v1, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[36:37], 0
-; GCN2-NEXT:  .LBB5_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    v_mov_b32_e32 v2, s34
-; GCN2-NEXT:    v_mov_b32_e32 v0, s6
-; GCN2-NEXT:    v_mov_b32_e32 v3, s35
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_atomic_cmpswap v0, v[2:3], v[0:1] glc
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT:    v_mov_b32_e32 v1, v0
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; GCN2-NEXT:    s_cbranch_execnz .LBB5_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN3-LABEL: global_atomic_xchg_f64_noret_offset_scalar:
-; GCN3:       ; %bb.0:
-; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    global_load_dword v1, v[0:1] offset:16
-; GCN3-NEXT:    s_mov_b64 s[34:35], 0
-; GCN3-NEXT:  .LBB5_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    v_mov_b32_e32 v2, s4
-; GCN3-NEXT:    v_mov_b32_e32 v0, s6
-; GCN3-NEXT:    v_mov_b32_e32 v3, s5
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT:    v_mov_b32_e32 v1, v0
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_execnz .LBB5_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
-; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; SI-LABEL: global_atomic_xchg_f64_noret_offset_scalar:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1029,83 +623,6 @@ define amdgpu_gfx void @global_atomic_xchg_f64_noret_offset_scalar(ptr addrspace
 }
 
 define amdgpu_gfx double @global_atomic_xchg_f64_ret_scalar(ptr addrspace(1) inreg %ptr, double inreg %in) {
-; GCN1-LABEL: global_atomic_xchg_f64_ret_scalar:
-; GCN1:       ; %bb.0:
-; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v0, s4
-; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    global_load_dword v0, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[34:35], 0
-; GCN1-NEXT:  .LBB6_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    v_mov_b32_e32 v3, s4
-; GCN1-NEXT:    v_mov_b32_e32 v1, s6
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v2, v0
-; GCN1-NEXT:    v_mov_b32_e32 v4, s5
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_atomic_cmpswap v0, v[3:4], v[1:2] glc
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
-; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GCN1-NEXT:    s_cbranch_execnz .LBB6_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN2-LABEL: global_atomic_xchg_f64_ret_scalar:
-; GCN2:       ; %bb.0:
-; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v0, s4
-; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    global_load_dword v0, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[34:35], 0
-; GCN2-NEXT:  .LBB6_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    v_mov_b32_e32 v3, s4
-; GCN2-NEXT:    v_mov_b32_e32 v1, s6
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v2, v0
-; GCN2-NEXT:    v_mov_b32_e32 v4, s5
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_atomic_cmpswap v0, v[3:4], v[1:2] glc
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
-; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GCN2-NEXT:    s_cbranch_execnz .LBB6_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN3-LABEL: global_atomic_xchg_f64_ret_scalar:
-; GCN3:       ; %bb.0:
-; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    global_load_dword v0, v[0:1]
-; GCN3-NEXT:    s_mov_b64 s[34:35], 0
-; GCN3-NEXT:  .LBB6_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    v_mov_b32_e32 v3, s4
-; GCN3-NEXT:    v_mov_b32_e32 v1, s6
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v2, v0
-; GCN3-NEXT:    v_mov_b32_e32 v4, s5
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_atomic_cmpswap v0, v[3:4], v[1:2] glc
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
-; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_execnz .LBB6_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
-; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; SI-LABEL: global_atomic_xchg_f64_ret_scalar:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1160,87 +677,6 @@ define amdgpu_gfx double @global_atomic_xchg_f64_ret_scalar(ptr addrspace(1) inr
 }
 
 define amdgpu_gfx double @global_atomic_xchg_f64_ret_offset_scalar(ptr addrspace(1) inreg %out, double inreg %in) {
-; GCN1-LABEL: global_atomic_xchg_f64_ret_offset_scalar:
-; GCN1:       ; %bb.0:
-; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    s_add_u32 s34, s4, 16
-; GCN1-NEXT:    s_addc_u32 s35, s5, 0
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    global_load_dword v0, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[36:37], 0
-; GCN1-NEXT:  .LBB7_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    v_mov_b32_e32 v3, s34
-; GCN1-NEXT:    v_mov_b32_e32 v1, s6
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v2, v0
-; GCN1-NEXT:    v_mov_b32_e32 v4, s35
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    global_atomic_cmpswap v0, v[3:4], v[1:2] glc
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
-; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; GCN1-NEXT:    s_cbranch_execnz .LBB7_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN2-LABEL: global_atomic_xchg_f64_ret_offset_scalar:
-; GCN2:       ; %bb.0:
-; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    s_add_u32 s34, s4, 16
-; GCN2-NEXT:    s_addc_u32 s35, s5, 0
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    global_load_dword v0, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[36:37], 0
-; GCN2-NEXT:  .LBB7_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    v_mov_b32_e32 v3, s34
-; GCN2-NEXT:    v_mov_b32_e32 v1, s6
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v2, v0
-; GCN2-NEXT:    v_mov_b32_e32 v4, s35
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    global_atomic_cmpswap v0, v[3:4], v[1:2] glc
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
-; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; GCN2-NEXT:    s_cbranch_execnz .LBB7_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN3-LABEL: global_atomic_xchg_f64_ret_offset_scalar:
-; GCN3:       ; %bb.0:
-; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    global_load_dword v0, v[0:1] offset:16
-; GCN3-NEXT:    s_mov_b64 s[34:35], 0
-; GCN3-NEXT:  .LBB7_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    v_mov_b32_e32 v3, s4
-; GCN3-NEXT:    v_mov_b32_e32 v1, s6
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v2, v0
-; GCN3-NEXT:    v_mov_b32_e32 v4, s5
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    global_atomic_cmpswap v0, v[3:4], v[1:2] offset:16 glc
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
-; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_execnz .LBB7_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
-; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; SI-LABEL: global_atomic_xchg_f64_ret_offset_scalar:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-- 
GitLab


From 1103a2a337e90d8c7cc417b89e43c7a33aaea21e Mon Sep 17 00:00:00 2001
From: Janek van Oirschot <5994977+JanekvO@users.noreply.github.com>
Date: Wed, 27 Mar 2024 11:59:56 +0000
Subject: [PATCH 394/404] Reland [AMDGPU] MCExpr-ify MC layer kernel descriptor
 (#86494)

Kernel descriptor attributes, with their respective emit and asm parse functionality, converted to MCExpr.

Relands #80855 with fixes
---
 llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp   |  40 +-
 llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h     |  11 +-
 .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp      | 194 ++++++---
 .../MCTargetDesc/AMDGPUMCKernelDescriptor.cpp |  98 +++++
 .../MCTargetDesc/AMDGPUMCKernelDescriptor.h   |  54 +++
 .../MCTargetDesc/AMDGPUTargetStreamer.cpp     | 404 +++++++++++-------
 .../MCTargetDesc/AMDGPUTargetStreamer.h       |  33 +-
 .../Target/AMDGPU/MCTargetDesc/CMakeLists.txt |   1 +
 .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp    |  41 --
 llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h |   7 -
 llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s        |  27 ++
 llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s    | 281 ++++++++++++
 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s     | 190 ++++++++
 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s     | 186 ++++++++
 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s     | 184 ++++++++
 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s      | 168 ++++++++
 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s      | 171 ++++++++
 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s    | 148 +++++++
 llvm/test/MC/AMDGPU/hsa-tg-split.s            |  74 ++++
 19 files changed, 1999 insertions(+), 313 deletions(-)
 create mode 100644 llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp
 create mode 100644 llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h
 create mode 100644 llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s
 create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s
 create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s
 create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s
 create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s
 create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s
 create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s
 create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s
 create mode 100644 llvm/test/MC/AMDGPU/hsa-tg-split.s

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index 72e8b59e0a40..052b231d62a3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -22,6 +22,7 @@
 #include "AMDKernelCodeT.h"
 #include "GCNSubtarget.h"
 #include "MCTargetDesc/AMDGPUInstPrinter.h"
+#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h"
 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
 #include "R600AsmPrinter.h"
 #include "SIMachineFunctionInfo.h"
@@ -428,38 +429,43 @@ uint16_t AMDGPUAsmPrinter::getAmdhsaKernelCodeProperties(
   return KernelCodeProperties;
 }
 
-amdhsa::kernel_descriptor_t AMDGPUAsmPrinter::getAmdhsaKernelDescriptor(
-    const MachineFunction &MF,
-    const SIProgramInfo &PI) const {
+MCKernelDescriptor
+AMDGPUAsmPrinter::getAmdhsaKernelDescriptor(const MachineFunction &MF,
+                                            const SIProgramInfo &PI) const {
   const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
   const Function &F = MF.getFunction();
   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
+  MCContext &Ctx = MF.getContext();
 
-  amdhsa::kernel_descriptor_t KernelDescriptor;
-  memset(&KernelDescriptor, 0x0, sizeof(KernelDescriptor));
+  MCKernelDescriptor KernelDescriptor;
 
   assert(isUInt<32>(PI.ScratchSize));
   assert(isUInt<32>(PI.getComputePGMRSrc1(STM)));
   assert(isUInt<32>(PI.getComputePGMRSrc2()));
 
-  KernelDescriptor.group_segment_fixed_size = PI.LDSSize;
-  KernelDescriptor.private_segment_fixed_size = PI.ScratchSize;
+  KernelDescriptor.group_segment_fixed_size =
+      MCConstantExpr::create(PI.LDSSize, Ctx);
+  KernelDescriptor.private_segment_fixed_size =
+      MCConstantExpr::create(PI.ScratchSize, Ctx);
 
   Align MaxKernArgAlign;
-  KernelDescriptor.kernarg_size = STM.getKernArgSegmentSize(F, MaxKernArgAlign);
+  KernelDescriptor.kernarg_size = MCConstantExpr::create(
+      STM.getKernArgSegmentSize(F, MaxKernArgAlign), Ctx);
 
-  KernelDescriptor.compute_pgm_rsrc1 = PI.getComputePGMRSrc1(STM);
-  KernelDescriptor.compute_pgm_rsrc2 = PI.getComputePGMRSrc2();
-  KernelDescriptor.kernel_code_properties = getAmdhsaKernelCodeProperties(MF);
+  KernelDescriptor.compute_pgm_rsrc1 =
+      MCConstantExpr::create(PI.getComputePGMRSrc1(STM), Ctx);
+  KernelDescriptor.compute_pgm_rsrc2 =
+      MCConstantExpr::create(PI.getComputePGMRSrc2(), Ctx);
+  KernelDescriptor.kernel_code_properties =
+      MCConstantExpr::create(getAmdhsaKernelCodeProperties(MF), Ctx);
 
   assert(STM.hasGFX90AInsts() || CurrentProgramInfo.ComputePGMRSrc3GFX90A == 0);
-  if (STM.hasGFX90AInsts())
-    KernelDescriptor.compute_pgm_rsrc3 =
-      CurrentProgramInfo.ComputePGMRSrc3GFX90A;
+  KernelDescriptor.compute_pgm_rsrc3 = MCConstantExpr::create(
+      STM.hasGFX90AInsts() ? CurrentProgramInfo.ComputePGMRSrc3GFX90A : 0, Ctx);
 
-  if (AMDGPU::hasKernargPreload(STM))
-    KernelDescriptor.kernarg_preload =
-        static_cast<uint16_t>(Info->getNumKernargPreloadedSGPRs());
+  KernelDescriptor.kernarg_preload = MCConstantExpr::create(
+      AMDGPU::hasKernargPreload(STM) ? Info->getNumKernargPreloadedSGPRs() : 0,
+      Ctx);
 
   return KernelDescriptor;
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
index 79326cd3d328..b8b2718d293e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
@@ -28,15 +28,12 @@ class MCCodeEmitter;
 class MCOperand;
 
 namespace AMDGPU {
+struct MCKernelDescriptor;
 namespace HSAMD {
 class MetadataStreamer;
 }
 } // namespace AMDGPU
 
-namespace amdhsa {
-struct kernel_descriptor_t;
-}
-
 class AMDGPUAsmPrinter final : public AsmPrinter {
 private:
   unsigned CodeObjectVersion;
@@ -75,9 +72,9 @@ private:
   uint16_t getAmdhsaKernelCodeProperties(
       const MachineFunction &MF) const;
 
-  amdhsa::kernel_descriptor_t getAmdhsaKernelDescriptor(
-      const MachineFunction &MF,
-      const SIProgramInfo &PI) const;
+  AMDGPU::MCKernelDescriptor
+  getAmdhsaKernelDescriptor(const MachineFunction &MF,
+                            const SIProgramInfo &PI) const;
 
   void initTargetStreamer(Module &M);
 
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 4648df199c74..294fc683fe92 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -8,6 +8,7 @@
 
 #include "AMDKernelCodeT.h"
 #include "MCTargetDesc/AMDGPUMCExpr.h"
+#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
 #include "MCTargetDesc/AMDGPUTargetStreamer.h"
 #include "SIDefines.h"
@@ -5417,7 +5418,9 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
   if (getParser().parseIdentifier(KernelName))
     return true;
 
-  kernel_descriptor_t KD = getDefaultAmdhsaKernelDescriptor(&getSTI());
+  AMDGPU::MCKernelDescriptor KD =
+      AMDGPU::MCKernelDescriptor::getDefaultAmdhsaKernelDescriptor(
+          &getSTI(), getContext());
 
   StringSet<> Seen;
 
@@ -5457,89 +5460,111 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
       return TokError(".amdhsa_ directives cannot be repeated");
 
     SMLoc ValStart = getLoc();
-    int64_t IVal;
-    if (getParser().parseAbsoluteExpression(IVal))
+    const MCExpr *ExprVal;
+    if (getParser().parseExpression(ExprVal))
       return true;
     SMLoc ValEnd = getLoc();
     SMRange ValRange = SMRange(ValStart, ValEnd);
 
-    if (IVal < 0)
-      return OutOfRangeError(ValRange);
-
+    int64_t IVal = 0;
     uint64_t Val = IVal;
+    bool EvaluatableExpr;
+    if ((EvaluatableExpr = ExprVal->evaluateAsAbsolute(IVal))) {
+      if (IVal < 0)
+        return OutOfRangeError(ValRange);
+      Val = IVal;
+    }
 
 #define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE)                           \
-  if (!isUInt<ENTRY##_WIDTH>(VALUE))                                           \
+  if (!isUInt<ENTRY##_WIDTH>(Val))                                             \
     return OutOfRangeError(RANGE);                                             \
-  AMDHSA_BITS_SET(FIELD, ENTRY, VALUE);
+  AMDGPU::MCKernelDescriptor::bits_set(FIELD, VALUE, ENTRY##_SHIFT, ENTRY,     \
+                                       getContext());
+
+// Some fields use the parsed value immediately which requires the expression to
+// be solvable.
+#define EXPR_RESOLVE_OR_ERROR(RESOLVED)                                        \
+  if (!(RESOLVED))                                                             \
+    return Error(IDRange.Start, "directive should have resolvable expression", \
+                 IDRange);
 
     if (ID == ".amdhsa_group_segment_fixed_size") {
-      if (!isUInt<sizeof(KD.group_segment_fixed_size) * CHAR_BIT>(Val))
+      if (!isUInt<sizeof(kernel_descriptor_t::group_segment_fixed_size) *
+                  CHAR_BIT>(Val))
         return OutOfRangeError(ValRange);
-      KD.group_segment_fixed_size = Val;
+      KD.group_segment_fixed_size = ExprVal;
     } else if (ID == ".amdhsa_private_segment_fixed_size") {
-      if (!isUInt<sizeof(KD.private_segment_fixed_size) * CHAR_BIT>(Val))
+      if (!isUInt<sizeof(kernel_descriptor_t::private_segment_fixed_size) *
+                  CHAR_BIT>(Val))
         return OutOfRangeError(ValRange);
-      KD.private_segment_fixed_size = Val;
+      KD.private_segment_fixed_size = ExprVal;
     } else if (ID == ".amdhsa_kernarg_size") {
-      if (!isUInt<sizeof(KD.kernarg_size) * CHAR_BIT>(Val))
+      if (!isUInt<sizeof(kernel_descriptor_t::kernarg_size) * CHAR_BIT>(Val))
         return OutOfRangeError(ValRange);
-      KD.kernarg_size = Val;
+      KD.kernarg_size = ExprVal;
     } else if (ID == ".amdhsa_user_sgpr_count") {
+      EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
       ExplicitUserSGPRCount = Val;
     } else if (ID == ".amdhsa_user_sgpr_private_segment_buffer") {
+      EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
       if (hasArchitectedFlatScratch())
         return Error(IDRange.Start,
                      "directive is not supported with architected flat scratch",
                      IDRange);
       PARSE_BITS_ENTRY(KD.kernel_code_properties,
                        KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
-                       Val, ValRange);
+                       ExprVal, ValRange);
       if (Val)
         ImpliedUserSGPRCount += 4;
     } else if (ID == ".amdhsa_user_sgpr_kernarg_preload_length") {
+      EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
       if (!hasKernargPreload())
         return Error(IDRange.Start, "directive requires gfx90a+", IDRange);
 
       if (Val > getMaxNumUserSGPRs())
         return OutOfRangeError(ValRange);
-      PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_LENGTH, Val,
+      PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_LENGTH, ExprVal,
                        ValRange);
       if (Val) {
         ImpliedUserSGPRCount += Val;
         PreloadLength = Val;
       }
     } else if (ID == ".amdhsa_user_sgpr_kernarg_preload_offset") {
+      EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
       if (!hasKernargPreload())
         return Error(IDRange.Start, "directive requires gfx90a+", IDRange);
 
       if (Val >= 1024)
         return OutOfRangeError(ValRange);
-      PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_OFFSET, Val,
+      PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_OFFSET, ExprVal,
                        ValRange);
       if (Val)
         PreloadOffset = Val;
     } else if (ID == ".amdhsa_user_sgpr_dispatch_ptr") {
+      EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
       PARSE_BITS_ENTRY(KD.kernel_code_properties,
-                       KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, Val,
+                       KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, ExprVal,
                        ValRange);
       if (Val)
         ImpliedUserSGPRCount += 2;
     } else if (ID == ".amdhsa_user_sgpr_queue_ptr") {
+      EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
       PARSE_BITS_ENTRY(KD.kernel_code_properties,
-                       KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, Val,
+                       KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, ExprVal,
                        ValRange);
       if (Val)
         ImpliedUserSGPRCount += 2;
     } else if (ID == ".amdhsa_user_sgpr_kernarg_segment_ptr") {
+      EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
       PARSE_BITS_ENTRY(KD.kernel_code_properties,
                        KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
-                       Val, ValRange);
+                       ExprVal, ValRange);
       if (Val)
         ImpliedUserSGPRCount += 2;
     } else if (ID == ".amdhsa_user_sgpr_dispatch_id") {
+      EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
       PARSE_BITS_ENTRY(KD.kernel_code_properties,
-                       KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, Val,
+                       KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, ExprVal,
                        ValRange);
       if (Val)
         ImpliedUserSGPRCount += 2;
@@ -5548,34 +5573,39 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
         return Error(IDRange.Start,
                      "directive is not supported with architected flat scratch",
                      IDRange);
+      EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
       PARSE_BITS_ENTRY(KD.kernel_code_properties,
-                       KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT, Val,
-                       ValRange);
+                       KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT,
+                       ExprVal, ValRange);
       if (Val)
         ImpliedUserSGPRCount += 2;
     } else if (ID == ".amdhsa_user_sgpr_private_segment_size") {
+      EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
       PARSE_BITS_ENTRY(KD.kernel_code_properties,
                        KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
-                       Val, ValRange);
+                       ExprVal, ValRange);
       if (Val)
         ImpliedUserSGPRCount += 1;
     } else if (ID == ".amdhsa_wavefront_size32") {
+      EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
       if (IVersion.Major < 10)
         return Error(IDRange.Start, "directive requires gfx10+", IDRange);
       EnableWavefrontSize32 = Val;
       PARSE_BITS_ENTRY(KD.kernel_code_properties,
-                       KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32,
-                       Val, ValRange);
+                       KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, ExprVal,
+                       ValRange);
     } else if (ID == ".amdhsa_uses_dynamic_stack") {
       PARSE_BITS_ENTRY(KD.kernel_code_properties,
-                       KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, Val, ValRange);
+                       KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, ExprVal,
+                       ValRange);
     } else if (ID == ".amdhsa_system_sgpr_private_segment_wavefront_offset") {
       if (hasArchitectedFlatScratch())
         return Error(IDRange.Start,
                      "directive is not supported with architected flat scratch",
                      IDRange);
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
-                       COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, Val, ValRange);
+                       COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
+                       ValRange);
     } else if (ID == ".amdhsa_enable_private_segment") {
       if (!hasArchitectedFlatScratch())
         return Error(
@@ -5583,42 +5613,48 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
             "directive is not supported without architected flat scratch",
             IDRange);
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
-                       COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, Val, ValRange);
+                       COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
+                       ValRange);
     } else if (ID == ".amdhsa_system_sgpr_workgroup_id_x") {
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
-                       COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, Val,
+                       COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, ExprVal,
                        ValRange);
     } else if (ID == ".amdhsa_system_sgpr_workgroup_id_y") {
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
-                       COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, Val,
+                       COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, ExprVal,
                        ValRange);
     } else if (ID == ".amdhsa_system_sgpr_workgroup_id_z") {
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
-                       COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, Val,
+                       COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, ExprVal,
                        ValRange);
     } else if (ID == ".amdhsa_system_sgpr_workgroup_info") {
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
-                       COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, Val,
+                       COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, ExprVal,
                        ValRange);
     } else if (ID == ".amdhsa_system_vgpr_workitem_id") {
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
-                       COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, Val,
+                       COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, ExprVal,
                        ValRange);
     } else if (ID == ".amdhsa_next_free_vgpr") {
+      EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
       VGPRRange = ValRange;
       NextFreeVGPR = Val;
     } else if (ID == ".amdhsa_next_free_sgpr") {
+      EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
       SGPRRange = ValRange;
       NextFreeSGPR = Val;
     } else if (ID == ".amdhsa_accum_offset") {
       if (!isGFX90A())
         return Error(IDRange.Start, "directive requires gfx90a+", IDRange);
+      EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
       AccumOffset = Val;
     } else if (ID == ".amdhsa_reserve_vcc") {
+      EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
       if (!isUInt<1>(Val))
         return OutOfRangeError(ValRange);
       ReserveVCC = Val;
     } else if (ID == ".amdhsa_reserve_flat_scratch") {
+      EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
       if (IVersion.Major < 7)
         return Error(IDRange.Start, "directive requires gfx7+", IDRange);
       if (hasArchitectedFlatScratch())
@@ -5638,97 +5674,105 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
                                  IDRange);
     } else if (ID == ".amdhsa_float_round_mode_32") {
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
-                       COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, Val, ValRange);
+                       COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, ExprVal,
+                       ValRange);
     } else if (ID == ".amdhsa_float_round_mode_16_64") {
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
-                       COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, Val, ValRange);
+                       COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, ExprVal,
+                       ValRange);
     } else if (ID == ".amdhsa_float_denorm_mode_32") {
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
-                       COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, Val, ValRange);
+                       COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, ExprVal,
+                       ValRange);
     } else if (ID == ".amdhsa_float_denorm_mode_16_64") {
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
-                       COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, Val,
+                       COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, ExprVal,
                        ValRange);
     } else if (ID == ".amdhsa_dx10_clamp") {
       if (IVersion.Major >= 12)
         return Error(IDRange.Start, "directive unsupported on gfx12+", IDRange);
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
-                       COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, Val,
+                       COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, ExprVal,
                        ValRange);
     } else if (ID == ".amdhsa_ieee_mode") {
       if (IVersion.Major >= 12)
         return Error(IDRange.Start, "directive unsupported on gfx12+", IDRange);
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
-                       COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, Val,
+                       COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, ExprVal,
                        ValRange);
     } else if (ID == ".amdhsa_fp16_overflow") {
       if (IVersion.Major < 9)
         return Error(IDRange.Start, "directive requires gfx9+", IDRange);
-      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, Val,
+      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
+                       COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, ExprVal,
                        ValRange);
     } else if (ID == ".amdhsa_tg_split") {
       if (!isGFX90A())
         return Error(IDRange.Start, "directive requires gfx90a+", IDRange);
-      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, Val,
-                       ValRange);
+      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT,
+                       ExprVal, ValRange);
     } else if (ID == ".amdhsa_workgroup_processor_mode") {
       if (IVersion.Major < 10)
         return Error(IDRange.Start, "directive requires gfx10+", IDRange);
-      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, Val,
+      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
+                       COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal,
                        ValRange);
     } else if (ID == ".amdhsa_memory_ordered") {
       if (IVersion.Major < 10)
         return Error(IDRange.Start, "directive requires gfx10+", IDRange);
-      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, Val,
+      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
+                       COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, ExprVal,
                        ValRange);
     } else if (ID == ".amdhsa_forward_progress") {
       if (IVersion.Major < 10)
         return Error(IDRange.Start, "directive requires gfx10+", IDRange);
-      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, Val,
+      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
+                       COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, ExprVal,
                        ValRange);
     } else if (ID == ".amdhsa_shared_vgpr_count") {
+      EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
       if (IVersion.Major < 10 || IVersion.Major >= 12)
         return Error(IDRange.Start, "directive requires gfx10 or gfx11",
                      IDRange);
       SharedVGPRCount = Val;
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3,
-                       COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, Val,
+                       COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal,
                        ValRange);
     } else if (ID == ".amdhsa_exception_fp_ieee_invalid_op") {
       PARSE_BITS_ENTRY(
           KD.compute_pgm_rsrc2,
-          COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION, Val,
-          ValRange);
+          COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION,
+          ExprVal, ValRange);
     } else if (ID == ".amdhsa_exception_fp_denorm_src") {
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
                        COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
-                       Val, ValRange);
+                       ExprVal, ValRange);
     } else if (ID == ".amdhsa_exception_fp_ieee_div_zero") {
       PARSE_BITS_ENTRY(
           KD.compute_pgm_rsrc2,
-          COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO, Val,
-          ValRange);
+          COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO,
+          ExprVal, ValRange);
     } else if (ID == ".amdhsa_exception_fp_ieee_overflow") {
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
                        COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
-                       Val, ValRange);
+                       ExprVal, ValRange);
     } else if (ID == ".amdhsa_exception_fp_ieee_underflow") {
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
                        COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
-                       Val, ValRange);
+                       ExprVal, ValRange);
     } else if (ID == ".amdhsa_exception_fp_ieee_inexact") {
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
                        COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
-                       Val, ValRange);
+                       ExprVal, ValRange);
     } else if (ID == ".amdhsa_exception_int_div_zero") {
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
                        COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
-                       Val, ValRange);
+                       ExprVal, ValRange);
     } else if (ID == ".amdhsa_round_robin_scheduling") {
       if (IVersion.Major < 12)
         return Error(IDRange.Start, "directive requires gfx12+", IDRange);
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
-                       COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, Val,
+                       COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, ExprVal,
                        ValRange);
     } else {
       return Error(IDRange.Start, "unknown .amdhsa_kernel directive", IDRange);
@@ -5755,15 +5799,18 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
   if (!isUInt<COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_WIDTH>(
           VGPRBlocks))
     return OutOfRangeError(VGPRRange);
-  AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
-                  COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, VGPRBlocks);
+  AMDGPU::MCKernelDescriptor::bits_set(
+      KD.compute_pgm_rsrc1, MCConstantExpr::create(VGPRBlocks, getContext()),
+      COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT,
+      COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, getContext());
 
   if (!isUInt<COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_WIDTH>(
           SGPRBlocks))
     return OutOfRangeError(SGPRRange);
-  AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
-                  COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT,
-                  SGPRBlocks);
+  AMDGPU::MCKernelDescriptor::bits_set(
+      KD.compute_pgm_rsrc1, MCConstantExpr::create(SGPRBlocks, getContext()),
+      COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT,
+      COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, getContext());
 
   if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount)
     return TokError("amdgpu_user_sgpr_count smaller than than implied by "
@@ -5774,11 +5821,17 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
 
   if (!isUInt<COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_WIDTH>(UserSGPRCount))
     return TokError("too many user SGPRs enabled");
-  AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_USER_SGPR_COUNT,
-                  UserSGPRCount);
-
-  if (PreloadLength && KD.kernarg_size &&
-      (PreloadLength * 4 + PreloadOffset * 4 > KD.kernarg_size))
+  AMDGPU::MCKernelDescriptor::bits_set(
+      KD.compute_pgm_rsrc2, MCConstantExpr::create(UserSGPRCount, getContext()),
+      COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_SHIFT,
+      COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, getContext());
+
+  int64_t IVal = 0;
+  if (!KD.kernarg_size->evaluateAsAbsolute(IVal))
+    return TokError("Kernarg size should be resolvable");
+  uint64_t kernarg_size = IVal;
+  if (PreloadLength && kernarg_size &&
+      (PreloadLength * 4 + PreloadOffset * 4 > kernarg_size))
     return TokError("Kernarg preload length + offset is larger than the "
                     "kernarg segment size");
 
@@ -5790,8 +5843,11 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
                       "increments of 4");
     if (AccumOffset > alignTo(std::max((uint64_t)1, NextFreeVGPR), 4))
       return TokError("accum_offset exceeds total VGPR allocation");
-    AMDHSA_BITS_SET(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
-                    (AccumOffset / 4 - 1));
+    MCKernelDescriptor::bits_set(
+        KD.compute_pgm_rsrc3,
+        MCConstantExpr::create(AccumOffset / 4 - 1, getContext()),
+        COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
+        COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, getContext());
   }
 
   if (IVersion.Major >= 10 && IVersion.Major < 12) {
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp
new file mode 100644
index 000000000000..77e7e30ff528
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp
@@ -0,0 +1,98 @@
+//===--- AMDHSAKernelDescriptor.h -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUMCKernelDescriptor.h"
+#include "AMDGPUMCTargetDesc.h"
+#include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/TargetParser/TargetParser.h"
+
+using namespace llvm;
+using namespace llvm::AMDGPU;
+
+MCKernelDescriptor
+MCKernelDescriptor::getDefaultAmdhsaKernelDescriptor(const MCSubtargetInfo *STI,
+                                                     MCContext &Ctx) {
+  IsaVersion Version = getIsaVersion(STI->getCPU());
+
+  MCKernelDescriptor KD;
+  const MCExpr *ZeroMCExpr = MCConstantExpr::create(0, Ctx);
+  const MCExpr *OneMCExpr = MCConstantExpr::create(1, Ctx);
+
+  KD.group_segment_fixed_size = ZeroMCExpr;
+  KD.private_segment_fixed_size = ZeroMCExpr;
+  KD.compute_pgm_rsrc1 = ZeroMCExpr;
+  KD.compute_pgm_rsrc2 = ZeroMCExpr;
+  KD.compute_pgm_rsrc3 = ZeroMCExpr;
+  KD.kernarg_size = ZeroMCExpr;
+  KD.kernel_code_properties = ZeroMCExpr;
+  KD.kernarg_preload = ZeroMCExpr;
+
+  MCKernelDescriptor::bits_set(
+      KD.compute_pgm_rsrc1,
+      MCConstantExpr::create(amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE, Ctx),
+      amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64_SHIFT,
+      amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, Ctx);
+  if (Version.Major < 12) {
+    MCKernelDescriptor::bits_set(
+        KD.compute_pgm_rsrc1, OneMCExpr,
+        amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP_SHIFT,
+        amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, Ctx);
+    MCKernelDescriptor::bits_set(
+        KD.compute_pgm_rsrc1, OneMCExpr,
+        amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE_SHIFT,
+        amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, Ctx);
+  }
+  MCKernelDescriptor::bits_set(
+      KD.compute_pgm_rsrc2, OneMCExpr,
+      amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X_SHIFT,
+      amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, Ctx);
+  if (Version.Major >= 10) {
+    if (STI->getFeatureBits().test(FeatureWavefrontSize32))
+      MCKernelDescriptor::bits_set(
+          KD.kernel_code_properties, OneMCExpr,
+          amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32_SHIFT,
+          amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, Ctx);
+    if (!STI->getFeatureBits().test(FeatureCuMode))
+      MCKernelDescriptor::bits_set(
+          KD.compute_pgm_rsrc1, OneMCExpr,
+          amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE_SHIFT,
+          amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, Ctx);
+
+    MCKernelDescriptor::bits_set(
+        KD.compute_pgm_rsrc1, OneMCExpr,
+        amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED_SHIFT,
+        amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, Ctx);
+  }
+  if (AMDGPU::isGFX90A(*STI) && STI->getFeatureBits().test(FeatureTgSplit))
+    MCKernelDescriptor::bits_set(
+        KD.compute_pgm_rsrc3, OneMCExpr,
+        amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT,
+        amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, Ctx);
+  return KD;
+}
+
+void MCKernelDescriptor::bits_set(const MCExpr *&Dst, const MCExpr *Value,
+                                  uint32_t Shift, uint32_t Mask,
+                                  MCContext &Ctx) {
+  auto Sft = MCConstantExpr::create(Shift, Ctx);
+  auto Msk = MCConstantExpr::create(Mask, Ctx);
+  Dst = MCBinaryExpr::createAnd(Dst, MCUnaryExpr::createNot(Msk, Ctx), Ctx);
+  Dst = MCBinaryExpr::createOr(Dst, MCBinaryExpr::createShl(Value, Sft, Ctx),
+                               Ctx);
+}
+
+const MCExpr *MCKernelDescriptor::bits_get(const MCExpr *Src, uint32_t Shift,
+                                           uint32_t Mask, MCContext &Ctx) {
+  auto Sft = MCConstantExpr::create(Shift, Ctx);
+  auto Msk = MCConstantExpr::create(Mask, Ctx);
+  return MCBinaryExpr::createLShr(MCBinaryExpr::createAnd(Src, Msk, Ctx), Sft,
+                                  Ctx);
+}
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h
new file mode 100644
index 000000000000..26958ac8b9ee
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h
@@ -0,0 +1,54 @@
+//===--- AMDGPUMCKernelDescriptor.h ---------------------------*- C++ -*---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// AMDHSA kernel descriptor MCExpr struct for use in MC layer. Uses
+/// AMDHSAKernelDescriptor.h for sizes and constants.
+///
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCKERNELDESCRIPTOR_H
+#define LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCKERNELDESCRIPTOR_H
+
+#include "llvm/Support/AMDHSAKernelDescriptor.h"
+
+namespace llvm {
+class MCExpr;
+class MCContext;
+class MCSubtargetInfo;
+namespace AMDGPU {
+
+struct MCKernelDescriptor {
+  const MCExpr *group_segment_fixed_size = nullptr;
+  const MCExpr *private_segment_fixed_size = nullptr;
+  const MCExpr *kernarg_size = nullptr;
+  const MCExpr *compute_pgm_rsrc3 = nullptr;
+  const MCExpr *compute_pgm_rsrc1 = nullptr;
+  const MCExpr *compute_pgm_rsrc2 = nullptr;
+  const MCExpr *kernel_code_properties = nullptr;
+  const MCExpr *kernarg_preload = nullptr;
+
+  static MCKernelDescriptor
+  getDefaultAmdhsaKernelDescriptor(const MCSubtargetInfo *STI, MCContext &Ctx);
+  // MCExpr for:
+  // Dst = Dst & ~Mask
+  // Dst = Dst | (Value << Shift)
+  static void bits_set(const MCExpr *&Dst, const MCExpr *Value, uint32_t Shift,
+                       uint32_t Mask, MCContext &Ctx);
+
+  // MCExpr for:
+  // return (Src & Mask) >> Shift
+  static const MCExpr *bits_get(const MCExpr *Src, uint32_t Shift,
+                                uint32_t Mask, MCContext &Ctx);
+};
+
+} // end namespace AMDGPU
+} // end namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCKERNELDESCRIPTOR_H
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
index 4742b0b3e52e..3006fcdb9282 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPUTargetStreamer.h"
+#include "AMDGPUMCKernelDescriptor.h"
 #include "AMDGPUPTNote.h"
 #include "AMDKernelCodeT.h"
 #include "Utils/AMDGPUBaseInfo.h"
@@ -307,94 +308,142 @@ bool AMDGPUTargetAsmStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) {
 
 void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(
     const MCSubtargetInfo &STI, StringRef KernelName,
-    const amdhsa::kernel_descriptor_t &KD, uint64_t NextVGPR, uint64_t NextSGPR,
+    const MCKernelDescriptor &KD, uint64_t NextVGPR, uint64_t NextSGPR,
     bool ReserveVCC, bool ReserveFlatScr) {
   IsaVersion IVersion = getIsaVersion(STI.getCPU());
+  const MCAsmInfo *MAI = getContext().getAsmInfo();
 
   OS << "\t.amdhsa_kernel " << KernelName << '\n';
 
-#define PRINT_FIELD(STREAM, DIRECTIVE, KERNEL_DESC, MEMBER_NAME, FIELD_NAME)   \
-  STREAM << "\t\t" << DIRECTIVE << " "                                         \
-         << AMDHSA_BITS_GET(KERNEL_DESC.MEMBER_NAME, FIELD_NAME) << '\n';
-
-  OS << "\t\t.amdhsa_group_segment_fixed_size " << KD.group_segment_fixed_size
-     << '\n';
-  OS << "\t\t.amdhsa_private_segment_fixed_size "
-     << KD.private_segment_fixed_size << '\n';
-  OS << "\t\t.amdhsa_kernarg_size " << KD.kernarg_size << '\n';
-
-  PRINT_FIELD(OS, ".amdhsa_user_sgpr_count", KD,
-              compute_pgm_rsrc2,
-              amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT);
+  auto PrintField = [&](const MCExpr *Expr, uint32_t Shift, uint32_t Mask,
+                        StringRef Directive) {
+    int64_t IVal;
+    OS << "\t\t" << Directive << ' ';
+    const MCExpr *pgm_rsrc1_bits =
+        MCKernelDescriptor::bits_get(Expr, Shift, Mask, getContext());
+    if (pgm_rsrc1_bits->evaluateAsAbsolute(IVal))
+      OS << static_cast<uint64_t>(IVal);
+    else
+      pgm_rsrc1_bits->print(OS, MAI);
+    OS << '\n';
+  };
+
+  OS << "\t\t.amdhsa_group_segment_fixed_size ";
+  KD.group_segment_fixed_size->print(OS, MAI);
+  OS << '\n';
+
+  OS << "\t\t.amdhsa_private_segment_fixed_size ";
+  KD.private_segment_fixed_size->print(OS, MAI);
+  OS << '\n';
+
+  OS << "\t\t.amdhsa_kernarg_size ";
+  KD.kernarg_size->print(OS, MAI);
+  OS << '\n';
+
+  PrintField(
+      KD.compute_pgm_rsrc2, amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_SHIFT,
+      amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, ".amdhsa_user_sgpr_count");
 
   if (!hasArchitectedFlatScratch(STI))
-    PRINT_FIELD(
-        OS, ".amdhsa_user_sgpr_private_segment_buffer", KD,
-        kernel_code_properties,
-        amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER);
-  PRINT_FIELD(OS, ".amdhsa_user_sgpr_dispatch_ptr", KD,
-              kernel_code_properties,
-              amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR);
-  PRINT_FIELD(OS, ".amdhsa_user_sgpr_queue_ptr", KD,
-              kernel_code_properties,
-              amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR);
-  PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_segment_ptr", KD,
-              kernel_code_properties,
-              amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR);
-  PRINT_FIELD(OS, ".amdhsa_user_sgpr_dispatch_id", KD,
-              kernel_code_properties,
-              amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID);
+    PrintField(
+        KD.kernel_code_properties,
+        amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER_SHIFT,
+        amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
+        ".amdhsa_user_sgpr_private_segment_buffer");
+  PrintField(KD.kernel_code_properties,
+             amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR_SHIFT,
+             amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR,
+             ".amdhsa_user_sgpr_dispatch_ptr");
+  PrintField(KD.kernel_code_properties,
+             amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR_SHIFT,
+             amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR,
+             ".amdhsa_user_sgpr_queue_ptr");
+  PrintField(KD.kernel_code_properties,
+             amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR_SHIFT,
+             amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
+             ".amdhsa_user_sgpr_kernarg_segment_ptr");
+  PrintField(KD.kernel_code_properties,
+             amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID_SHIFT,
+             amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID,
+             ".amdhsa_user_sgpr_dispatch_id");
   if (!hasArchitectedFlatScratch(STI))
-    PRINT_FIELD(OS, ".amdhsa_user_sgpr_flat_scratch_init", KD,
-                kernel_code_properties,
-                amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT);
+    PrintField(KD.kernel_code_properties,
+               amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT_SHIFT,
+               amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT,
+               ".amdhsa_user_sgpr_flat_scratch_init");
   if (hasKernargPreload(STI)) {
-    PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_preload_length ", KD,
-                kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_LENGTH);
-    PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_preload_offset ", KD,
-                kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_OFFSET);
+    PrintField(KD.kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_LENGTH_SHIFT,
+               amdhsa::KERNARG_PRELOAD_SPEC_LENGTH,
+               ".amdhsa_user_sgpr_kernarg_preload_length");
+    PrintField(KD.kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_OFFSET_SHIFT,
+               amdhsa::KERNARG_PRELOAD_SPEC_OFFSET,
+               ".amdhsa_user_sgpr_kernarg_preload_offset");
   }
-  PRINT_FIELD(OS, ".amdhsa_user_sgpr_private_segment_size", KD,
-              kernel_code_properties,
-              amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE);
+  PrintField(
+      KD.kernel_code_properties,
+      amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE_SHIFT,
+      amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
+      ".amdhsa_user_sgpr_private_segment_size");
   if (IVersion.Major >= 10)
-    PRINT_FIELD(OS, ".amdhsa_wavefront_size32", KD,
-                kernel_code_properties,
-                amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32);
+    PrintField(KD.kernel_code_properties,
+               amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32_SHIFT,
+               amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32,
+               ".amdhsa_wavefront_size32");
   if (CodeObjectVersion >= AMDGPU::AMDHSA_COV5)
-    PRINT_FIELD(OS, ".amdhsa_uses_dynamic_stack", KD, kernel_code_properties,
-                amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK);
-  PRINT_FIELD(OS,
-              (hasArchitectedFlatScratch(STI)
-                   ? ".amdhsa_enable_private_segment"
-                   : ".amdhsa_system_sgpr_private_segment_wavefront_offset"),
-              KD, compute_pgm_rsrc2,
-              amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT);
-  PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_x", KD,
-              compute_pgm_rsrc2,
-              amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X);
-  PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_y", KD,
-              compute_pgm_rsrc2,
-              amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y);
-  PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_z", KD,
-              compute_pgm_rsrc2,
-              amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z);
-  PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_info", KD,
-              compute_pgm_rsrc2,
-              amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO);
-  PRINT_FIELD(OS, ".amdhsa_system_vgpr_workitem_id", KD,
-              compute_pgm_rsrc2,
-              amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID);
+    PrintField(KD.kernel_code_properties,
+               amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK_SHIFT,
+               amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK,
+               ".amdhsa_uses_dynamic_stack");
+  PrintField(KD.compute_pgm_rsrc2,
+             amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT_SHIFT,
+             amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT,
+             (hasArchitectedFlatScratch(STI)
+                  ? ".amdhsa_enable_private_segment"
+                  : ".amdhsa_system_sgpr_private_segment_wavefront_offset"));
+  PrintField(KD.compute_pgm_rsrc2,
+             amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X_SHIFT,
+             amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X,
+             ".amdhsa_system_sgpr_workgroup_id_x");
+  PrintField(KD.compute_pgm_rsrc2,
+             amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y_SHIFT,
+             amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y,
+             ".amdhsa_system_sgpr_workgroup_id_y");
+  PrintField(KD.compute_pgm_rsrc2,
+             amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z_SHIFT,
+             amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z,
+             ".amdhsa_system_sgpr_workgroup_id_z");
+  PrintField(KD.compute_pgm_rsrc2,
+             amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO_SHIFT,
+             amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO,
+             ".amdhsa_system_sgpr_workgroup_info");
+  PrintField(KD.compute_pgm_rsrc2,
+             amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID_SHIFT,
+             amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID,
+             ".amdhsa_system_vgpr_workitem_id");
 
   // These directives are required.
   OS << "\t\t.amdhsa_next_free_vgpr " << NextVGPR << '\n';
   OS << "\t\t.amdhsa_next_free_sgpr " << NextSGPR << '\n';
 
-  if (AMDGPU::isGFX90A(STI))
-    OS << "\t\t.amdhsa_accum_offset " <<
-      (AMDHSA_BITS_GET(KD.compute_pgm_rsrc3,
-                       amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET) + 1) * 4
-      << '\n';
+  if (AMDGPU::isGFX90A(STI)) {
+    // MCExpr equivalent of taking the (accum_offset + 1) * 4.
+    const MCExpr *accum_bits = MCKernelDescriptor::bits_get(
+        KD.compute_pgm_rsrc3,
+        amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
+        amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, getContext());
+    accum_bits = MCBinaryExpr::createAdd(
+        accum_bits, MCConstantExpr::create(1, getContext()), getContext());
+    accum_bits = MCBinaryExpr::createMul(
+        accum_bits, MCConstantExpr::create(4, getContext()), getContext());
+    OS << "\t\t.amdhsa_accum_offset ";
+    int64_t IVal;
+    if (accum_bits->evaluateAsAbsolute(IVal)) {
+      OS << static_cast<uint64_t>(IVal);
+    } else {
+      accum_bits->print(OS, MAI);
+    }
+    OS << '\n';
+  }
 
   if (!ReserveVCC)
     OS << "\t\t.amdhsa_reserve_vcc " << ReserveVCC << '\n';
@@ -411,74 +460,105 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(
     break;
   }
 
-  PRINT_FIELD(OS, ".amdhsa_float_round_mode_32", KD,
-              compute_pgm_rsrc1,
-              amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32);
-  PRINT_FIELD(OS, ".amdhsa_float_round_mode_16_64", KD,
-              compute_pgm_rsrc1,
-              amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64);
-  PRINT_FIELD(OS, ".amdhsa_float_denorm_mode_32", KD,
-              compute_pgm_rsrc1,
-              amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32);
-  PRINT_FIELD(OS, ".amdhsa_float_denorm_mode_16_64", KD,
-              compute_pgm_rsrc1,
-              amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64);
+  PrintField(KD.compute_pgm_rsrc1,
+             amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32_SHIFT,
+             amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32,
+             ".amdhsa_float_round_mode_32");
+  PrintField(KD.compute_pgm_rsrc1,
+             amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64_SHIFT,
+             amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64,
+             ".amdhsa_float_round_mode_16_64");
+  PrintField(KD.compute_pgm_rsrc1,
+             amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32_SHIFT,
+             amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32,
+             ".amdhsa_float_denorm_mode_32");
+  PrintField(KD.compute_pgm_rsrc1,
+             amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64_SHIFT,
+             amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64,
+             ".amdhsa_float_denorm_mode_16_64");
   if (IVersion.Major < 12) {
-    PRINT_FIELD(OS, ".amdhsa_dx10_clamp", KD, compute_pgm_rsrc1,
-                amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP);
-    PRINT_FIELD(OS, ".amdhsa_ieee_mode", KD, compute_pgm_rsrc1,
-                amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE);
+    PrintField(KD.compute_pgm_rsrc1,
+               amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP_SHIFT,
+               amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP,
+               ".amdhsa_dx10_clamp");
+    PrintField(KD.compute_pgm_rsrc1,
+               amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE_SHIFT,
+               amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE,
+               ".amdhsa_ieee_mode");
+  }
+  if (IVersion.Major >= 9) {
+    PrintField(KD.compute_pgm_rsrc1,
+               amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL_SHIFT,
+               amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL,
+               ".amdhsa_fp16_overflow");
   }
-  if (IVersion.Major >= 9)
-    PRINT_FIELD(OS, ".amdhsa_fp16_overflow", KD,
-                compute_pgm_rsrc1,
-                amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL);
   if (AMDGPU::isGFX90A(STI))
-    PRINT_FIELD(OS, ".amdhsa_tg_split", KD,
-                compute_pgm_rsrc3,
-                amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT);
+    PrintField(KD.compute_pgm_rsrc3,
+               amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT,
+               amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, ".amdhsa_tg_split");
   if (IVersion.Major >= 10) {
-    PRINT_FIELD(OS, ".amdhsa_workgroup_processor_mode", KD,
-                compute_pgm_rsrc1,
-                amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE);
-    PRINT_FIELD(OS, ".amdhsa_memory_ordered", KD,
-                compute_pgm_rsrc1,
-                amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED);
-    PRINT_FIELD(OS, ".amdhsa_forward_progress", KD,
-                compute_pgm_rsrc1,
-                amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS);
+    PrintField(KD.compute_pgm_rsrc1,
+               amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE_SHIFT,
+               amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE,
+               ".amdhsa_workgroup_processor_mode");
+    PrintField(KD.compute_pgm_rsrc1,
+               amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED_SHIFT,
+               amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED,
+               ".amdhsa_memory_ordered");
+    PrintField(KD.compute_pgm_rsrc1,
+               amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS_SHIFT,
+               amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS,
+               ".amdhsa_forward_progress");
   }
   if (IVersion.Major >= 10 && IVersion.Major < 12) {
-    PRINT_FIELD(OS, ".amdhsa_shared_vgpr_count", KD, compute_pgm_rsrc3,
-                amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT);
+    PrintField(KD.compute_pgm_rsrc3,
+               amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT_SHIFT,
+               amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT,
+               ".amdhsa_shared_vgpr_count");
   }
-  if (IVersion.Major >= 12)
-    PRINT_FIELD(OS, ".amdhsa_round_robin_scheduling", KD, compute_pgm_rsrc1,
-                amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN);
-  PRINT_FIELD(
-      OS, ".amdhsa_exception_fp_ieee_invalid_op", KD,
-      compute_pgm_rsrc2,
-      amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION);
-  PRINT_FIELD(OS, ".amdhsa_exception_fp_denorm_src", KD,
-              compute_pgm_rsrc2,
-              amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE);
-  PRINT_FIELD(
-      OS, ".amdhsa_exception_fp_ieee_div_zero", KD,
-      compute_pgm_rsrc2,
-      amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO);
-  PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_overflow", KD,
-              compute_pgm_rsrc2,
-              amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW);
-  PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_underflow", KD,
-              compute_pgm_rsrc2,
-              amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW);
-  PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_inexact", KD,
-              compute_pgm_rsrc2,
-              amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT);
-  PRINT_FIELD(OS, ".amdhsa_exception_int_div_zero", KD,
-              compute_pgm_rsrc2,
-              amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO);
-#undef PRINT_FIELD
+  if (IVersion.Major >= 12) {
+    PrintField(KD.compute_pgm_rsrc1,
+               amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN_SHIFT,
+               amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN,
+               ".amdhsa_round_robin_scheduling");
+  }
+  PrintField(
+      KD.compute_pgm_rsrc2,
+      amdhsa::
+          COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION_SHIFT,
+      amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION,
+      ".amdhsa_exception_fp_ieee_invalid_op");
+  PrintField(
+      KD.compute_pgm_rsrc2,
+      amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE_SHIFT,
+      amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
+      ".amdhsa_exception_fp_denorm_src");
+  PrintField(
+      KD.compute_pgm_rsrc2,
+      amdhsa::
+          COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO_SHIFT,
+      amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO,
+      ".amdhsa_exception_fp_ieee_div_zero");
+  PrintField(
+      KD.compute_pgm_rsrc2,
+      amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW_SHIFT,
+      amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
+      ".amdhsa_exception_fp_ieee_overflow");
+  PrintField(
+      KD.compute_pgm_rsrc2,
+      amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW_SHIFT,
+      amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
+      ".amdhsa_exception_fp_ieee_underflow");
+  PrintField(
+      KD.compute_pgm_rsrc2,
+      amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT_SHIFT,
+      amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
+      ".amdhsa_exception_fp_ieee_inexact");
+  PrintField(
+      KD.compute_pgm_rsrc2,
+      amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO_SHIFT,
+      amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
+      ".amdhsa_exception_int_div_zero");
 
   OS << "\t.end_amdhsa_kernel\n";
 }
@@ -835,7 +915,7 @@ bool AMDGPUTargetELFStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) {
 
 void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor(
     const MCSubtargetInfo &STI, StringRef KernelName,
-    const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR,
+    const MCKernelDescriptor &KernelDescriptor, uint64_t NextVGPR,
     uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) {
   auto &Streamer = getStreamer();
   auto &Context = Streamer.getContext();
@@ -853,7 +933,7 @@ void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor(
   // Kernel descriptor symbol's type and size are fixed.
   KernelDescriptorSymbol->setType(ELF::STT_OBJECT);
   KernelDescriptorSymbol->setSize(
-      MCConstantExpr::create(sizeof(KernelDescriptor), Context));
+      MCConstantExpr::create(sizeof(amdhsa::kernel_descriptor_t), Context));
 
   // The visibility of the kernel code symbol must be protected or less to allow
   // static relocations from the kernel descriptor to be used.
@@ -861,31 +941,43 @@ void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor(
     KernelCodeSymbol->setVisibility(ELF::STV_PROTECTED);
 
   Streamer.emitLabel(KernelDescriptorSymbol);
-  Streamer.emitInt32(KernelDescriptor.group_segment_fixed_size);
-  Streamer.emitInt32(KernelDescriptor.private_segment_fixed_size);
-  Streamer.emitInt32(KernelDescriptor.kernarg_size);
-
-  for (uint8_t Res : KernelDescriptor.reserved0)
-    Streamer.emitInt8(Res);
+  Streamer.emitValue(
+      KernelDescriptor.group_segment_fixed_size,
+      sizeof(amdhsa::kernel_descriptor_t::group_segment_fixed_size));
+  Streamer.emitValue(
+      KernelDescriptor.private_segment_fixed_size,
+      sizeof(amdhsa::kernel_descriptor_t::private_segment_fixed_size));
+  Streamer.emitValue(KernelDescriptor.kernarg_size,
+                     sizeof(amdhsa::kernel_descriptor_t::kernarg_size));
+
+  for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved0); ++i)
+    Streamer.emitInt8(0u);
 
   // FIXME: Remove the use of VK_AMDGPU_REL64 in the expression below. The
   // expression being created is:
   //   (start of kernel code) - (start of kernel descriptor)
   // It implies R_AMDGPU_REL64, but ends up being R_AMDGPU_ABS64.
-  Streamer.emitValue(MCBinaryExpr::createSub(
-      MCSymbolRefExpr::create(
-          KernelCodeSymbol, MCSymbolRefExpr::VK_AMDGPU_REL64, Context),
-      MCSymbolRefExpr::create(
-          KernelDescriptorSymbol, MCSymbolRefExpr::VK_None, Context),
-      Context),
-      sizeof(KernelDescriptor.kernel_code_entry_byte_offset));
-  for (uint8_t Res : KernelDescriptor.reserved1)
-    Streamer.emitInt8(Res);
-  Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc3);
-  Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc1);
-  Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc2);
-  Streamer.emitInt16(KernelDescriptor.kernel_code_properties);
-  Streamer.emitInt16(KernelDescriptor.kernarg_preload);
-  for (uint8_t Res : KernelDescriptor.reserved3)
-    Streamer.emitInt8(Res);
+  Streamer.emitValue(
+      MCBinaryExpr::createSub(
+          MCSymbolRefExpr::create(KernelCodeSymbol,
+                                  MCSymbolRefExpr::VK_AMDGPU_REL64, Context),
+          MCSymbolRefExpr::create(KernelDescriptorSymbol,
+                                  MCSymbolRefExpr::VK_None, Context),
+          Context),
+      sizeof(amdhsa::kernel_descriptor_t::kernel_code_entry_byte_offset));
+  for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved1); ++i)
+    Streamer.emitInt8(0u);
+  Streamer.emitValue(KernelDescriptor.compute_pgm_rsrc3,
+                     sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc3));
+  Streamer.emitValue(KernelDescriptor.compute_pgm_rsrc1,
+                     sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc1));
+  Streamer.emitValue(KernelDescriptor.compute_pgm_rsrc2,
+                     sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc2));
+  Streamer.emitValue(
+      KernelDescriptor.kernel_code_properties,
+      sizeof(amdhsa::kernel_descriptor_t::kernel_code_properties));
+  Streamer.emitValue(KernelDescriptor.kernarg_preload,
+                     sizeof(amdhsa::kernel_descriptor_t::kernarg_preload));
+  for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved3); ++i)
+    Streamer.emitInt8(0u);
 }
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
index 5aa80ff578c6..706897a5dc1f 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
@@ -22,15 +22,13 @@ class MCSymbol;
 class formatted_raw_ostream;
 
 namespace AMDGPU {
+
+struct MCKernelDescriptor;
 namespace HSAMD {
 struct Metadata;
 }
 } // namespace AMDGPU
 
-namespace amdhsa {
-struct kernel_descriptor_t;
-}
-
 class AMDGPUTargetStreamer : public MCTargetStreamer {
   AMDGPUPALMetadata PALMetadata;
 
@@ -94,10 +92,11 @@ public:
     return true;
   }
 
-  virtual void EmitAmdhsaKernelDescriptor(
-      const MCSubtargetInfo &STI, StringRef KernelName,
-      const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR,
-      uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) {}
+  virtual void
+  EmitAmdhsaKernelDescriptor(const MCSubtargetInfo &STI, StringRef KernelName,
+                             const AMDGPU::MCKernelDescriptor &KernelDescriptor,
+                             uint64_t NextVGPR, uint64_t NextSGPR,
+                             bool ReserveVCC, bool ReserveFlatScr) {}
 
   static StringRef getArchNameFromElfMach(unsigned ElfMach);
   static unsigned getElfMach(StringRef GPU);
@@ -150,10 +149,11 @@ public:
   bool EmitKernargPreloadHeader(const MCSubtargetInfo &STI,
                                 bool TrapEnabled) override;
 
-  void EmitAmdhsaKernelDescriptor(
-      const MCSubtargetInfo &STI, StringRef KernelName,
-      const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR,
-      uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) override;
+  void
+  EmitAmdhsaKernelDescriptor(const MCSubtargetInfo &STI, StringRef KernelName,
+                             const AMDGPU::MCKernelDescriptor &KernelDescriptor,
+                             uint64_t NextVGPR, uint64_t NextSGPR,
+                             bool ReserveVCC, bool ReserveFlatScr) override;
 };
 
 class AMDGPUTargetELFStreamer final : public AMDGPUTargetStreamer {
@@ -205,10 +205,11 @@ public:
   bool EmitKernargPreloadHeader(const MCSubtargetInfo &STI,
                                 bool TrapEnabled) override;
 
-  void EmitAmdhsaKernelDescriptor(
-      const MCSubtargetInfo &STI, StringRef KernelName,
-      const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR,
-      uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) override;
+  void
+  EmitAmdhsaKernelDescriptor(const MCSubtargetInfo &STI, StringRef KernelName,
+                             const AMDGPU::MCKernelDescriptor &KernelDescriptor,
+                             uint64_t NextVGPR, uint64_t NextSGPR,
+                             bool ReserveVCC, bool ReserveFlatScr) override;
 };
 }
 #endif
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt
index 0842a58f794b..14a02b6d8e36 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt
@@ -8,6 +8,7 @@ add_llvm_component_library(LLVMAMDGPUDesc
   AMDGPUMCExpr.cpp
   AMDGPUMCTargetDesc.cpp
   AMDGPUTargetStreamer.cpp
+  AMDGPUMCKernelDescriptor.cpp
   R600InstPrinter.cpp
   R600MCCodeEmitter.cpp
   R600MCTargetDesc.cpp
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 7bb84d78442b..5d44396b07b6 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -1221,47 +1221,6 @@ void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header,
   }
 }
 
-amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor(
-    const MCSubtargetInfo *STI) {
-  IsaVersion Version = getIsaVersion(STI->getCPU());
-
-  amdhsa::kernel_descriptor_t KD;
-  memset(&KD, 0, sizeof(KD));
-
-  AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
-                  amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64,
-                  amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE);
-  if (Version.Major >= 12) {
-    AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
-                    amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, 0);
-    AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
-                    amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_DISABLE_PERF, 0);
-  } else {
-    AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
-                    amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, 1);
-    AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
-                    amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, 1);
-  }
-  AMDHSA_BITS_SET(KD.compute_pgm_rsrc2,
-                  amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1);
-  if (Version.Major >= 10) {
-    AMDHSA_BITS_SET(KD.kernel_code_properties,
-                    amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32,
-                    STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0);
-    AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
-                    amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE,
-                    STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1);
-    AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
-                    amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, 1);
-  }
-  if (AMDGPU::isGFX90A(*STI)) {
-    AMDHSA_BITS_SET(KD.compute_pgm_rsrc3,
-                    amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT,
-                    STI->getFeatureBits().test(FeatureTgSplit) ? 1 : 0);
-  }
-  return KD;
-}
-
 bool isGroupSegment(const GlobalValue *GV) {
   return GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
 }
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index f4f9a787100b..943588fe701c 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -34,10 +34,6 @@ class StringRef;
 class Triple;
 class raw_ostream;
 
-namespace amdhsa {
-struct kernel_descriptor_t;
-}
-
 namespace AMDGPU {
 
 struct IsaVersion;
@@ -855,9 +851,6 @@ unsigned mapWMMA3AddrTo2AddrOpcode(unsigned Opc);
 void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header,
                                const MCSubtargetInfo *STI);
 
-amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor(
-    const MCSubtargetInfo *STI);
-
 bool isGroupSegment(const GlobalValue *GV);
 bool isGlobalSegment(const GlobalValue *GV);
 bool isReadOnlySegment(const GlobalValue *GV);
diff --git a/llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s b/llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s
new file mode 100644
index 000000000000..4623500987be
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s
@@ -0,0 +1,27 @@
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck --check-prefix=ASM %s
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -filetype=obj < %s > %t
+// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
+
+// OBJDUMP:       0000 00000000 0f000000 00000000 00000000
+
+.text
+
+.p2align 8
+.type caller,@function
+caller:
+  s_endpgm
+
+.rodata
+
+.p2align 6
+.amdhsa_kernel caller
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset 4
+  .amdhsa_private_segment_fixed_size max(7, callee1.private_seg_size, callee2.private_seg_size)
+.end_amdhsa_kernel
+
+.set callee1.private_seg_size, 4
+.set callee2.private_seg_size, 15
+
+// ASM: .amdhsa_private_segment_fixed_size max(7, callee1.private_seg_size, callee2.private_seg_size)
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s b/llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s
new file mode 100644
index 000000000000..fab3e893352b
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s
@@ -0,0 +1,281 @@
+// RUN: not llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a %s 2>&1 | FileCheck --check-prefix=ASM %s
+
+// Some expression currently require (immediately) solvable expressions, i.e.,
+// they don't depend on yet-unknown symbolic values.
+
+.text
+// ASM: .text
+
+.amdhsa_code_object_version 4
+// ASM: .amdhsa_code_object_version 4
+
+.p2align 8
+.type user_sgpr_count,@function
+user_sgpr_count:
+  s_endpgm
+
+.p2align 6
+.amdhsa_kernel user_sgpr_count
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset 4
+  .amdhsa_user_sgpr_count defined_boolean
+.end_amdhsa_kernel
+
+// ASM: error: directive should have resolvable expression
+// ASM-NEXT:   .amdhsa_user_sgpr_count
+
+.p2align 8
+.type user_sgpr_private_segment_buffer,@function
+user_sgpr_private_segment_buffer:
+  s_endpgm
+
+.amdhsa_kernel user_sgpr_private_segment_buffer
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset 4
+  .amdhsa_user_sgpr_private_segment_buffer defined_boolean
+.end_amdhsa_kernel
+
+// ASM: error: directive should have resolvable expression
+// ASM-NEXT:   .amdhsa_user_sgpr_private_segment_buffer
+
+.p2align 8
+.type user_sgpr_kernarg_preload_length,@function
+user_sgpr_kernarg_preload_length:
+  s_endpgm
+
+.amdhsa_kernel user_sgpr_kernarg_preload_length
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset 4
+  .amdhsa_user_sgpr_kernarg_preload_length defined_boolean
+.end_amdhsa_kernel
+
+// ASM: error: directive should have resolvable expression
+// ASM-NEXT:   .amdhsa_user_sgpr_kernarg_preload_length defined_boolean
+
+.p2align 8
+.type user_sgpr_kernarg_preload_offset,@function
+user_sgpr_kernarg_preload_offset:
+  s_endpgm
+
+.amdhsa_kernel user_sgpr_kernarg_preload_offset
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset 4
+  .amdhsa_user_sgpr_kernarg_preload_offset defined_boolean
+.end_amdhsa_kernel
+
+// ASM: error: directive should have resolvable expression
+// ASM-NEXT:   .amdhsa_user_sgpr_kernarg_preload_offset defined_boolean
+
+.p2align 8
+.type user_sgpr_dispatch_ptr,@function
+user_sgpr_dispatch_ptr:
+  s_endpgm
+
+.p2align 6
+.amdhsa_kernel user_sgpr_dispatch_ptr
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset 4
+  .amdhsa_user_sgpr_dispatch_ptr defined_boolean
+.end_amdhsa_kernel
+
+// ASM: error: directive should have resolvable expression
+// ASM-NEXT:   .amdhsa_user_sgpr_dispatch_ptr
+
+.p2align 8
+.type user_sgpr_queue_ptr,@function
+user_sgpr_queue_ptr:
+  s_endpgm
+
+.p2align 6
+.amdhsa_kernel user_sgpr_queue_ptr
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset 4
+  .amdhsa_user_sgpr_queue_ptr defined_boolean
+.end_amdhsa_kernel
+
+// ASM: error: directive should have resolvable expression
+// ASM-NEXT:   .amdhsa_user_sgpr_queue_ptr
+
+.p2align 8
+.type user_sgpr_kernarg_segment_ptr,@function
+user_sgpr_kernarg_segment_ptr:
+  s_endpgm
+
+.p2align 6
+.amdhsa_kernel user_sgpr_kernarg_segment_ptr
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset 4
+  .amdhsa_user_sgpr_kernarg_segment_ptr defined_boolean
+.end_amdhsa_kernel
+
+// ASM: error: directive should have resolvable expression
+// ASM-NEXT:   .amdhsa_user_sgpr_kernarg_segment_ptr
+
+.p2align 8
+.type user_sgpr_dispatch_id,@function
+user_sgpr_dispatch_id:
+  s_endpgm
+
+.p2align 6
+.amdhsa_kernel user_sgpr_dispatch_id
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset 4
+  .amdhsa_user_sgpr_dispatch_id defined_boolean
+.end_amdhsa_kernel
+
+// ASM: error: directive should have resolvable expression
+// ASM-NEXT:   .amdhsa_user_sgpr_dispatch_id
+
+.p2align 8
+.type user_sgpr_flat_scratch_init,@function
+user_sgpr_flat_scratch_init:
+  s_endpgm
+
+.p2align 6
+.amdhsa_kernel user_sgpr_flat_scratch_init
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset 4
+  .amdhsa_user_sgpr_flat_scratch_init defined_boolean
+.end_amdhsa_kernel
+
+// ASM: error: directive should have resolvable expression
+// ASM-NEXT:   .amdhsa_user_sgpr_flat_scratch_init
+
+.p2align 8
+.type user_sgpr_private_segment_size,@function
+user_sgpr_private_segment_size:
+  s_endpgm
+
+.p2align 6
+.amdhsa_kernel user_sgpr_private_segment_size
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset 4
+  .amdhsa_user_sgpr_private_segment_size defined_boolean
+.end_amdhsa_kernel
+
+// ASM: error: directive should have resolvable expression
+// ASM-NEXT:   .amdhsa_user_sgpr_private_segment_size
+
+.p2align 8
+.type wavefront_size32,@function
+wavefront_size32:
+  s_endpgm
+
+.p2align 6
+.amdhsa_kernel wavefront_size32
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset 4
+  .amdhsa_wavefront_size32 defined_boolean
+.end_amdhsa_kernel
+
+// ASM: error: directive should have resolvable expression
+// ASM-NEXT:   .amdhsa_wavefront_size32
+
+.p2align 8
+.type next_free_vgpr,@function
+next_free_vgpr:
+  s_endpgm
+
+.p2align 6
+.amdhsa_kernel next_free_vgpr
+  .amdhsa_next_free_vgpr defined_boolean
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset 4
+.end_amdhsa_kernel
+
+// ASM: error: directive should have resolvable expression
+// ASM-NEXT:   .amdhsa_next_free_vgpr
+
+.p2align 8
+.type next_free_sgpr,@function
+next_free_sgpr:
+  s_endpgm
+
+.p2align 6
+.amdhsa_kernel next_free_sgpr
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr defined_boolean
+  .amdhsa_accum_offset 4
+.end_amdhsa_kernel
+
+// ASM: error: directive should have resolvable expression
+// ASM-NEXT:   .amdhsa_next_free_sgpr
+
+.p2align 8
+.type accum_offset,@function
+accum_offset:
+  s_endpgm
+
+.p2align 6
+.amdhsa_kernel accum_offset
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset defined_boolean
+.end_amdhsa_kernel
+
+// ASM: error: directive should have resolvable expression
+// ASM-NEXT:   .amdhsa_accum_offset
+
+.p2align 8
+.type reserve_vcc,@function
+reserve_vcc:
+  s_endpgm
+
+.p2align 6
+.amdhsa_kernel reserve_vcc
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset 4
+  .amdhsa_reserve_vcc defined_boolean
+.end_amdhsa_kernel
+
+// ASM: error: directive should have resolvable expression
+// ASM-NEXT:   .amdhsa_reserve_vcc
+
+.p2align 8
+.type reserve_flat_scratch,@function
+reserve_flat_scratch:
+  s_endpgm
+
+.p2align 6
+.amdhsa_kernel reserve_flat_scratch
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset 4
+  .amdhsa_reserve_flat_scratch defined_boolean
+.end_amdhsa_kernel
+
+// ASM: error: directive should have resolvable expression
+// ASM-NEXT:   .amdhsa_reserve_flat_scratch
+
+.p2align 8
+.type shared_vgpr_count,@function
+shared_vgpr_count:
+  s_endpgm
+
+.p2align 6
+.amdhsa_kernel shared_vgpr_count
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset 4
+  .amdhsa_shared_vgpr_count defined_boolean
+.end_amdhsa_kernel
+
+// ASM: error: directive should have resolvable expression
+// ASM-NEXT:   .amdhsa_shared_vgpr_count
+
+.set defined_boolean, 1
+
+// ASM:       .set defined_boolean, 1
+// ASM-NEXT:  .no_dead_strip defined_boolean
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s
new file mode 100644
index 000000000000..95af59c413ae
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s
@@ -0,0 +1,190 @@
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck --check-prefix=ASM %s
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1010 -filetype=obj < %s > %t
+// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
+
+// When going from asm -> asm, the expressions should remain the same (i.e., symbolic).
+// When going from asm -> obj, the expressions should get resolved (through fixups),
+
+// OBJDUMP: Contents of section .rodata
+// expr_defined_later
+// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000
+// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0030 00f0afe4 801f007f 000c0000 00000000
+// expr_defined
+// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000
+// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0070 00f0afe4 801f007f 000c0000 00000000
+
+.text
+// ASM: .text
+
+.amdhsa_code_object_version 4
+// ASM: .amdhsa_code_object_version 4
+
+.p2align 8
+.type expr_defined_later,@function
+expr_defined_later:
+  s_endpgm
+
+.p2align 8
+.type expr_defined,@function
+expr_defined:
+  s_endpgm
+
+.rodata
+// ASM: .rodata
+
+.p2align 6
+.amdhsa_kernel expr_defined_later
+  .amdhsa_group_segment_fixed_size defined_value+2
+  .amdhsa_private_segment_fixed_size defined_value+3
+  .amdhsa_system_vgpr_workitem_id defined_2_bits
+  .amdhsa_float_round_mode_32 defined_2_bits
+  .amdhsa_float_round_mode_16_64 defined_2_bits
+  .amdhsa_float_denorm_mode_32 defined_2_bits
+  .amdhsa_float_denorm_mode_16_64 defined_2_bits
+  .amdhsa_system_sgpr_workgroup_id_x defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_y defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_z defined_boolean
+  .amdhsa_system_sgpr_workgroup_info defined_boolean
+  .amdhsa_fp16_overflow defined_boolean
+  .amdhsa_workgroup_processor_mode defined_boolean
+  .amdhsa_memory_ordered defined_boolean
+  .amdhsa_forward_progress defined_boolean
+  .amdhsa_exception_fp_ieee_invalid_op defined_boolean
+  .amdhsa_exception_fp_denorm_src defined_boolean
+  .amdhsa_exception_fp_ieee_div_zero defined_boolean
+  .amdhsa_exception_fp_ieee_overflow defined_boolean
+  .amdhsa_exception_fp_ieee_underflow defined_boolean
+  .amdhsa_exception_fp_ieee_inexact defined_boolean
+  .amdhsa_exception_int_div_zero defined_boolean
+  .amdhsa_uses_dynamic_stack defined_boolean
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+.end_amdhsa_kernel
+
+.set defined_value, 41
+.set defined_2_bits, 3
+.set defined_boolean, 1
+
+.p2align 6
+.amdhsa_kernel expr_defined
+  .amdhsa_group_segment_fixed_size defined_value+1
+  .amdhsa_private_segment_fixed_size defined_value+2
+  .amdhsa_system_vgpr_workitem_id defined_2_bits
+  .amdhsa_float_round_mode_32 defined_2_bits
+  .amdhsa_float_round_mode_16_64 defined_2_bits
+  .amdhsa_float_denorm_mode_32 defined_2_bits
+  .amdhsa_float_denorm_mode_16_64 defined_2_bits
+  .amdhsa_system_sgpr_workgroup_id_x defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_y defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_z defined_boolean
+  .amdhsa_system_sgpr_workgroup_info defined_boolean
+  .amdhsa_fp16_overflow defined_boolean
+  .amdhsa_workgroup_processor_mode defined_boolean
+  .amdhsa_memory_ordered defined_boolean
+  .amdhsa_forward_progress defined_boolean
+  .amdhsa_exception_fp_ieee_invalid_op defined_boolean
+  .amdhsa_exception_fp_denorm_src defined_boolean
+  .amdhsa_exception_fp_ieee_div_zero defined_boolean
+  .amdhsa_exception_fp_ieee_overflow defined_boolean
+  .amdhsa_exception_fp_ieee_underflow defined_boolean
+  .amdhsa_exception_fp_ieee_inexact defined_boolean
+  .amdhsa_exception_int_div_zero defined_boolean
+  .amdhsa_uses_dynamic_stack defined_boolean
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel expr_defined_later
+// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2
+// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3
+// ASM-NEXT: .amdhsa_kernarg_size 0
+// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1)>>0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4
+// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&32)>>5
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6
+// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10
+// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11
+// ASM-NEXT: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 0
+// ASM-NEXT: .amdhsa_reserve_xnack_mask 1
+// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18
+// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21
+// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23
+// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26
+// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&536870912)>>29
+// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&1073741824)>>30
+// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2147483648)>>31
+// ASM-NEXT: .amdhsa_shared_vgpr_count 0
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29
+// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30
+// ASM-NEXT: .end_amdhsa_kernel
+
+// ASM:       .set defined_value, 41
+// ASM-NEXT:  .no_dead_strip defined_value
+// ASM-NEXT:  .set defined_2_bits, 3
+// ASM-NEXT:  .no_dead_strip defined_2_bits
+// ASM-NEXT:  .set defined_boolean, 1
+// ASM-NEXT:  .no_dead_strip defined_boolean
+
+// ASM: .amdhsa_kernel expr_defined
+// ASM-NEXT: .amdhsa_group_segment_fixed_size 42
+// ASM-NEXT: .amdhsa_private_segment_fixed_size 43
+// ASM-NEXT: .amdhsa_kernarg_size 0
+// ASM-NEXT: .amdhsa_user_sgpr_count 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+// ASM-NEXT: .amdhsa_wavefront_size32 1
+// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3
+// ASM-NEXT: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 0
+// ASM-NEXT: .amdhsa_reserve_xnack_mask 1
+// ASM-NEXT: .amdhsa_float_round_mode_32 3
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 3
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 3
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3
+// ASM-NEXT: .amdhsa_dx10_clamp 1
+// ASM-NEXT: .amdhsa_ieee_mode 1
+// ASM-NEXT: .amdhsa_fp16_overflow 1
+// ASM-NEXT: .amdhsa_workgroup_processor_mode 1
+// ASM-NEXT: .amdhsa_memory_ordered 1
+// ASM-NEXT: .amdhsa_forward_progress 1
+// ASM-NEXT: .amdhsa_shared_vgpr_count 0
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1
+// ASM-NEXT: .amdhsa_exception_int_div_zero 1
+// ASM-NEXT: .end_amdhsa_kernel
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s
new file mode 100644
index 000000000000..e1107fb69ba4
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s
@@ -0,0 +1,186 @@
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefix=ASM %s
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1100 -filetype=obj < %s > %t
+// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
+
+// When going from asm -> asm, the expressions should remain the same (i.e., symbolic).
+// When going from asm -> obj, the expressions should get resolved (through fixups),
+
+// OBJDUMP: Contents of section .rodata
+// expr_defined_later
+// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000
+// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0030 00f0afe4 811f007f 000c0000 00000000
+// expr_defined
+// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000
+// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0070 00f0afe4 811f007f 000c0000 00000000
+
+.text
+// ASM: .text
+
+.amdhsa_code_object_version 4
+// ASM: .amdhsa_code_object_version 4
+
+.p2align 8
+.type expr_defined_later,@function
+expr_defined_later:
+  s_endpgm
+
+.p2align 8
+.type expr_defined,@function
+expr_defined:
+  s_endpgm
+
+.rodata
+// ASM: .rodata
+
+.p2align 6
+.amdhsa_kernel expr_defined_later
+  .amdhsa_group_segment_fixed_size defined_value+2
+  .amdhsa_private_segment_fixed_size defined_value+3
+  .amdhsa_system_vgpr_workitem_id defined_2_bits
+  .amdhsa_float_round_mode_32 defined_2_bits
+  .amdhsa_float_round_mode_16_64 defined_2_bits
+  .amdhsa_float_denorm_mode_32 defined_2_bits
+  .amdhsa_float_denorm_mode_16_64 defined_2_bits
+  .amdhsa_system_sgpr_workgroup_id_x defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_y defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_z defined_boolean
+  .amdhsa_system_sgpr_workgroup_info defined_boolean
+  .amdhsa_fp16_overflow defined_boolean
+  .amdhsa_workgroup_processor_mode defined_boolean
+  .amdhsa_memory_ordered defined_boolean
+  .amdhsa_forward_progress defined_boolean
+  .amdhsa_exception_fp_ieee_invalid_op defined_boolean
+  .amdhsa_exception_fp_denorm_src defined_boolean
+  .amdhsa_exception_fp_ieee_div_zero defined_boolean
+  .amdhsa_exception_fp_ieee_overflow defined_boolean
+  .amdhsa_exception_fp_ieee_underflow defined_boolean
+  .amdhsa_exception_fp_ieee_inexact defined_boolean
+  .amdhsa_exception_int_div_zero defined_boolean
+  .amdhsa_enable_private_segment defined_boolean
+  .amdhsa_uses_dynamic_stack defined_boolean
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+.end_amdhsa_kernel
+
+.set defined_value, 41
+.set defined_2_bits, 3
+.set defined_boolean, 1
+
+.p2align 6
+.amdhsa_kernel expr_defined
+  .amdhsa_group_segment_fixed_size defined_value+1
+  .amdhsa_private_segment_fixed_size defined_value+2
+  .amdhsa_system_vgpr_workitem_id defined_2_bits
+  .amdhsa_float_round_mode_32 defined_2_bits
+  .amdhsa_float_round_mode_16_64 defined_2_bits
+  .amdhsa_float_denorm_mode_32 defined_2_bits
+  .amdhsa_float_denorm_mode_16_64 defined_2_bits
+  .amdhsa_system_sgpr_workgroup_id_x defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_y defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_z defined_boolean
+  .amdhsa_system_sgpr_workgroup_info defined_boolean
+  .amdhsa_fp16_overflow defined_boolean
+  .amdhsa_workgroup_processor_mode defined_boolean
+  .amdhsa_memory_ordered defined_boolean
+  .amdhsa_forward_progress defined_boolean
+  .amdhsa_exception_fp_ieee_invalid_op defined_boolean
+  .amdhsa_exception_fp_denorm_src defined_boolean
+  .amdhsa_exception_fp_ieee_div_zero defined_boolean
+  .amdhsa_exception_fp_ieee_overflow defined_boolean
+  .amdhsa_exception_fp_ieee_underflow defined_boolean
+  .amdhsa_exception_fp_ieee_inexact defined_boolean
+  .amdhsa_exception_int_div_zero defined_boolean
+  .amdhsa_enable_private_segment defined_boolean
+  .amdhsa_uses_dynamic_stack defined_boolean
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel expr_defined_later
+// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2
+// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3
+// ASM-NEXT: .amdhsa_kernarg_size 0
+// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6
+// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10
+// ASM-NEXT: .amdhsa_enable_private_segment (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11
+// ASM-NEXT: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 0
+// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18
+// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21
+// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23
+// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26
+// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&536870912)>>29
+// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&1073741824)>>30
+// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2147483648)>>31
+// ASM-NEXT: .amdhsa_shared_vgpr_count 0
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29
+// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30
+// ASM-NEXT: .end_amdhsa_kernel
+
+// ASM:       .set defined_value, 41
+// ASM-NEXT:  .no_dead_strip defined_value
+// ASM-NEXT:  .set defined_2_bits, 3
+// ASM-NEXT:  .no_dead_strip defined_2_bits
+// ASM-NEXT:  .set defined_boolean, 1
+// ASM-NEXT:  .no_dead_strip defined_boolean
+
+// ASM: .amdhsa_kernel expr_defined
+// ASM-NEXT: .amdhsa_group_segment_fixed_size 42
+// ASM-NEXT: .amdhsa_private_segment_fixed_size 43
+// ASM-NEXT: .amdhsa_kernarg_size 0
+// ASM-NEXT: .amdhsa_user_sgpr_count 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+// ASM-NEXT: .amdhsa_wavefront_size32 1
+// ASM-NEXT: .amdhsa_enable_private_segment 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3
+// ASM-NEXT: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 0
+// ASM-NEXT: .amdhsa_float_round_mode_32 3
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 3
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 3
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3
+// ASM-NEXT: .amdhsa_dx10_clamp 1
+// ASM-NEXT: .amdhsa_ieee_mode 1
+// ASM-NEXT: .amdhsa_fp16_overflow 1
+// ASM-NEXT: .amdhsa_workgroup_processor_mode 1
+// ASM-NEXT: .amdhsa_memory_ordered 1
+// ASM-NEXT: .amdhsa_forward_progress 1
+// ASM-NEXT: .amdhsa_shared_vgpr_count 0
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1
+// ASM-NEXT: .amdhsa_exception_int_div_zero 1
+// ASM-NEXT: .end_amdhsa_kernel
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s
new file mode 100644
index 000000000000..449616d35186
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s
@@ -0,0 +1,184 @@
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefix=ASM %s
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1200 -filetype=obj < %s > %t
+// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
+
+// When going from asm -> asm, the expressions should remain the same (i.e., symbolic).
+// When going from asm -> obj, the expressions should get resolved (through fixups),
+
+// OBJDUMP: Contents of section .rodata
+// expr_defined_later
+// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000
+// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0030 00f02fe4 811f007f 000c0000 00000000
+// expr_defined
+// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000
+// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0070 00f02fe4 811f007f 000c0000 00000000
+
+.text
+// ASM: .text
+
+.amdhsa_code_object_version 4
+// ASM: .amdhsa_code_object_version 4
+
+.p2align 8
+.type expr_defined_later,@function
+expr_defined_later:
+  s_endpgm
+
+.p2align 8
+.type expr_defined,@function
+expr_defined:
+  s_endpgm
+
+.rodata
+// ASM: .rodata
+
+.p2align 6
+.amdhsa_kernel expr_defined_later
+  .amdhsa_group_segment_fixed_size defined_value+2
+  .amdhsa_private_segment_fixed_size defined_value+3
+  .amdhsa_system_vgpr_workitem_id defined_2_bits
+  .amdhsa_float_round_mode_32 defined_2_bits
+  .amdhsa_float_round_mode_16_64 defined_2_bits
+  .amdhsa_float_denorm_mode_32 defined_2_bits
+  .amdhsa_float_denorm_mode_16_64 defined_2_bits
+  .amdhsa_system_sgpr_workgroup_id_x defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_y defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_z defined_boolean
+  .amdhsa_system_sgpr_workgroup_info defined_boolean
+  .amdhsa_fp16_overflow defined_boolean
+  .amdhsa_workgroup_processor_mode defined_boolean
+  .amdhsa_memory_ordered defined_boolean
+  .amdhsa_forward_progress defined_boolean
+  .amdhsa_exception_fp_ieee_invalid_op defined_boolean
+  .amdhsa_exception_fp_denorm_src defined_boolean
+  .amdhsa_exception_fp_ieee_div_zero defined_boolean
+  .amdhsa_exception_fp_ieee_overflow defined_boolean
+  .amdhsa_exception_fp_ieee_underflow defined_boolean
+  .amdhsa_exception_fp_ieee_inexact defined_boolean
+  .amdhsa_exception_int_div_zero defined_boolean
+  .amdhsa_round_robin_scheduling defined_boolean
+  .amdhsa_enable_private_segment defined_boolean
+  .amdhsa_uses_dynamic_stack defined_boolean
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+.end_amdhsa_kernel
+
+.set defined_value, 41
+.set defined_2_bits, 3
+.set defined_boolean, 1
+
+.p2align 6
+.amdhsa_kernel expr_defined
+  .amdhsa_group_segment_fixed_size defined_value+1
+  .amdhsa_private_segment_fixed_size defined_value+2
+  .amdhsa_system_vgpr_workitem_id defined_2_bits
+  .amdhsa_float_round_mode_32 defined_2_bits
+  .amdhsa_float_round_mode_16_64 defined_2_bits
+  .amdhsa_float_denorm_mode_32 defined_2_bits
+  .amdhsa_float_denorm_mode_16_64 defined_2_bits
+  .amdhsa_system_sgpr_workgroup_id_x defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_y defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_z defined_boolean
+  .amdhsa_system_sgpr_workgroup_info defined_boolean
+  .amdhsa_fp16_overflow defined_boolean
+  .amdhsa_workgroup_processor_mode defined_boolean
+  .amdhsa_memory_ordered defined_boolean
+  .amdhsa_forward_progress defined_boolean
+  .amdhsa_exception_fp_ieee_invalid_op defined_boolean
+  .amdhsa_exception_fp_denorm_src defined_boolean
+  .amdhsa_exception_fp_ieee_div_zero defined_boolean
+  .amdhsa_exception_fp_ieee_overflow defined_boolean
+  .amdhsa_exception_fp_ieee_underflow defined_boolean
+  .amdhsa_exception_fp_ieee_inexact defined_boolean
+  .amdhsa_exception_int_div_zero defined_boolean
+  .amdhsa_round_robin_scheduling defined_boolean
+  .amdhsa_enable_private_segment defined_boolean
+  .amdhsa_uses_dynamic_stack defined_boolean
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel expr_defined_later
+// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2
+// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3
+// ASM-NEXT: .amdhsa_kernarg_size 0
+// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6
+// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10
+// ASM-NEXT: .amdhsa_enable_private_segment (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11
+// ASM-NEXT: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 0
+// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18
+// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26
+// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&536870912)>>29
+// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&1073741824)>>30
+// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&2147483648)>>31
+// ASM-NEXT: .amdhsa_round_robin_scheduling (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29
+// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30
+// ASM-NEXT: .end_amdhsa_kernel
+
+// ASM:       .set defined_value, 41
+// ASM-NEXT:  .no_dead_strip defined_value
+// ASM-NEXT:  .set defined_2_bits, 3
+// ASM-NEXT:  .no_dead_strip defined_2_bits
+// ASM-NEXT:  .set defined_boolean, 1
+// ASM-NEXT:  .no_dead_strip defined_boolean
+
+// ASM: .amdhsa_kernel expr_defined
+// ASM-NEXT: .amdhsa_group_segment_fixed_size 42
+// ASM-NEXT: .amdhsa_private_segment_fixed_size 43
+// ASM-NEXT: .amdhsa_kernarg_size 0
+// ASM-NEXT: .amdhsa_user_sgpr_count 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+// ASM-NEXT: .amdhsa_wavefront_size32 1
+// ASM-NEXT: .amdhsa_enable_private_segment 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3
+// ASM-NEXT: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 0
+// ASM-NEXT: .amdhsa_float_round_mode_32 3
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 3
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 3
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3
+// ASM-NEXT: .amdhsa_fp16_overflow 1
+// ASM-NEXT: .amdhsa_workgroup_processor_mode 1
+// ASM-NEXT: .amdhsa_memory_ordered 1
+// ASM-NEXT: .amdhsa_forward_progress 1
+// ASM-NEXT: .amdhsa_round_robin_scheduling 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1
+// ASM-NEXT: .amdhsa_exception_int_div_zero 1
+// ASM-NEXT: .end_amdhsa_kernel
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s
new file mode 100644
index 000000000000..c7e05441b45f
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s
@@ -0,0 +1,168 @@
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck --check-prefix=ASM %s
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx700 -filetype=obj < %s > %t
+// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
+
+// When going from asm -> asm, the expressions should remain the same (i.e., symbolic).
+// When going from asm -> obj, the expressions should get resolved (through fixups),
+
+// OBJDUMP: Contents of section .rodata
+// expr_defined_later
+// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000
+// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0030 00f0af00 801f007f 00080000 00000000
+// expr_defined
+// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000
+// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0070 00f0af00 801f007f 00080000 00000000
+
+.text
+// ASM: .text
+
+.amdhsa_code_object_version 4
+// ASM: .amdhsa_code_object_version 4
+
+.p2align 8
+.type expr_defined_later,@function
+expr_defined_later:
+  s_endpgm
+
+.p2align 8
+.type expr_defined,@function
+expr_defined:
+  s_endpgm
+
+.rodata
+// ASM: .rodata
+
+.p2align 6
+.amdhsa_kernel expr_defined_later
+  .amdhsa_group_segment_fixed_size defined_value+2
+  .amdhsa_private_segment_fixed_size defined_value+3
+  .amdhsa_system_vgpr_workitem_id defined_2_bits
+  .amdhsa_float_round_mode_32 defined_2_bits
+  .amdhsa_float_round_mode_16_64 defined_2_bits
+  .amdhsa_float_denorm_mode_32 defined_2_bits
+  .amdhsa_float_denorm_mode_16_64 defined_2_bits
+  .amdhsa_system_sgpr_workgroup_id_x defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_y defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_z defined_boolean
+  .amdhsa_system_sgpr_workgroup_info defined_boolean
+  .amdhsa_exception_fp_ieee_invalid_op defined_boolean
+  .amdhsa_exception_fp_denorm_src defined_boolean
+  .amdhsa_exception_fp_ieee_div_zero defined_boolean
+  .amdhsa_exception_fp_ieee_overflow defined_boolean
+  .amdhsa_exception_fp_ieee_underflow defined_boolean
+  .amdhsa_exception_fp_ieee_inexact defined_boolean
+  .amdhsa_exception_int_div_zero defined_boolean
+  .amdhsa_uses_dynamic_stack defined_boolean
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+.end_amdhsa_kernel
+
+.set defined_value, 41
+.set defined_2_bits, 3
+.set defined_boolean, 1
+
+.p2align 6
+.amdhsa_kernel expr_defined
+  .amdhsa_group_segment_fixed_size defined_value+1
+  .amdhsa_private_segment_fixed_size defined_value+2
+  .amdhsa_system_vgpr_workitem_id defined_2_bits
+  .amdhsa_float_round_mode_32 defined_2_bits
+  .amdhsa_float_round_mode_16_64 defined_2_bits
+  .amdhsa_float_denorm_mode_32 defined_2_bits
+  .amdhsa_float_denorm_mode_16_64 defined_2_bits
+  .amdhsa_system_sgpr_workgroup_id_x defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_y defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_z defined_boolean
+  .amdhsa_system_sgpr_workgroup_info defined_boolean
+  .amdhsa_exception_fp_ieee_invalid_op defined_boolean
+  .amdhsa_exception_fp_denorm_src defined_boolean
+  .amdhsa_exception_fp_ieee_div_zero defined_boolean
+  .amdhsa_exception_fp_ieee_overflow defined_boolean
+  .amdhsa_exception_fp_ieee_underflow defined_boolean
+  .amdhsa_exception_fp_ieee_inexact defined_boolean
+  .amdhsa_exception_int_div_zero defined_boolean
+  .amdhsa_uses_dynamic_stack defined_boolean
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel expr_defined_later
+// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2
+// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3
+// ASM-NEXT: .amdhsa_kernarg_size 0
+// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((0&(~2048))|(defined_boolean<<11))&1)>>0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((0&(~2048))|(defined_boolean<<11))&2)>>1
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((0&(~2048))|(defined_boolean<<11))&4)>>2
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((0&(~2048))|(defined_boolean<<11))&8)>>3
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((0&(~2048))|(defined_boolean<<11))&16)>>4
+// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((0&(~2048))|(defined_boolean<<11))&32)>>5
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((0&(~2048))|(defined_boolean<<11))&64)>>6
+// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11
+// ASM-NEXT: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 0
+// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18
+// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21
+// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29
+// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30
+// ASM-NEXT: .end_amdhsa_kernel
+
+// ASM:       .set defined_value, 41
+// ASM-NEXT:  .no_dead_strip defined_value
+// ASM-NEXT:  .set defined_2_bits, 3
+// ASM-NEXT:  .no_dead_strip defined_2_bits
+// ASM-NEXT:  .set defined_boolean, 1
+// ASM-NEXT:  .no_dead_strip defined_boolean
+
+// ASM: .amdhsa_kernel expr_defined
+// ASM-NEXT: .amdhsa_group_segment_fixed_size 42
+// ASM-NEXT: .amdhsa_private_segment_fixed_size 43
+// ASM-NEXT: .amdhsa_kernarg_size 0
+// ASM-NEXT: .amdhsa_user_sgpr_count 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3
+// ASM-NEXT: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 0
+// ASM-NEXT: .amdhsa_float_round_mode_32 3
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 3
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 3
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3
+// ASM-NEXT: .amdhsa_dx10_clamp 1
+// ASM-NEXT: .amdhsa_ieee_mode 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1
+// ASM-NEXT: .amdhsa_exception_int_div_zero 1
+// ASM-NEXT: .end_amdhsa_kernel
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s
new file mode 100644
index 000000000000..49a5015987a6
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s
@@ -0,0 +1,171 @@
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx801 < %s | FileCheck --check-prefix=ASM %s
+
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx801 -filetype=obj < %s > %t
+// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
+
+// When going from asm -> asm, the expressions should remain the same (i.e., symbolic).
+// When going from asm -> obj, the expressions should get resolved (through fixups),
+
+// OBJDUMP: Contents of section .rodata
+// expr_defined_later
+// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000
+// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0030 00f0af00 801f007f 00080000 00000000
+// expr_defined
+// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000
+// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0070 00f0af00 801f007f 00080000 00000000
+
+.text
+// ASM: .text
+
+.amdhsa_code_object_version 4
+// ASM: .amdhsa_code_object_version 4
+
+.p2align 8
+.type expr_defined_later,@function
+expr_defined_later:
+  s_endpgm
+
+.p2align 8
+.type expr_defined,@function
+expr_defined:
+  s_endpgm
+
+.rodata
+// ASM: .rodata
+
+.p2align 6
+.amdhsa_kernel expr_defined_later
+  .amdhsa_group_segment_fixed_size defined_value+2
+  .amdhsa_private_segment_fixed_size defined_value+3
+  .amdhsa_system_vgpr_workitem_id defined_2_bits
+  .amdhsa_float_round_mode_32 defined_2_bits
+  .amdhsa_float_round_mode_16_64 defined_2_bits
+  .amdhsa_float_denorm_mode_32 defined_2_bits
+  .amdhsa_float_denorm_mode_16_64 defined_2_bits
+  .amdhsa_system_sgpr_workgroup_id_x defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_y defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_z defined_boolean
+  .amdhsa_system_sgpr_workgroup_info defined_boolean
+  .amdhsa_exception_fp_ieee_invalid_op defined_boolean
+  .amdhsa_exception_fp_denorm_src defined_boolean
+  .amdhsa_exception_fp_ieee_div_zero defined_boolean
+  .amdhsa_exception_fp_ieee_overflow defined_boolean
+  .amdhsa_exception_fp_ieee_underflow defined_boolean
+  .amdhsa_exception_fp_ieee_inexact defined_boolean
+  .amdhsa_exception_int_div_zero defined_boolean
+  .amdhsa_uses_dynamic_stack defined_boolean
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+.end_amdhsa_kernel
+
+.set defined_value, 41
+.set defined_2_bits, 3
+.set defined_boolean, 1
+
+.p2align 6
+.amdhsa_kernel expr_defined
+  .amdhsa_group_segment_fixed_size defined_value+1
+  .amdhsa_private_segment_fixed_size defined_value+2
+  .amdhsa_system_vgpr_workitem_id defined_2_bits
+  .amdhsa_float_round_mode_32 defined_2_bits
+  .amdhsa_float_round_mode_16_64 defined_2_bits
+  .amdhsa_float_denorm_mode_32 defined_2_bits
+  .amdhsa_float_denorm_mode_16_64 defined_2_bits
+  .amdhsa_system_sgpr_workgroup_id_x defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_y defined_boolean
+  .amdhsa_system_sgpr_workgroup_id_z defined_boolean
+  .amdhsa_system_sgpr_workgroup_info defined_boolean
+  .amdhsa_exception_fp_ieee_invalid_op defined_boolean
+  .amdhsa_exception_fp_denorm_src defined_boolean
+  .amdhsa_exception_fp_ieee_div_zero defined_boolean
+  .amdhsa_exception_fp_ieee_overflow defined_boolean
+  .amdhsa_exception_fp_ieee_underflow defined_boolean
+  .amdhsa_exception_fp_ieee_inexact defined_boolean
+  .amdhsa_exception_int_div_zero defined_boolean
+  .amdhsa_uses_dynamic_stack defined_boolean
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel expr_defined_later
+// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2
+// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3
+// ASM-NEXT: .amdhsa_kernarg_size 0
+// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((0&(~2048))|(defined_boolean<<11))&1)>>0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((0&(~2048))|(defined_boolean<<11))&2)>>1
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((0&(~2048))|(defined_boolean<<11))&4)>>2
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((0&(~2048))|(defined_boolean<<11))&8)>>3
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((0&(~2048))|(defined_boolean<<11))&16)>>4
+// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((0&(~2048))|(defined_boolean<<11))&32)>>5
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((0&(~2048))|(defined_boolean<<11))&64)>>6
+// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11
+// ASM-NEXT: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 0
+// ASM-NEXT: .amdhsa_reserve_xnack_mask 1
+// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18
+// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21
+// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29
+// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30
+// ASM-NEXT: .end_amdhsa_kernel
+
+// ASM:       .set defined_value, 41
+// ASM-NEXT:  .no_dead_strip defined_value
+// ASM-NEXT:  .set defined_2_bits, 3
+// ASM-NEXT:  .no_dead_strip defined_2_bits
+// ASM-NEXT:  .set defined_boolean, 1
+// ASM-NEXT:  .no_dead_strip defined_boolean
+
+// ASM: .amdhsa_kernel expr_defined
+// ASM-NEXT: .amdhsa_group_segment_fixed_size 42
+// ASM-NEXT: .amdhsa_private_segment_fixed_size 43
+// ASM-NEXT: .amdhsa_kernarg_size 0
+// ASM-NEXT: .amdhsa_user_sgpr_count 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3
+// ASM-NEXT: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 0
+// ASM-NEXT: .amdhsa_reserve_xnack_mask 1
+// ASM-NEXT: .amdhsa_float_round_mode_32 3
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 3
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 3
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3
+// ASM-NEXT: .amdhsa_dx10_clamp 1
+// ASM-NEXT: .amdhsa_ieee_mode 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1
+// ASM-NEXT: .amdhsa_exception_int_div_zero 1
+// ASM-NEXT: .end_amdhsa_kernel
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s
new file mode 100644
index 000000000000..b7f89239160f
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s
@@ -0,0 +1,148 @@
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck --check-prefix=ASM %s
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -filetype=obj < %s > %t
+// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
+
+// When going from asm -> asm, the expressions should remain the same (i.e., symbolic).
+// When going from asm -> obj, the expressions should get resolved (through fixups),
+
+// OBJDUMP: Contents of section .rodata
+// expr_defined_later
+// OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000100
+// OBJDUMP-NEXT: 0030 0000ac04 81000000 00000000 00000000
+// expr_defined
+// OBJDUMP-NEXT: 0040 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000100
+// OBJDUMP-NEXT: 0070 0000ac04 81000000 00000000 00000000
+
+.text
+// ASM: .text
+
+.amdhsa_code_object_version 4
+// ASM: .amdhsa_code_object_version 4
+
+.p2align 8
+.type expr_defined_later,@function
+expr_defined_later:
+  s_endpgm
+
+.p2align 8
+.type expr_defined,@function
+expr_defined:
+  s_endpgm
+
+.rodata
+// ASM: .rodata
+
+.p2align 6
+.amdhsa_kernel expr_defined_later
+  .amdhsa_system_sgpr_private_segment_wavefront_offset defined_boolean
+  .amdhsa_dx10_clamp defined_boolean
+  .amdhsa_ieee_mode defined_boolean
+  .amdhsa_fp16_overflow defined_boolean
+  .amdhsa_tg_split defined_boolean
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset 4
+.end_amdhsa_kernel
+
+.set defined_boolean, 1
+
+.p2align 6
+.amdhsa_kernel expr_defined
+  .amdhsa_system_sgpr_private_segment_wavefront_offset defined_boolean
+  .amdhsa_dx10_clamp defined_boolean
+  .amdhsa_ieee_mode defined_boolean
+  .amdhsa_fp16_overflow defined_boolean
+  .amdhsa_tg_split defined_boolean
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset 4
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel expr_defined_later
+// ASM-NEXT: .amdhsa_group_segment_fixed_size 0
+// ASM-NEXT: .amdhsa_private_segment_fixed_size 0
+// ASM-NEXT: .amdhsa_kernarg_size 0
+// ASM-NEXT: .amdhsa_user_sgpr_count (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11
+// ASM-NEXT: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 0
+// ASM-NEXT: .amdhsa_accum_offset (((((((0&(~65536))|(defined_boolean<<16))&(~63))|(0<<0))&63)>>0)+1)*4
+// ASM-NEXT: .amdhsa_reserve_xnack_mask 1
+// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18
+// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21
+// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23
+// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26
+// ASM-NEXT: .amdhsa_tg_split (((((0&(~65536))|(defined_boolean<<16))&(~63))|(0<<0))&65536)>>16
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29
+// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30
+// ASM-NEXT: .end_amdhsa_kernel
+
+// ASM:       .set defined_boolean, 1
+// ASM-NEXT:  .no_dead_strip defined_boolean
+
+// ASM: .amdhsa_kernel expr_defined
+// ASM-NEXT: .amdhsa_group_segment_fixed_size 0
+// ASM-NEXT: .amdhsa_private_segment_fixed_size 0
+// ASM-NEXT: .amdhsa_kernarg_size 0
+// ASM-NEXT: .amdhsa_user_sgpr_count 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 0
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 0
+// ASM-NEXT: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 0
+// ASM-NEXT: .amdhsa_accum_offset 4
+// ASM-NEXT: .amdhsa_reserve_xnack_mask 1
+// ASM-NEXT: .amdhsa_float_round_mode_32 0
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 0
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 0
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3
+// ASM-NEXT: .amdhsa_dx10_clamp 1
+// ASM-NEXT: .amdhsa_ieee_mode 1
+// ASM-NEXT: .amdhsa_fp16_overflow 1
+// ASM-NEXT: .amdhsa_tg_split 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 0
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src 0
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 0
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 0
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 0
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 0
+// ASM-NEXT: .amdhsa_exception_int_div_zero 0
+// ASM-NEXT: .end_amdhsa_kernel
diff --git a/llvm/test/MC/AMDGPU/hsa-tg-split.s b/llvm/test/MC/AMDGPU/hsa-tg-split.s
new file mode 100644
index 000000000000..5a4d3e2c279c
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/hsa-tg-split.s
@@ -0,0 +1,74 @@
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+xnack,+tgsplit < %s | FileCheck --check-prefix=ASM %s
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+xnack,+tgsplit -filetype=obj < %s > %t
+// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
+
+// OBJDUMP: Contents of section .rodata
+// OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000100
+// OBJDUMP-NEXT: 0030 0000ac00 80000000 00000000 00000000
+
+.text
+// ASM: .text
+
+.amdgcn_target "amdgcn-amd-amdhsa--gfx90a:xnack+"
+// ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx90a:xnack+"
+
+.amdhsa_code_object_version 4
+// ASM: .amdhsa_code_object_version 4
+
+.p2align 8
+.type minimal,@function
+minimal:
+  s_endpgm
+
+.rodata
+// ASM: .rodata
+
+.p2align 6
+.amdhsa_kernel minimal
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_accum_offset 4
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel minimal
+// ASM-NEXT: .amdhsa_group_segment_fixed_size 0
+// ASM-NEXT: .amdhsa_private_segment_fixed_size 0
+// ASM-NEXT: .amdhsa_kernarg_size 0
+// ASM-NEXT: .amdhsa_user_sgpr_count 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 0
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 0
+// ASM-NEXT: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 0
+// ASM-NEXT: .amdhsa_accum_offset 4
+// ASM-NEXT: .amdhsa_reserve_xnack_mask 1
+// ASM-NEXT: .amdhsa_float_round_mode_32 0
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 0
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 0
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3
+// ASM-NEXT: .amdhsa_dx10_clamp 1
+// ASM-NEXT: .amdhsa_ieee_mode 1
+// ASM-NEXT: .amdhsa_fp16_overflow 0
+// ASM-NEXT: .amdhsa_tg_split 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 0
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src 0
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 0
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 0
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 0
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 0
+// ASM-NEXT: .amdhsa_exception_int_div_zero 0
+// ASM-NEXT: .end_amdhsa_kernel
-- 
GitLab


From 408c36522f7eb8638314b584995daf5790968842 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Wed, 27 Mar 2024 12:00:32 +0000
Subject: [PATCH 395/404] [gn build] Port 1103a2a337e9

---
 .../gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn
index 12d875cf40c9..5ba91fcec83a 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn
@@ -104,6 +104,7 @@ static_library("MCTargetDesc") {
     "AMDGPUMCAsmInfo.cpp",
     "AMDGPUMCCodeEmitter.cpp",
     "AMDGPUMCExpr.cpp",
+    "AMDGPUMCKernelDescriptor.cpp",
     "AMDGPUMCTargetDesc.cpp",
     "AMDGPUTargetStreamer.cpp",
     "R600InstPrinter.cpp",
-- 
GitLab


From 51388fbab1b9454dfe24c4ac1c8b4a009162386a Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 26 Mar 2024 18:37:15 +0000
Subject: [PATCH 396/404] [DAG] visitSub - reuse existing SDLoc instead of
 regenerating it. NFC.

---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 39ee95d7007c..a021e0e19fc3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -3884,7 +3884,7 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
   if (SDValue V = foldSubToAvg(N, DL))
     return V;
 
-  if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, SDLoc(N)))
+  if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, DL))
     return V;
 
   if (SDValue V = foldSubToUSubSat(VT, N, DL))
@@ -3949,7 +3949,7 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
       if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0))
         if (ConstantSDNode *C = isConstOrConstSplat(N1.getOperand(1)))
           if (C->getAPIntValue() == (BitWidth - 1))
-            return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0);
+            return DAG.getNode(ISD::ABS, DL, VT, S0);
     }
   }
 
-- 
GitLab


From 9247f3185c7e1f7a2c1071fa61e283deb21091aa Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 26 Mar 2024 18:41:47 +0000
Subject: [PATCH 397/404] [DAG] foldAddSubOfSignBit - reuse existing SDLoc
 instead of regenerating it. NFC.

---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index a021e0e19fc3..36abe27d2621 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -2555,7 +2555,8 @@ SDValue DAGCombiner::foldSubToAvg(SDNode *N, const SDLoc &DL) {
 
 /// Try to fold a 'not' shifted sign-bit with add/sub with constant operand into
 /// a shift and add with a different constant.
-static SDValue foldAddSubOfSignBit(SDNode *N, SelectionDAG &DAG) {
+static SDValue foldAddSubOfSignBit(SDNode *N, const SDLoc &DL,
+                                   SelectionDAG &DAG) {
   assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&
          "Expecting add or sub");
 
@@ -2583,7 +2584,6 @@ static SDValue foldAddSubOfSignBit(SDNode *N, SelectionDAG &DAG) {
   // Eliminate the 'not' by adjusting the shift and add/sub constant:
   // add (srl (not X), 31), C --> add (sra X, 31), (C + 1)
   // sub C, (srl (not X), 31) --> add (srl X, 31), (C - 1)
-  SDLoc DL(N);
   if (SDValue NewC = DAG.FoldConstantArithmetic(
           IsAdd ? ISD::ADD : ISD::SUB, DL, VT,
           {ConstantOp, DAG.getConstant(1, DL, VT)})) {
@@ -2878,7 +2878,7 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
   if (SDValue V = foldAddSubBoolOfMaskedVal(N, DL, DAG))
     return V;
 
-  if (SDValue V = foldAddSubOfSignBit(N, DAG))
+  if (SDValue V = foldAddSubOfSignBit(N, DL, DAG))
     return V;
 
   // Try to match AVGFLOOR fixedwidth pattern
@@ -3877,7 +3877,7 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
   if (SDValue V = foldAddSubBoolOfMaskedVal(N, DL, DAG))
     return V;
 
-  if (SDValue V = foldAddSubOfSignBit(N, DAG))
+  if (SDValue V = foldAddSubOfSignBit(N, DL, DAG))
     return V;
 
   // Try to match AVGCEIL fixedwidth pattern
-- 
GitLab


From 875aed17b978bf58a01d31572af6964e91a9f641 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 27 Mar 2024 12:13:03 +0000
Subject: [PATCH 398/404] [X86] Add combineExtractFromVectorLoad helper -
 pulled out of combineExtractVectorElt

Prep work for #85419 to make it easier to reuse in other combines
---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 78 ++++++++++++++++---------
 1 file changed, 49 insertions(+), 29 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 9bad38f97c6a..4cd0bebe01bb 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -43995,6 +43995,49 @@ static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
                      Extract->getOperand(1));
 }
 
+// If this extract is from a loaded vector value and will be used as an
+// integer, that requires a potentially expensive XMM -> GPR transfer.
+// Additionally, if we can convert to a scalar integer load, that will likely
+// be folded into a subsequent integer op.
+// Note: Unlike the related fold for this in DAGCombiner, this is not limited
+//       to a single-use of the loaded vector. For the reasons above, we
+//       expect this to be profitable even if it creates an extra load.
+static SDValue
+combineExtractFromVectorLoad(SDNode *N, SDValue InputVector, uint64_t Idx,
+                             const SDLoc &dl, SelectionDAG &DAG,
+                             TargetLowering::DAGCombinerInfo &DCI) {
+  assert(N->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+         "Only EXTRACT_VECTOR_ELT supported so far");
+
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  EVT SrcVT = InputVector.getValueType();
+  EVT VT = N->getValueType(0);
+
+  bool LikelyUsedAsVector = any_of(N->uses(), [](SDNode *Use) {
+    return Use->getOpcode() == ISD::STORE ||
+           Use->getOpcode() == ISD::INSERT_VECTOR_ELT ||
+           Use->getOpcode() == ISD::SCALAR_TO_VECTOR;
+  });
+
+  auto *LoadVec = dyn_cast<LoadSDNode>(InputVector);
+  if (LoadVec && ISD::isNormalLoad(LoadVec) && VT.isInteger() &&
+      SrcVT.getVectorElementType() == VT && DCI.isAfterLegalizeDAG() &&
+      !LikelyUsedAsVector && LoadVec->isSimple()) {
+    SDValue NewPtr = TLI.getVectorElementPointer(
+        DAG, LoadVec->getBasePtr(), SrcVT, DAG.getVectorIdxConstant(Idx, dl));
+    unsigned PtrOff = VT.getSizeInBits() * Idx / 8;
+    MachinePointerInfo MPI = LoadVec->getPointerInfo().getWithOffset(PtrOff);
+    Align Alignment = commonAlignment(LoadVec->getAlign(), PtrOff);
+    SDValue Load =
+        DAG.getLoad(VT, dl, LoadVec->getChain(), NewPtr, MPI, Alignment,
+                    LoadVec->getMemOperand()->getFlags(), LoadVec->getAAInfo());
+    DAG.makeEquivalentMemoryOrdering(LoadVec, Load);
+    return Load;
+  }
+
+  return SDValue();
+}
+
 // Attempt to peek through a target shuffle and extract the scalar from the
 // source.
 static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
@@ -44600,6 +44643,11 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
   if (SDValue V = scalarizeExtEltFP(N, DAG, Subtarget))
     return V;
 
+  if (CIdx)
+    if (SDValue V = combineExtractFromVectorLoad(
+            N, InputVector, CIdx->getZExtValue(), dl, DAG, DCI))
+      return V;
+
   // Attempt to extract a i1 element by using MOVMSK to extract the signbits
   // and then testing the relevant element.
   //
@@ -44645,34 +44693,6 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
     }
   }
 
-  // If this extract is from a loaded vector value and will be used as an
-  // integer, that requires a potentially expensive XMM -> GPR transfer.
-  // Additionally, if we can convert to a scalar integer load, that will likely
-  // be folded into a subsequent integer op.
-  // Note: Unlike the related fold for this in DAGCombiner, this is not limited
-  //       to a single-use of the loaded vector. For the reasons above, we
-  //       expect this to be profitable even if it creates an extra load.
-  bool LikelyUsedAsVector = any_of(N->uses(), [](SDNode *Use) {
-    return Use->getOpcode() == ISD::STORE ||
-           Use->getOpcode() == ISD::INSERT_VECTOR_ELT ||
-           Use->getOpcode() == ISD::SCALAR_TO_VECTOR;
-  });
-  auto *LoadVec = dyn_cast<LoadSDNode>(InputVector);
-  if (LoadVec && CIdx && ISD::isNormalLoad(LoadVec) && VT.isInteger() &&
-      SrcVT.getVectorElementType() == VT && DCI.isAfterLegalizeDAG() &&
-      !LikelyUsedAsVector && LoadVec->isSimple()) {
-    SDValue NewPtr =
-        TLI.getVectorElementPointer(DAG, LoadVec->getBasePtr(), SrcVT, EltIdx);
-    unsigned PtrOff = VT.getSizeInBits() * CIdx->getZExtValue() / 8;
-    MachinePointerInfo MPI = LoadVec->getPointerInfo().getWithOffset(PtrOff);
-    Align Alignment = commonAlignment(LoadVec->getAlign(), PtrOff);
-    SDValue Load =
-        DAG.getLoad(VT, dl, LoadVec->getChain(), NewPtr, MPI, Alignment,
-                    LoadVec->getMemOperand()->getFlags(), LoadVec->getAAInfo());
-    DAG.makeEquivalentMemoryOrdering(LoadVec, Load);
-    return Load;
-  }
-
   return SDValue();
 }
 
@@ -48273,7 +48293,7 @@ static SDValue combineAndShuffleNot(SDNode *N, SelectionDAG &DAG,
 
   // We do not split for SSE at all, but we need to split vectors for AVX1 and
   // AVX2.
-  if (!Subtarget.useAVX512Regs() && VT.is512BitVector() && 
+  if (!Subtarget.useAVX512Regs() && VT.is512BitVector() &&
       TLI.isTypeLegal(VT.getHalfNumVectorElementsVT(*DAG.getContext()))) {
     SDValue LoX, HiX;
     std::tie(LoX, HiX) = splitVector(X, DAG, DL);
-- 
GitLab


From e82765bf07a978674c0e75c8b2e20f154ae24a4c Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 27 Mar 2024 12:18:54 +0000
Subject: [PATCH 399/404] [X86] masked_store.ll - add nounwind to remove cfi
 noise

---
 llvm/test/CodeGen/X86/masked_store.ll | 79 +++++++++++----------------
 1 file changed, 33 insertions(+), 46 deletions(-)

diff --git a/llvm/test/CodeGen/X86/masked_store.ll b/llvm/test/CodeGen/X86/masked_store.ll
index 898b34e969b1..03245ea31730 100644
--- a/llvm/test/CodeGen/X86/masked_store.ll
+++ b/llvm/test/CodeGen/X86/masked_store.ll
@@ -12,7 +12,7 @@
 ; vXf64
 ;
 
-define void @store_v1f64_v1i64(<1 x i64> %trigger, ptr %addr, <1 x double> %val) {
+define void @store_v1f64_v1i64(<1 x i64> %trigger, ptr %addr, <1 x double> %val) nounwind {
 ; SSE-LABEL: store_v1f64_v1i64:
 ; SSE:       ## %bb.0:
 ; SSE-NEXT:    testq %rdi, %rdi
@@ -46,7 +46,7 @@ define void @store_v1f64_v1i64(<1 x i64> %trigger, ptr %addr, <1 x double> %val)
   ret void
 }
 
-define void @store_v2f64_v2i64(<2 x i64> %trigger, ptr %addr, <2 x double> %val) {
+define void @store_v2f64_v2i64(<2 x i64> %trigger, ptr %addr, <2 x double> %val) nounwind {
 ; SSE-LABEL: store_v2f64_v2i64:
 ; SSE:       ## %bb.0:
 ; SSE-NEXT:    movmskpd %xmm0, %eax
@@ -106,7 +106,7 @@ define void @store_v2f64_v2i64(<2 x i64> %trigger, ptr %addr, <2 x double> %val)
   ret void
 }
 
-define void @store_v4f64_v4i64(<4 x i64> %trigger, ptr %addr, <4 x double> %val) {
+define void @store_v4f64_v4i64(<4 x i64> %trigger, ptr %addr, <4 x double> %val) nounwind {
 ; SSE2-LABEL: store_v4f64_v4i64:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
@@ -222,7 +222,7 @@ define void @store_v4f64_v4i64(<4 x i64> %trigger, ptr %addr, <4 x double> %val)
 ; vXf32
 ;
 
-define void @store_v2f32_v2i32(<2 x i32> %trigger, ptr %addr, <2 x float> %val) {
+define void @store_v2f32_v2i32(<2 x i32> %trigger, ptr %addr, <2 x float> %val) nounwind {
 ; SSE2-LABEL: store_v2f32_v2i32:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
@@ -314,7 +314,7 @@ define void @store_v2f32_v2i32(<2 x i32> %trigger, ptr %addr, <2 x float> %val)
   ret void
 }
 
-define void @store_v4f32_v4i32(<4 x float> %x, ptr %ptr, <4 x float> %y, <4 x i32> %mask) {
+define void @store_v4f32_v4i32(<4 x float> %x, ptr %ptr, <4 x float> %y, <4 x i32> %mask) nounwind {
 ; SSE2-LABEL: store_v4f32_v4i32:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    movmskps %xmm2, %eax
@@ -425,7 +425,7 @@ define void @store_v4f32_v4i32(<4 x float> %x, ptr %ptr, <4 x float> %y, <4 x i3
   ret void
 }
 
-define void @store_v8f32_v8i32(<8 x float> %x, ptr %ptr, <8 x float> %y, <8 x i32> %mask) {
+define void @store_v8f32_v8i32(<8 x float> %x, ptr %ptr, <8 x float> %y, <8 x i32> %mask) nounwind {
 ; SSE2-LABEL: store_v8f32_v8i32:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    packssdw %xmm5, %xmm4
@@ -605,7 +605,7 @@ define void @store_v8f32_v8i32(<8 x float> %x, ptr %ptr, <8 x float> %y, <8 x i3
   ret void
 }
 
-define void @store_v16f32_v16i32(<16 x float> %x, ptr %ptr, <16 x float> %y, <16 x i32> %mask) {
+define void @store_v16f32_v16i32(<16 x float> %x, ptr %ptr, <16 x float> %y, <16 x i32> %mask) nounwind {
 ; SSE2-LABEL: store_v16f32_v16i32:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm4
@@ -914,7 +914,7 @@ define void @store_v16f32_v16i32(<16 x float> %x, ptr %ptr, <16 x float> %y, <16
 ; vXi64
 ;
 
-define void @store_v2i64_v2i64(<2 x i64> %trigger, ptr %addr, <2 x i64> %val) {
+define void @store_v2i64_v2i64(<2 x i64> %trigger, ptr %addr, <2 x i64> %val) nounwind {
 ; SSE2-LABEL: store_v2i64_v2i64:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    movmskpd %xmm0, %eax
@@ -998,7 +998,7 @@ define void @store_v2i64_v2i64(<2 x i64> %trigger, ptr %addr, <2 x i64> %val) {
   ret void
 }
 
-define void @store_v4i64_v4i64(<4 x i64> %trigger, ptr %addr, <4 x i64> %val) {
+define void @store_v4i64_v4i64(<4 x i64> %trigger, ptr %addr, <4 x i64> %val) nounwind {
 ; SSE2-LABEL: store_v4i64_v4i64:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]
@@ -1122,7 +1122,7 @@ define void @store_v4i64_v4i64(<4 x i64> %trigger, ptr %addr, <4 x i64> %val) {
 ; vXi32
 ;
 
-define void @store_v1i32_v1i32(<1 x i32> %trigger, ptr %addr, <1 x i32> %val) {
+define void @store_v1i32_v1i32(<1 x i32> %trigger, ptr %addr, <1 x i32> %val) nounwind {
 ; SSE-LABEL: store_v1i32_v1i32:
 ; SSE:       ## %bb.0:
 ; SSE-NEXT:    testl %edi, %edi
@@ -1156,7 +1156,7 @@ define void @store_v1i32_v1i32(<1 x i32> %trigger, ptr %addr, <1 x i32> %val) {
   ret void
 }
 
-define void @store_v2i32_v2i32(<2 x i32> %trigger, ptr %addr, <2 x i32> %val) {
+define void @store_v2i32_v2i32(<2 x i32> %trigger, ptr %addr, <2 x i32> %val) nounwind {
 ; SSE2-LABEL: store_v2i32_v2i32:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
@@ -1256,7 +1256,7 @@ define void @store_v2i32_v2i32(<2 x i32> %trigger, ptr %addr, <2 x i32> %val) {
   ret void
 }
 
-define void @store_v4i32_v4i32(<4 x i32> %trigger, ptr %addr, <4 x i32> %val) {
+define void @store_v4i32_v4i32(<4 x i32> %trigger, ptr %addr, <4 x i32> %val) nounwind {
 ; SSE2-LABEL: store_v4i32_v4i32:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
@@ -1370,7 +1370,7 @@ define void @store_v4i32_v4i32(<4 x i32> %trigger, ptr %addr, <4 x i32> %val) {
   ret void
 }
 
-define void @store_v8i32_v8i32(<8 x i32> %trigger, ptr %addr, <8 x i32> %val) {
+define void @store_v8i32_v8i32(<8 x i32> %trigger, ptr %addr, <8 x i32> %val) nounwind {
 ; SSE2-LABEL: store_v8i32_v8i32:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
@@ -1560,7 +1560,7 @@ define void @store_v8i32_v8i32(<8 x i32> %trigger, ptr %addr, <8 x i32> %val) {
 ; vXi16
 ;
 
-define void @store_v8i16_v8i16(<8 x i16> %trigger, ptr %addr, <8 x i16> %val) {
+define void @store_v8i16_v8i16(<8 x i16> %trigger, ptr %addr, <8 x i16> %val) nounwind {
 ; SSE2-LABEL: store_v8i16_v8i16:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
@@ -1907,7 +1907,7 @@ define void @store_v8i16_v8i16(<8 x i16> %trigger, ptr %addr, <8 x i16> %val) {
   ret void
 }
 
-define void @store_v16i16_v16i16(<16 x i16> %trigger, ptr %addr, <16 x i16> %val) {
+define void @store_v16i16_v16i16(<16 x i16> %trigger, ptr %addr, <16 x i16> %val) nounwind {
 ; SSE2-LABEL: store_v16i16_v16i16:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
@@ -2676,7 +2676,7 @@ define void @store_v16i16_v16i16(<16 x i16> %trigger, ptr %addr, <16 x i16> %val
 ; vXi8
 ;
 
-define void @store_v16i8_v16i8(<16 x i8> %trigger, ptr %addr, <16 x i8> %val) {
+define void @store_v16i8_v16i8(<16 x i8> %trigger, ptr %addr, <16 x i8> %val) nounwind {
 ; SSE2-LABEL: store_v16i8_v16i8:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
@@ -3273,7 +3273,7 @@ define void @store_v16i8_v16i8(<16 x i8> %trigger, ptr %addr, <16 x i8> %val) {
   ret void
 }
 
-define void @store_v32i8_v32i8(<32 x i8> %trigger, ptr %addr, <32 x i8> %val) {
+define void @store_v32i8_v32i8(<32 x i8> %trigger, ptr %addr, <32 x i8> %val) nounwind {
 ; SSE2-LABEL: store_v32i8_v32i8:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
@@ -4670,7 +4670,7 @@ define void @store_v32i8_v32i8(<32 x i8> %trigger, ptr %addr, <32 x i8> %val) {
 
 ;;; Stores with Constant Masks
 
-define void @mstore_constmask_v4i32_v4i32(<4 x i32> %trigger, ptr %addr, <4 x i32> %val) {
+define void @mstore_constmask_v4i32_v4i32(<4 x i32> %trigger, ptr %addr, <4 x i32> %val) nounwind {
 ; SSE-LABEL: mstore_constmask_v4i32_v4i32:
 ; SSE:       ## %bb.0:
 ; SSE-NEXT:    movups %xmm1, (%rdi)
@@ -4693,7 +4693,7 @@ define void @mstore_constmask_v4i32_v4i32(<4 x i32> %trigger, ptr %addr, <4 x i3
 
 ; Make sure we are able to detect all ones constant mask after type legalization
 ; to avoid masked stores.
-define void @mstore_constmask_allones_split(<16 x i64> %trigger, ptr %addr, <16 x i64> %val) {
+define void @mstore_constmask_allones_split(<16 x i64> %trigger, ptr %addr, <16 x i64> %val) nounwind {
 ; SSE2-LABEL: mstore_constmask_allones_split:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm0
@@ -4810,7 +4810,7 @@ define void @mstore_constmask_allones_split(<16 x i64> %trigger, ptr %addr, <16
 
 ;  When only one element of the mask is set, reduce to a scalar store.
 
-define void @one_mask_bit_set1(ptr %addr, <4 x i32> %val) {
+define void @one_mask_bit_set1(ptr %addr, <4 x i32> %val) nounwind {
 ; SSE-LABEL: one_mask_bit_set1:
 ; SSE:       ## %bb.0:
 ; SSE-NEXT:    movss %xmm0, (%rdi)
@@ -4832,7 +4832,7 @@ define void @one_mask_bit_set1(ptr %addr, <4 x i32> %val) {
 
 ; Choose a different element to show that the correct address offset is produced.
 
-define void @one_mask_bit_set2(ptr %addr, <4 x float> %val) {
+define void @one_mask_bit_set2(ptr %addr, <4 x float> %val) nounwind {
 ; SSE2-LABEL: one_mask_bit_set2:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
@@ -4860,7 +4860,7 @@ define void @one_mask_bit_set2(ptr %addr, <4 x float> %val) {
 
 ; Choose a different scalar type and a high element of a 256-bit vector because AVX doesn't support those evenly.
 
-define void @one_mask_bit_set3(ptr %addr, <4 x i64> %val) {
+define void @one_mask_bit_set3(ptr %addr, <4 x i64> %val) nounwind {
 ; SSE-LABEL: one_mask_bit_set3:
 ; SSE:       ## %bb.0:
 ; SSE-NEXT:    movlps %xmm1, 16(%rdi)
@@ -4886,7 +4886,7 @@ define void @one_mask_bit_set3(ptr %addr, <4 x i64> %val) {
 
 ; Choose a different scalar type and a high element of a 256-bit vector because AVX doesn't support those evenly.
 
-define void @one_mask_bit_set4(ptr %addr, <4 x double> %val) {
+define void @one_mask_bit_set4(ptr %addr, <4 x double> %val) nounwind {
 ; SSE-LABEL: one_mask_bit_set4:
 ; SSE:       ## %bb.0:
 ; SSE-NEXT:    movhps %xmm1, 24(%rdi)
@@ -4912,7 +4912,7 @@ define void @one_mask_bit_set4(ptr %addr, <4 x double> %val) {
 
 ; Try a 512-bit vector to make sure AVX doesn't die and AVX512 works as expected.
 
-define void @one_mask_bit_set5(ptr %addr, <8 x double> %val) {
+define void @one_mask_bit_set5(ptr %addr, <8 x double> %val) nounwind {
 ; SSE-LABEL: one_mask_bit_set5:
 ; SSE:       ## %bb.0:
 ; SSE-NEXT:    movlps %xmm3, 48(%rdi)
@@ -4944,7 +4944,7 @@ define void @one_mask_bit_set5(ptr %addr, <8 x double> %val) {
 }
 
 ; Try one elt in each half of a vector that needs to split
-define void @one_mask_bit_set6(ptr %addr, <16 x i64> %val) {
+define void @one_mask_bit_set6(ptr %addr, <16 x i64> %val) nounwind {
 ; SSE2-LABEL: one_mask_bit_set6:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    movlps %xmm3, 48(%rdi)
@@ -4999,7 +4999,7 @@ define void @one_mask_bit_set6(ptr %addr, <16 x i64> %val) {
   ret void
 }
 
-define void @top_bits_unset_stack() {
+define void @top_bits_unset_stack() nounwind {
 ; SSE-LABEL: top_bits_unset_stack:
 ; SSE:       ## %bb.0: ## %entry
 ; SSE-NEXT:    xorps %xmm0, %xmm0
@@ -5047,7 +5047,6 @@ define void @top_bits_unset_stack() {
 ; X86-AVX512-LABEL: top_bits_unset_stack:
 ; X86-AVX512:       ## %bb.0: ## %entry
 ; X86-AVX512-NEXT:    subl $76, %esp
-; X86-AVX512-NEXT:    .cfi_def_cfa_offset 80
 ; X86-AVX512-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
 ; X86-AVX512-NEXT:    movb $63, %al
 ; X86-AVX512-NEXT:    kmovd %eax, %k1
@@ -5064,7 +5063,7 @@ entry:
 
 ; SimplifyDemandedBits eliminates an ashr here.
 
-define void @masked_store_bool_mask_demand_trunc_sext(<4 x double> %x, ptr %p, <4 x i32> %masksrc) {
+define void @masked_store_bool_mask_demand_trunc_sext(<4 x double> %x, ptr %p, <4 x i32> %masksrc) nounwind {
 ; SSE-LABEL: masked_store_bool_mask_demand_trunc_sext:
 ; SSE:       ## %bb.0:
 ; SSE-NEXT:    pslld $31, %xmm2
@@ -5160,7 +5159,7 @@ define void @masked_store_bool_mask_demand_trunc_sext(<4 x double> %x, ptr %p, <
 
 ; PR26697
 
-define void @one_mask_bit_set1_variable(ptr %addr, <4 x float> %val, <4 x i32> %mask) {
+define void @one_mask_bit_set1_variable(ptr %addr, <4 x float> %val, <4 x i32> %mask) nounwind {
 ; SSE2-LABEL: one_mask_bit_set1_variable:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    movmskps %xmm1, %eax
@@ -5267,7 +5266,7 @@ define void @one_mask_bit_set1_variable(ptr %addr, <4 x float> %val, <4 x i32> %
 ; This needs to be widened to v4i32.
 ; This used to assert in type legalization. PR38436
 ; FIXME: The codegen for AVX512 should use KSHIFT to zero the upper bits of the mask.
-define void @widen_masked_store(<3 x i32> %v, ptr %p, <3 x i1> %mask) {
+define void @widen_masked_store(<3 x i32> %v, ptr %p, <3 x i1> %mask) nounwind {
 ; SSE2-LABEL: widen_masked_store:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    andb $1, %sil
@@ -5448,7 +5447,7 @@ define void @widen_masked_store(<3 x i32> %v, ptr %p, <3 x i1> %mask) {
   ret void
 }
 
-define void @zero_mask(ptr %addr, <2 x double> %val) {
+define void @zero_mask(ptr %addr, <2 x double> %val) nounwind {
 ; SSE-LABEL: zero_mask:
 ; SSE:       ## %bb.0:
 ; SSE-NEXT:    retq
@@ -5464,7 +5463,7 @@ define void @zero_mask(ptr %addr, <2 x double> %val) {
   ret void
 }
 
-define void @PR11210(<4 x float> %x, ptr %ptr, <4 x float> %y, <2 x i64> %mask) {
+define void @PR11210(<4 x float> %x, ptr %ptr, <4 x float> %y, <2 x i64> %mask) nounwind {
 ; SSE2-LABEL: PR11210:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    movmskps %xmm2, %eax
@@ -5638,7 +5637,7 @@ define void @PR11210(<4 x float> %x, ptr %ptr, <4 x float> %y, <2 x i64> %mask)
   ret void
 }
 
-define void @store_v24i32_v24i32_stride6_vf4_only_even_numbered_elts(ptr %trigger.ptr, ptr %val.ptr, ptr %dst) {
+define void @store_v24i32_v24i32_stride6_vf4_only_even_numbered_elts(ptr %trigger.ptr, ptr %val.ptr, ptr %dst) nounwind {
 ; SSE2-LABEL: store_v24i32_v24i32_stride6_vf4_only_even_numbered_elts:
 ; SSE2:       ## %bb.0:
 ; SSE2-NEXT:    movdqa (%rdi), %xmm6
@@ -5874,23 +5873,11 @@ define void @store_v24i32_v24i32_stride6_vf4_only_even_numbered_elts(ptr %trigge
 ; SSE4-LABEL: store_v24i32_v24i32_stride6_vf4_only_even_numbered_elts:
 ; SSE4:       ## %bb.0:
 ; SSE4-NEXT:    pushq %rbp
-; SSE4-NEXT:    .cfi_def_cfa_offset 16
 ; SSE4-NEXT:    pushq %r15
-; SSE4-NEXT:    .cfi_def_cfa_offset 24
 ; SSE4-NEXT:    pushq %r14
-; SSE4-NEXT:    .cfi_def_cfa_offset 32
 ; SSE4-NEXT:    pushq %r13
-; SSE4-NEXT:    .cfi_def_cfa_offset 40
 ; SSE4-NEXT:    pushq %r12
-; SSE4-NEXT:    .cfi_def_cfa_offset 48
 ; SSE4-NEXT:    pushq %rbx
-; SSE4-NEXT:    .cfi_def_cfa_offset 56
-; SSE4-NEXT:    .cfi_offset %rbx, -56
-; SSE4-NEXT:    .cfi_offset %r12, -48
-; SSE4-NEXT:    .cfi_offset %r13, -40
-; SSE4-NEXT:    .cfi_offset %r14, -32
-; SSE4-NEXT:    .cfi_offset %r15, -24
-; SSE4-NEXT:    .cfi_offset %rbp, -16
 ; SSE4-NEXT:    movdqa (%rdi), %xmm1
 ; SSE4-NEXT:    movdqa 32(%rdi), %xmm2
 ; SSE4-NEXT:    movdqa 64(%rdi), %xmm0
@@ -6266,7 +6253,7 @@ define void @store_v24i32_v24i32_stride6_vf4_only_even_numbered_elts(ptr %trigge
 }
 
 ; From https://reviews.llvm.org/rGf8d9097168b7#1165311
-define void @undefshuffle(<8 x i1> %i0, ptr %src, ptr %dst) #0 {
+define void @undefshuffle(<8 x i1> %i0, ptr %src, ptr %dst) nounwind {
 ; SSE2-LABEL: undefshuffle:
 ; SSE2:       ## %bb.0: ## %else
 ; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-- 
GitLab


From b8cc838427efa80eb5ca4ec7c8adb53e4adfc4c7 Mon Sep 17 00:00:00 2001
From: komalverma04 <komal148btit21@igdtuw.ac.in>
Date: Wed, 27 Mar 2024 05:51:27 -0700
Subject: [PATCH 400/404] [analyzer][docs] Document the
 `optin.performance.Padding` checker (#86411)

Closes #73675

Co-authored-by: Balazs Benics <benicsbalazs@gmail.com>
Co-authored-by: NagyDonat <donat.nagy@ericsson.com>
---
 clang/docs/analyzer/checkers.rst              | 83 ++++++++++++++++++-
 .../clang/StaticAnalyzer/Checkers/Checkers.td |  2 +-
 2 files changed, 82 insertions(+), 3 deletions(-)

diff --git a/clang/docs/analyzer/checkers.rst b/clang/docs/analyzer/checkers.rst
index 66da1c7b35f2..8af99a021ebd 100644
--- a/clang/docs/analyzer/checkers.rst
+++ b/clang/docs/analyzer/checkers.rst
@@ -849,10 +849,89 @@ Check for performance anti-patterns when using Grand Central Dispatch.
 
 .. _optin-performance-Padding:
 
-optin.performance.Padding
-"""""""""""""""""""""""""
+optin.performance.Padding (C, C++, ObjC)
+""""""""""""""""""""""""""""""""""""""""
 Check for excessively padded structs.
 
+This checker detects structs with excessive padding, which can lead to wasted
+memory thus decreased performance by reducing the effectiveness of the
+processor cache. Padding bytes are added by compilers to align data accesses
+as some processors require data to be aligned to certain boundaries. On others,
+unaligned data access are possible, but impose significantly larger latencies.
+
+To avoid padding bytes, the fields of a struct should be ordered by decreasing
+by alignment. Usually, its easier to think of the ``sizeof`` of the fields, and
+ordering the fields by ``sizeof`` would usually also lead to the same optimal
+layout.
+
+In rare cases, one can use the ``#pragma pack(1)`` directive to enforce a packed
+layout too, but it can significantly increase the access times, so reordering the
+fields is usually a better solution.
+
+
+.. code-block:: cpp
+
+ // warn: Excessive padding in 'struct NonOptimal' (35 padding bytes, where 3 is optimal)
+ struct NonOptimal {
+   char c1;
+   // 7 bytes of padding
+   std::int64_t big1; // 8 bytes
+   char c2;
+   // 7 bytes of padding
+   std::int64_t big2; // 8 bytes
+   char c3;
+   // 7 bytes of padding
+   std::int64_t big3; // 8 bytes
+   char c4;
+   // 7 bytes of padding
+   std::int64_t big4; // 8 bytes
+   char c5;
+   // 7 bytes of padding
+ };
+ static_assert(sizeof(NonOptimal) == 4*8+5+5*7);
+
+ // no-warning: The fields are nicely aligned to have the minimal amount of padding bytes.
+ struct Optimal {
+   std::int64_t big1; // 8 bytes
+   std::int64_t big2; // 8 bytes
+   std::int64_t big3; // 8 bytes
+   std::int64_t big4; // 8 bytes
+   char c1;
+   char c2;
+   char c3;
+   char c4;
+   char c5;
+   // 3 bytes of padding
+ };
+ static_assert(sizeof(Optimal) == 4*8+5+3);
+
+ // no-warning: Bit packing representation is also accepted by this checker, but
+ // it can significantly increase access times, so prefer reordering the fields.
+ #pragma pack(1)
+ struct BitPacked {
+   char c1;
+   std::int64_t big1; // 8 bytes
+   char c2;
+   std::int64_t big2; // 8 bytes
+   char c3;
+   std::int64_t big3; // 8 bytes
+   char c4;
+   std::int64_t big4; // 8 bytes
+   char c5;
+ };
+ static_assert(sizeof(BitPacked) == 4*8+5);
+
+The ``AllowedPad`` option can be used to specify a threshold for the number
+padding bytes raising the warning. If the number of padding bytes of the struct
+and the optimal number of padding bytes differ by more than the threshold value,
+a warning will be raised.
+
+By default, the ``AllowedPad`` threshold is 24 bytes.
+
+To override this threshold to e.g. 4 bytes, use the
+``-analyzer-config optin.performance.Padding:AllowedPad=4`` option.
+
+
 .. _optin-portability-UnixAPI:
 
 optin.portability.UnixAPI
diff --git a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
index bf46766d44b3..5fe5c9286dab 100644
--- a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
+++ b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
@@ -908,7 +908,7 @@ def PaddingChecker : Checker<"Padding">,
                   "24",
                   Released>
   ]>,
-  Documentation<NotDocumented>;
+  Documentation<HasDocumentation>;
 
 } // end: "padding"
 
-- 
GitLab


From 4f9aab2b500d3df0cc5d54f2d29c8199507af66c Mon Sep 17 00:00:00 2001
From: Pierre van Houtryve <pierre.vanhoutryve@amd.com>
Date: Wed, 27 Mar 2024 13:53:36 +0100
Subject: [PATCH 401/404] [NFC][TableGen][GlobalISel] Move MIR pattern parsing
 out of combiner (#86789)

Reland of cfa0833ccc7450a322e709583e894e4c96ce682e
---
 llvm/utils/TableGen/Common/CMakeLists.txt     |   2 +
 .../Common/GlobalISel/CombinerUtils.cpp       |  23 +
 .../Common/GlobalISel/CombinerUtils.h         |   4 +
 .../Common/GlobalISel/PatternParser.cpp       | 462 +++++++++++++++++
 .../Common/GlobalISel/PatternParser.h         | 118 +++++
 .../TableGen/GlobalISelCombinerEmitter.cpp    | 479 +-----------------
 6 files changed, 623 insertions(+), 465 deletions(-)
 create mode 100644 llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.cpp
 create mode 100644 llvm/utils/TableGen/Common/GlobalISel/PatternParser.cpp
 create mode 100644 llvm/utils/TableGen/Common/GlobalISel/PatternParser.h

diff --git a/llvm/utils/TableGen/Common/CMakeLists.txt b/llvm/utils/TableGen/Common/CMakeLists.txt
index 0440f027f286..c31ed5a1de69 100644
--- a/llvm/utils/TableGen/Common/CMakeLists.txt
+++ b/llvm/utils/TableGen/Common/CMakeLists.txt
@@ -12,10 +12,12 @@ set(LLVM_LINK_COMPONENTS
 
 add_llvm_library(LLVMTableGenCommon STATIC OBJECT EXCLUDE_FROM_ALL
   GlobalISel/CodeExpander.cpp
+  GlobalISel/CombinerUtils.cpp
   GlobalISel/CXXPredicates.cpp
   GlobalISel/GlobalISelMatchTable.cpp
   GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp
   GlobalISel/MatchDataInfo.cpp
+  GlobalISel/PatternParser.cpp
   GlobalISel/Patterns.cpp
 
   AsmWriterInst.cpp
diff --git a/llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.cpp b/llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.cpp
new file mode 100644
index 000000000000..37e630605095
--- /dev/null
+++ b/llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.cpp
@@ -0,0 +1,23 @@
+//===- CombinerUtils.cpp --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "CombinerUtils.h"
+#include "llvm/ADT/StringSet.h"
+
+namespace llvm {
+
+StringRef insertStrRef(StringRef S) {
+  if (S.empty())
+    return {};
+
+  static StringSet<> Pool;
+  auto [It, Inserted] = Pool.insert(S);
+  return It->getKey();
+}
+
+} // namespace llvm
diff --git a/llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.h b/llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.h
index 8cb2514a10e8..82a64c63edbd 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.h
+++ b/llvm/utils/TableGen/Common/GlobalISel/CombinerUtils.h
@@ -65,6 +65,10 @@ inline const DagInit *getDagWithOperatorOfSubClass(const Init &N,
         return I;
   return nullptr;
 }
+
+/// Copies a StringRef into a static pool to preserve it.
+StringRef insertStrRef(StringRef S);
+
 } // namespace llvm
 
 #endif
diff --git a/llvm/utils/TableGen/Common/GlobalISel/PatternParser.cpp b/llvm/utils/TableGen/Common/GlobalISel/PatternParser.cpp
new file mode 100644
index 000000000000..1d6c4c73a264
--- /dev/null
+++ b/llvm/utils/TableGen/Common/GlobalISel/PatternParser.cpp
@@ -0,0 +1,462 @@
+//===- PatternParser.cpp ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Common/GlobalISel/PatternParser.h"
+#include "Basic/CodeGenIntrinsics.h"
+#include "Common/CodeGenTarget.h"
+#include "Common/GlobalISel/CombinerUtils.h"
+#include "Common/GlobalISel/Patterns.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/PrettyStackTrace.h"
+#include "llvm/Support/SaveAndRestore.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
+
+namespace llvm {
+namespace gi {
+static constexpr StringLiteral MIFlagsEnumClassName = "MIFlagEnum";
+
+namespace {
+class PrettyStackTraceParse : public PrettyStackTraceEntry {
+  const Record &Def;
+
+public:
+  PrettyStackTraceParse(const Record &Def) : Def(Def) {}
+
+  void print(raw_ostream &OS) const override {
+    if (Def.isSubClassOf("GICombineRule"))
+      OS << "Parsing GICombineRule '" << Def.getName() << '\'';
+    else if (Def.isSubClassOf(PatFrag::ClassName))
+      OS << "Parsing " << PatFrag::ClassName << " '" << Def.getName() << '\'';
+    else
+      OS << "Parsing '" << Def.getName() << '\'';
+    OS << '\n';
+  }
+};
+} // namespace
+
+bool PatternParser::parsePatternList(
+    const DagInit &List,
+    function_ref<bool(std::unique_ptr<Pattern>)> ParseAction,
+    StringRef Operator, StringRef AnonPatNamePrefix) {
+  if (List.getOperatorAsDef(DiagLoc)->getName() != Operator) {
+    PrintError(DiagLoc, "Expected " + Operator + " operator");
+    return false;
+  }
+
+  if (List.getNumArgs() == 0) {
+    PrintError(DiagLoc, Operator + " pattern list is empty");
+    return false;
+  }
+
+  // The match section consists of a list of matchers and predicates. Parse each
+  // one and add the equivalent GIMatchDag nodes, predicates, and edges.
+  for (unsigned I = 0; I < List.getNumArgs(); ++I) {
+    Init *Arg = List.getArg(I);
+    std::string Name = List.getArgName(I)
+                           ? List.getArgName(I)->getValue().str()
+                           : ("__" + AnonPatNamePrefix + "_" + Twine(I)).str();
+
+    if (auto Pat = parseInstructionPattern(*Arg, Name)) {
+      if (!ParseAction(std::move(Pat)))
+        return false;
+      continue;
+    }
+
+    if (auto Pat = parseWipMatchOpcodeMatcher(*Arg, Name)) {
+      if (!ParseAction(std::move(Pat)))
+        return false;
+      continue;
+    }
+
+    // Parse arbitrary C++ code
+    if (const auto *StringI = dyn_cast<StringInit>(Arg)) {
+      auto CXXPat = std::make_unique<CXXPattern>(*StringI, insertStrRef(Name));
+      if (!ParseAction(std::move(CXXPat)))
+        return false;
+      continue;
+    }
+
+    PrintError(DiagLoc,
+               "Failed to parse pattern: '" + Arg->getAsString() + '\'');
+    return false;
+  }
+
+  return true;
+}
+
+static const CodeGenInstruction &
+getInstrForIntrinsic(const CodeGenTarget &CGT, const CodeGenIntrinsic *I) {
+  StringRef Opc;
+  if (I->isConvergent) {
+    Opc = I->hasSideEffects ? "G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS"
+                            : "G_INTRINSIC_CONVERGENT";
+  } else {
+    Opc = I->hasSideEffects ? "G_INTRINSIC_W_SIDE_EFFECTS" : "G_INTRINSIC";
+  }
+
+  RecordKeeper &RK = I->TheDef->getRecords();
+  return CGT.getInstruction(RK.getDef(Opc));
+}
+
+static const CodeGenIntrinsic *getCodeGenIntrinsic(Record *R) {
+  // Intrinsics need to have a static lifetime because the match table keeps
+  // references to CodeGenIntrinsic objects.
+  static DenseMap<const Record *, std::unique_ptr<CodeGenIntrinsic>>
+      AllIntrinsics;
+
+  auto &Ptr = AllIntrinsics[R];
+  if (!Ptr)
+    Ptr = std::make_unique<CodeGenIntrinsic>(R, std::vector<Record *>());
+  return Ptr.get();
+}
+
+std::unique_ptr<Pattern>
+PatternParser::parseInstructionPattern(const Init &Arg, StringRef Name) {
+  const DagInit *DagPat = dyn_cast<DagInit>(&Arg);
+  if (!DagPat)
+    return nullptr;
+
+  std::unique_ptr<InstructionPattern> Pat;
+  if (const DagInit *IP = getDagWithOperatorOfSubClass(Arg, "Instruction")) {
+    auto &Instr = CGT.getInstruction(IP->getOperatorAsDef(DiagLoc));
+    Pat =
+        std::make_unique<CodeGenInstructionPattern>(Instr, insertStrRef(Name));
+  } else if (const DagInit *IP =
+                 getDagWithOperatorOfSubClass(Arg, "Intrinsic")) {
+    Record *TheDef = IP->getOperatorAsDef(DiagLoc);
+    const CodeGenIntrinsic *Intrin = getCodeGenIntrinsic(TheDef);
+    const CodeGenInstruction &Instr = getInstrForIntrinsic(CGT, Intrin);
+    Pat =
+        std::make_unique<CodeGenInstructionPattern>(Instr, insertStrRef(Name));
+    cast<CodeGenInstructionPattern>(*Pat).setIntrinsic(Intrin);
+  } else if (const DagInit *PFP =
+                 getDagWithOperatorOfSubClass(Arg, PatFrag::ClassName)) {
+    const Record *Def = PFP->getOperatorAsDef(DiagLoc);
+    const PatFrag *PF = parsePatFrag(Def);
+    if (!PF)
+      return nullptr; // Already diagnosed by parsePatFrag
+    Pat = std::make_unique<PatFragPattern>(*PF, insertStrRef(Name));
+  } else if (const DagInit *BP =
+                 getDagWithOperatorOfSubClass(Arg, BuiltinPattern::ClassName)) {
+    Pat = std::make_unique<BuiltinPattern>(*BP->getOperatorAsDef(DiagLoc),
+                                           insertStrRef(Name));
+  } else
+    return nullptr;
+
+  for (unsigned K = 0; K < DagPat->getNumArgs(); ++K) {
+    Init *Arg = DagPat->getArg(K);
+    if (auto *DagArg = getDagWithSpecificOperator(*Arg, "MIFlags")) {
+      if (!parseInstructionPatternMIFlags(*Pat, DagArg))
+        return nullptr;
+      continue;
+    }
+
+    if (!parseInstructionPatternOperand(*Pat, Arg, DagPat->getArgName(K)))
+      return nullptr;
+  }
+
+  if (!Pat->checkSemantics(DiagLoc))
+    return nullptr;
+
+  return std::move(Pat);
+}
+
+std::unique_ptr<Pattern>
+PatternParser::parseWipMatchOpcodeMatcher(const Init &Arg, StringRef Name) {
+  const DagInit *Matcher = getDagWithSpecificOperator(Arg, "wip_match_opcode");
+  if (!Matcher)
+    return nullptr;
+
+  if (Matcher->getNumArgs() == 0) {
+    PrintError(DiagLoc, "Empty wip_match_opcode");
+    return nullptr;
+  }
+
+  // Each argument is an opcode that can match.
+  auto Result = std::make_unique<AnyOpcodePattern>(insertStrRef(Name));
+  for (const auto &Arg : Matcher->getArgs()) {
+    Record *OpcodeDef = getDefOfSubClass(*Arg, "Instruction");
+    if (OpcodeDef) {
+      Result->addOpcode(&CGT.getInstruction(OpcodeDef));
+      continue;
+    }
+
+    PrintError(DiagLoc, "Arguments to wip_match_opcode must be instructions");
+    return nullptr;
+  }
+
+  return std::move(Result);
+}
+
+bool PatternParser::parseInstructionPatternOperand(InstructionPattern &IP,
+                                                   const Init *OpInit,
+                                                   const StringInit *OpName) {
+  const auto ParseErr = [&]() {
+    PrintError(DiagLoc,
+               "cannot parse operand '" + OpInit->getAsUnquotedString() + "' ");
+    if (OpName)
+      PrintNote(DiagLoc,
+                "operand name is '" + OpName->getAsUnquotedString() + '\'');
+    return false;
+  };
+
+  // untyped immediate, e.g. 0
+  if (const auto *IntImm = dyn_cast<IntInit>(OpInit)) {
+    std::string Name = OpName ? OpName->getAsUnquotedString() : "";
+    IP.addOperand(IntImm->getValue(), insertStrRef(Name), PatternType());
+    return true;
+  }
+
+  // typed immediate, e.g. (i32 0)
+  if (const auto *DagOp = dyn_cast<DagInit>(OpInit)) {
+    if (DagOp->getNumArgs() != 1)
+      return ParseErr();
+
+    const Record *TyDef = DagOp->getOperatorAsDef(DiagLoc);
+    auto ImmTy = PatternType::get(DiagLoc, TyDef,
+                                  "cannot parse immediate '" +
+                                      DagOp->getAsUnquotedString() + '\'');
+    if (!ImmTy)
+      return false;
+
+    if (!IP.hasAllDefs()) {
+      PrintError(DiagLoc, "out operand of '" + IP.getInstName() +
+                              "' cannot be an immediate");
+      return false;
+    }
+
+    const auto *Val = dyn_cast<IntInit>(DagOp->getArg(0));
+    if (!Val)
+      return ParseErr();
+
+    std::string Name = OpName ? OpName->getAsUnquotedString() : "";
+    IP.addOperand(Val->getValue(), insertStrRef(Name), *ImmTy);
+    return true;
+  }
+
+  // Typed operand e.g. $x/$z in (G_FNEG $x, $z)
+  if (auto *DefI = dyn_cast<DefInit>(OpInit)) {
+    if (!OpName) {
+      PrintError(DiagLoc, "expected an operand name after '" +
+                              OpInit->getAsString() + '\'');
+      return false;
+    }
+    const Record *Def = DefI->getDef();
+    auto Ty = PatternType::get(DiagLoc, Def, "cannot parse operand type");
+    if (!Ty)
+      return false;
+    IP.addOperand(insertStrRef(OpName->getAsUnquotedString()), *Ty);
+    return true;
+  }
+
+  // Untyped operand e.g. $x/$z in (G_FNEG $x, $z)
+  if (isa<UnsetInit>(OpInit)) {
+    assert(OpName && "Unset w/ no OpName?");
+    IP.addOperand(insertStrRef(OpName->getAsUnquotedString()), PatternType());
+    return true;
+  }
+
+  return ParseErr();
+}
+
+bool PatternParser::parseInstructionPatternMIFlags(InstructionPattern &IP,
+                                                   const DagInit *Op) {
+  auto *CGIP = dyn_cast<CodeGenInstructionPattern>(&IP);
+  if (!CGIP) {
+    PrintError(DiagLoc,
+               "matching/writing MIFlags is only allowed on CodeGenInstruction "
+               "patterns");
+    return false;
+  }
+
+  const auto CheckFlagEnum = [&](const Record *R) {
+    if (!R->isSubClassOf(MIFlagsEnumClassName)) {
+      PrintError(DiagLoc, "'" + R->getName() + "' is not a subclass of '" +
+                              MIFlagsEnumClassName + "'");
+      return false;
+    }
+
+    return true;
+  };
+
+  if (CGIP->getMIFlagsInfo()) {
+    PrintError(DiagLoc, "MIFlags can only be present once on an instruction");
+    return false;
+  }
+
+  auto &FI = CGIP->getOrCreateMIFlagsInfo();
+  for (unsigned K = 0; K < Op->getNumArgs(); ++K) {
+    const Init *Arg = Op->getArg(K);
+
+    // Match/set a flag: (MIFlags FmNoNans)
+    if (const auto *Def = dyn_cast<DefInit>(Arg)) {
+      const Record *R = Def->getDef();
+      if (!CheckFlagEnum(R))
+        return false;
+
+      FI.addSetFlag(R);
+      continue;
+    }
+
+    // Do not match a flag/unset a flag: (MIFlags (not FmNoNans))
+    if (const DagInit *NotDag = getDagWithSpecificOperator(*Arg, "not")) {
+      for (const Init *NotArg : NotDag->getArgs()) {
+        const DefInit *DefArg = dyn_cast<DefInit>(NotArg);
+        if (!DefArg) {
+          PrintError(DiagLoc, "cannot parse '" + NotArg->getAsUnquotedString() +
+                                  "': expected a '" + MIFlagsEnumClassName +
+                                  "'");
+          return false;
+        }
+
+        const Record *R = DefArg->getDef();
+        if (!CheckFlagEnum(R))
+          return false;
+
+        FI.addUnsetFlag(R);
+        continue;
+      }
+
+      continue;
+    }
+
+    // Copy flags from a matched instruction: (MIFlags $mi)
+    if (isa<UnsetInit>(Arg)) {
+      FI.addCopyFlag(insertStrRef(Op->getArgName(K)->getAsUnquotedString()));
+      continue;
+    }
+  }
+
+  return true;
+}
+
+std::unique_ptr<PatFrag> PatternParser::parsePatFragImpl(const Record *Def) {
+  auto StackTrace = PrettyStackTraceParse(*Def);
+  if (!Def->isSubClassOf(PatFrag::ClassName))
+    return nullptr;
+
+  const DagInit *Ins = Def->getValueAsDag("InOperands");
+  if (Ins->getOperatorAsDef(Def->getLoc())->getName() != "ins") {
+    PrintError(Def, "expected 'ins' operator for " + PatFrag::ClassName +
+                        " in operands list");
+    return nullptr;
+  }
+
+  const DagInit *Outs = Def->getValueAsDag("OutOperands");
+  if (Outs->getOperatorAsDef(Def->getLoc())->getName() != "outs") {
+    PrintError(Def, "expected 'outs' operator for " + PatFrag::ClassName +
+                        " out operands list");
+    return nullptr;
+  }
+
+  auto Result = std::make_unique<PatFrag>(*Def);
+  if (!parsePatFragParamList(*Outs, [&](StringRef Name, unsigned Kind) {
+        Result->addOutParam(insertStrRef(Name), (PatFrag::ParamKind)Kind);
+        return true;
+      }))
+    return nullptr;
+
+  if (!parsePatFragParamList(*Ins, [&](StringRef Name, unsigned Kind) {
+        Result->addInParam(insertStrRef(Name), (PatFrag::ParamKind)Kind);
+        return true;
+      }))
+    return nullptr;
+
+  const ListInit *Alts = Def->getValueAsListInit("Alternatives");
+  unsigned AltIdx = 0;
+  for (const Init *Alt : *Alts) {
+    const auto *PatDag = dyn_cast<DagInit>(Alt);
+    if (!PatDag) {
+      PrintError(Def, "expected dag init for PatFrag pattern alternative");
+      return nullptr;
+    }
+
+    PatFrag::Alternative &A = Result->addAlternative();
+    const auto AddPat = [&](std::unique_ptr<Pattern> Pat) {
+      A.Pats.push_back(std::move(Pat));
+      return true;
+    };
+
+    SaveAndRestore<ArrayRef<SMLoc>> DiagLocSAR(DiagLoc, Def->getLoc());
+    if (!parsePatternList(
+            *PatDag, AddPat, "pattern",
+            /*AnonPatPrefix*/
+            (Def->getName() + "_alt" + Twine(AltIdx++) + "_pattern").str()))
+      return nullptr;
+  }
+
+  if (!Result->buildOperandsTables() || !Result->checkSemantics())
+    return nullptr;
+
+  return Result;
+}
+
+bool PatternParser::parsePatFragParamList(
+    const DagInit &OpsList,
+    function_ref<bool(StringRef, unsigned)> ParseAction) {
+  for (unsigned K = 0; K < OpsList.getNumArgs(); ++K) {
+    const StringInit *Name = OpsList.getArgName(K);
+    const Init *Ty = OpsList.getArg(K);
+
+    if (!Name) {
+      PrintError(DiagLoc, "all operands must be named'");
+      return false;
+    }
+    const std::string NameStr = Name->getAsUnquotedString();
+
+    PatFrag::ParamKind OpKind;
+    if (isSpecificDef(*Ty, "gi_imm"))
+      OpKind = PatFrag::PK_Imm;
+    else if (isSpecificDef(*Ty, "root"))
+      OpKind = PatFrag::PK_Root;
+    else if (isa<UnsetInit>(Ty) ||
+             isSpecificDef(*Ty, "gi_mo")) // no type = gi_mo.
+      OpKind = PatFrag::PK_MachineOperand;
+    else {
+      PrintError(
+          DiagLoc,
+          '\'' + NameStr +
+              "' operand type was expected to be 'root', 'gi_imm' or 'gi_mo'");
+      return false;
+    }
+
+    if (!ParseAction(NameStr, (unsigned)OpKind))
+      return false;
+  }
+
+  return true;
+}
+
+const PatFrag *PatternParser::parsePatFrag(const Record *Def) {
+  // Cache already parsed PatFrags to avoid doing extra work.
+  static DenseMap<const Record *, std::unique_ptr<PatFrag>> ParsedPatFrags;
+
+  auto It = ParsedPatFrags.find(Def);
+  if (It != ParsedPatFrags.end()) {
+    SeenPatFrags.insert(It->second.get());
+    return It->second.get();
+  }
+
+  std::unique_ptr<PatFrag> NewPatFrag = parsePatFragImpl(Def);
+  if (!NewPatFrag) {
+    PrintError(Def, "Could not parse " + PatFrag::ClassName + " '" +
+                        Def->getName() + "'");
+    // Put a nullptr in the map so we don't attempt parsing this again.
+    ParsedPatFrags[Def] = nullptr;
+    return nullptr;
+  }
+
+  const auto *Res = NewPatFrag.get();
+  ParsedPatFrags[Def] = std::move(NewPatFrag);
+  SeenPatFrags.insert(Res);
+  return Res;
+}
+
+} // namespace gi
+} // namespace llvm
diff --git a/llvm/utils/TableGen/Common/GlobalISel/PatternParser.h b/llvm/utils/TableGen/Common/GlobalISel/PatternParser.h
new file mode 100644
index 000000000000..cd6f524075cd
--- /dev/null
+++ b/llvm/utils/TableGen/Common/GlobalISel/PatternParser.h
@@ -0,0 +1,118 @@
+//===- PatternParser.h ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file Contains tools to parse MIR patterns from TableGen DAG elements.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_UTILS_GLOBALISEL_PATTERNPARSER_H
+#define LLVM_UTILS_GLOBALISEL_PATTERNPARSER_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLFunctionalExtras.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Support/SMLoc.h"
+#include <memory>
+
+namespace llvm {
+class CodeGenTarget;
+class DagInit;
+class Init;
+class Record;
+class StringRef;
+class StringInit;
+
+namespace gi {
+class InstructionPattern;
+class Pattern;
+class PatFrag;
+
+/// Helper class to parse MIR Pattern lists.
+///
+/// e.g., `(match (G_FADD $x, $y, $z), (G_FNEG $y, $z))`
+class PatternParser {
+  const CodeGenTarget &CGT;
+  ArrayRef<SMLoc> DiagLoc;
+
+  mutable SmallPtrSet<const PatFrag *, 2> SeenPatFrags;
+
+public:
+  PatternParser(const CodeGenTarget &CGT, ArrayRef<SMLoc> DiagLoc)
+      : CGT(CGT), DiagLoc(DiagLoc) {}
+
+  /// Parses a list of patterns such as:
+  ///   (Operator (Pattern1 ...), (Pattern2 ...))
+  /// \param List         DagInit of the expected pattern list.
+  /// \param ParseAction  Callback to handle a succesfully parsed pattern.
+  /// \param Operator     The name of the operator, e.g. "match"
+  /// \param AnonPatNamePrefix Prefix for anonymous pattern names.
+  /// \return true on success, false on failure.
+  bool
+  parsePatternList(const DagInit &List,
+                   function_ref<bool(std::unique_ptr<Pattern>)> ParseAction,
+                   StringRef Operator, StringRef AnonPatNamePrefix);
+
+  /// \returns all PatFrags encountered by this PatternParser.
+  const auto &getSeenPatFrags() const { return SeenPatFrags; }
+
+private:
+  /// Parse any InstructionPattern from a TableGen Init.
+  /// \param Arg Init to parse.
+  /// \param PatName Name of the pattern that will be parsed.
+  /// \return the parsed pattern on success, nullptr on failure.
+  std::unique_ptr<Pattern> parseInstructionPattern(const Init &Arg,
+                                                   StringRef PatName);
+
+  /// Parse a WipOpcodeMatcher from a TableGen Init.
+  /// \param Arg Init to parse.
+  /// \param PatName Name of the pattern that will be parsed.
+  /// \return the parsed pattern on success, nullptr on failure.
+  std::unique_ptr<Pattern> parseWipMatchOpcodeMatcher(const Init &Arg,
+                                                      StringRef PatName);
+
+  /// Parses an Operand of an InstructionPattern from a TableGen Init.
+  /// \param IP InstructionPattern for which we're parsing.
+  /// \param OpInit Init to parse.
+  /// \param OpName Name of the operand to parse.
+  /// \return true on success, false on failure.
+  bool parseInstructionPatternOperand(InstructionPattern &IP,
+                                      const Init *OpInit,
+                                      const StringInit *OpName);
+
+  /// Parses a MIFlag for an InstructionPattern from a TableGen Init.
+  /// \param IP InstructionPattern for which we're parsing.
+  /// \param Op Init to parse.
+  /// \return true on success, false on failure.
+  bool parseInstructionPatternMIFlags(InstructionPattern &IP,
+                                      const DagInit *Op);
+
+  /// (Uncached) PatFrag parsing implementation.
+  /// \param Def PatFrag def to parsee.
+  /// \return the parsed PatFrag on success, nullptr on failure.
+  std::unique_ptr<PatFrag> parsePatFragImpl(const Record *Def);
+
+  /// Parses the in or out parameter list of a PatFrag.
+  /// \param OpsList Init to parse.
+  /// \param ParseAction Callback on successful parse, with the name of
+  ///                     the parameter and its \ref PatFrag::ParamKind
+  /// \return true on success, false on failure.
+  bool
+  parsePatFragParamList(const DagInit &OpsList,
+                        function_ref<bool(StringRef, unsigned)> ParseAction);
+
+  /// Cached PatFrag parser. This avoids duplicate work by keeping track of
+  /// already-parsed PatFrags.
+  /// \param Def PatFrag def to parsee.
+  /// \return the parsed PatFrag on success, nullptr on failure.
+  const PatFrag *parsePatFrag(const Record *Def);
+};
+
+} // namespace gi
+} // namespace llvm
+
+#endif
diff --git a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
index 39b9f8a2ae17..1ae6efd4a7d0 100644
--- a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
@@ -36,6 +36,7 @@
 #include "Common/GlobalISel/GlobalISelMatchTable.h"
 #include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h"
 #include "Common/GlobalISel/MatchDataInfo.h"
+#include "Common/GlobalISel/PatternParser.h"
 #include "Common/GlobalISel/Patterns.h"
 #include "Common/SubtargetFeatureInfo.h"
 #include "llvm/ADT/APInt.h"
@@ -80,7 +81,6 @@ cl::opt<bool> DebugTypeInfer("gicombiner-debug-typeinfer",
 
 constexpr StringLiteral CXXApplyPrefix = "GICXXCustomAction_CombineApply";
 constexpr StringLiteral CXXPredPrefix = "GICXXPred_MI_Predicate_";
-constexpr StringLiteral MIFlagsEnumClassName = "MIFlagEnum";
 
 //===- CodeExpansions Helpers  --------------------------------------------===//
 
@@ -109,17 +109,6 @@ void declareTempRegExpansion(CodeExpansions &CE, unsigned TempRegID,
 
 //===- Misc. Helpers  -----------------------------------------------------===//
 
-/// Copies a StringRef into a static pool to preserve it.
-/// Most Pattern classes use StringRef so we need this.
-StringRef insertStrRef(StringRef S) {
-  if (S.empty())
-    return {};
-
-  static StringSet<> Pool;
-  auto [It, Inserted] = Pool.insert(S);
-  return It->getKey();
-}
-
 template <typename Container> auto keys(Container &&C) {
   return map_range(C, [](auto &Entry) -> auto & { return Entry.first; });
 }
@@ -639,8 +628,9 @@ public:
                      SubtargetFeatureInfoMap &SubtargetFeatures,
                      Record &RuleDef, unsigned ID,
                      std::vector<RuleMatcher> &OutRMs)
-      : CGT(CGT), SubtargetFeatures(SubtargetFeatures), RuleDef(RuleDef),
-        RuleID(ID), OutRMs(OutRMs) {}
+      : Parser(CGT, RuleDef.getLoc()), CGT(CGT),
+        SubtargetFeatures(SubtargetFeatures), RuleDef(RuleDef), RuleID(ID),
+        OutRMs(OutRMs) {}
 
   /// Parses all fields in the RuleDef record.
   bool parseAll();
@@ -718,26 +708,6 @@ private:
   bool buildRuleOperandsTable();
 
   bool parseDefs(const DagInit &Def);
-  bool
-  parsePatternList(const DagInit &List,
-                   function_ref<bool(std::unique_ptr<Pattern>)> ParseAction,
-                   StringRef Operator, ArrayRef<SMLoc> DiagLoc,
-                   StringRef AnonPatNamePrefix) const;
-
-  std::unique_ptr<Pattern> parseInstructionPattern(const Init &Arg,
-                                                   StringRef PatName) const;
-  std::unique_ptr<Pattern> parseWipMatchOpcodeMatcher(const Init &Arg,
-                                                      StringRef PatName) const;
-  bool parseInstructionPatternOperand(InstructionPattern &IP,
-                                      const Init *OpInit,
-                                      const StringInit *OpName) const;
-  bool parseInstructionPatternMIFlags(InstructionPattern &IP,
-                                      const DagInit *Op) const;
-  std::unique_ptr<PatFrag> parsePatFragImpl(const Record *Def) const;
-  bool parsePatFragParamList(
-      ArrayRef<SMLoc> DiagLoc, const DagInit &OpsList,
-      function_ref<bool(StringRef, PatFrag::ParamKind)> ParseAction) const;
-  const PatFrag *parsePatFrag(const Record *Def) const;
 
   bool emitMatchPattern(CodeExpansions &CE, const PatternAlternatives &Alts,
                         const InstructionPattern &IP);
@@ -781,6 +751,7 @@ private:
       DenseSet<const Pattern *> &SeenPats, OperandDefLookupFn LookupOperandDef,
       OperandMapperFnRef OperandMapper = [](const auto &O) { return O; });
 
+  PatternParser Parser;
   const CodeGenTarget &CGT;
   SubtargetFeatureInfoMap &SubtargetFeatures;
   Record &RuleDef;
@@ -808,9 +779,6 @@ private:
 
   SmallVector<MatchDataInfo, 2> MatchDatas;
   SmallVector<PatternAlternatives, 1> PermutationsToEmit;
-
-  // print()/debug-only members.
-  mutable SmallPtrSet<const PatFrag *, 2> SeenPatFrags;
 };
 
 bool CombineRuleBuilder::parseAll() {
@@ -819,16 +787,16 @@ bool CombineRuleBuilder::parseAll() {
   if (!parseDefs(*RuleDef.getValueAsDag("Defs")))
     return false;
 
-  if (!parsePatternList(
+  if (!Parser.parsePatternList(
           *RuleDef.getValueAsDag("Match"),
           [this](auto Pat) { return addMatchPattern(std::move(Pat)); }, "match",
-          RuleDef.getLoc(), (RuleDef.getName() + "_match").str()))
+          (RuleDef.getName() + "_match").str()))
     return false;
 
-  if (!parsePatternList(
+  if (!Parser.parsePatternList(
           *RuleDef.getValueAsDag("Apply"),
           [this](auto Pat) { return addApplyPattern(std::move(Pat)); }, "apply",
-          RuleDef.getLoc(), (RuleDef.getName() + "_apply").str()))
+          (RuleDef.getName() + "_apply").str()))
     return false;
 
   if (!buildRuleOperandsTable() || !typecheckPatterns() || !findRoots() ||
@@ -884,9 +852,10 @@ void CombineRuleBuilder::print(raw_ostream &OS) const {
     OS << "  )\n";
   }
 
-  if (!SeenPatFrags.empty()) {
+  const auto &SeenPFs = Parser.getSeenPatFrags();
+  if (!SeenPFs.empty()) {
     OS << "  (PatFrags\n";
-    for (const auto *PF : SeenPatFrags) {
+    for (const auto *PF : Parser.getSeenPatFrags()) {
       PF->print(OS, /*Indent=*/"    ");
       OS << '\n';
     }
@@ -1500,426 +1469,6 @@ bool CombineRuleBuilder::parseDefs(const DagInit &Def) {
   return true;
 }
 
-bool CombineRuleBuilder::parsePatternList(
-    const DagInit &List,
-    function_ref<bool(std::unique_ptr<Pattern>)> ParseAction,
-    StringRef Operator, ArrayRef<SMLoc> DiagLoc,
-    StringRef AnonPatNamePrefix) const {
-  if (List.getOperatorAsDef(RuleDef.getLoc())->getName() != Operator) {
-    ::PrintError(DiagLoc, "Expected " + Operator + " operator");
-    return false;
-  }
-
-  if (List.getNumArgs() == 0) {
-    ::PrintError(DiagLoc, Operator + " pattern list is empty");
-    return false;
-  }
-
-  // The match section consists of a list of matchers and predicates. Parse each
-  // one and add the equivalent GIMatchDag nodes, predicates, and edges.
-  for (unsigned I = 0; I < List.getNumArgs(); ++I) {
-    Init *Arg = List.getArg(I);
-    std::string Name = List.getArgName(I)
-                           ? List.getArgName(I)->getValue().str()
-                           : ("__" + AnonPatNamePrefix + "_" + Twine(I)).str();
-
-    if (auto Pat = parseInstructionPattern(*Arg, Name)) {
-      if (!ParseAction(std::move(Pat)))
-        return false;
-      continue;
-    }
-
-    if (auto Pat = parseWipMatchOpcodeMatcher(*Arg, Name)) {
-      if (!ParseAction(std::move(Pat)))
-        return false;
-      continue;
-    }
-
-    // Parse arbitrary C++ code
-    if (const auto *StringI = dyn_cast<StringInit>(Arg)) {
-      auto CXXPat = std::make_unique<CXXPattern>(*StringI, insertStrRef(Name));
-      if (!ParseAction(std::move(CXXPat)))
-        return false;
-      continue;
-    }
-
-    ::PrintError(DiagLoc,
-                 "Failed to parse pattern: '" + Arg->getAsString() + "'");
-    return false;
-  }
-
-  return true;
-}
-
-static const CodeGenInstruction &
-getInstrForIntrinsic(const CodeGenTarget &CGT, const CodeGenIntrinsic *I) {
-  StringRef Opc;
-  if (I->isConvergent) {
-    Opc = I->hasSideEffects ? "G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS"
-                            : "G_INTRINSIC_CONVERGENT";
-  } else {
-    Opc = I->hasSideEffects ? "G_INTRINSIC_W_SIDE_EFFECTS" : "G_INTRINSIC";
-  }
-
-  RecordKeeper &RK = I->TheDef->getRecords();
-  return CGT.getInstruction(RK.getDef(Opc));
-}
-
-static const CodeGenIntrinsic *getCodeGenIntrinsic(Record *R) {
-  // Intrinsics need to have a static lifetime because the match table keeps
-  // references to CodeGenIntrinsic objects.
-  static DenseMap<const Record *, std::unique_ptr<CodeGenIntrinsic>>
-      AllIntrinsics;
-
-  auto &Ptr = AllIntrinsics[R];
-  if (!Ptr)
-    Ptr = std::make_unique<CodeGenIntrinsic>(R, std::vector<Record *>());
-  return Ptr.get();
-}
-
-std::unique_ptr<Pattern>
-CombineRuleBuilder::parseInstructionPattern(const Init &Arg,
-                                            StringRef Name) const {
-  const DagInit *DagPat = dyn_cast<DagInit>(&Arg);
-  if (!DagPat)
-    return nullptr;
-
-  std::unique_ptr<InstructionPattern> Pat;
-  if (const DagInit *IP = getDagWithOperatorOfSubClass(Arg, "Instruction")) {
-    auto &Instr = CGT.getInstruction(IP->getOperatorAsDef(RuleDef.getLoc()));
-    Pat =
-        std::make_unique<CodeGenInstructionPattern>(Instr, insertStrRef(Name));
-  } else if (const DagInit *IP =
-                 getDagWithOperatorOfSubClass(Arg, "Intrinsic")) {
-    Record *TheDef = IP->getOperatorAsDef(RuleDef.getLoc());
-    const CodeGenIntrinsic *Intrin = getCodeGenIntrinsic(TheDef);
-    const CodeGenInstruction &Instr = getInstrForIntrinsic(CGT, Intrin);
-    Pat =
-        std::make_unique<CodeGenInstructionPattern>(Instr, insertStrRef(Name));
-    cast<CodeGenInstructionPattern>(*Pat).setIntrinsic(Intrin);
-  } else if (const DagInit *PFP =
-                 getDagWithOperatorOfSubClass(Arg, PatFrag::ClassName)) {
-    const Record *Def = PFP->getOperatorAsDef(RuleDef.getLoc());
-    const PatFrag *PF = parsePatFrag(Def);
-    if (!PF)
-      return nullptr; // Already diagnosed by parsePatFrag
-    Pat = std::make_unique<PatFragPattern>(*PF, insertStrRef(Name));
-  } else if (const DagInit *BP =
-                 getDagWithOperatorOfSubClass(Arg, BuiltinPattern::ClassName)) {
-    Pat = std::make_unique<BuiltinPattern>(
-        *BP->getOperatorAsDef(RuleDef.getLoc()), insertStrRef(Name));
-  } else
-    return nullptr;
-
-  for (unsigned K = 0; K < DagPat->getNumArgs(); ++K) {
-    Init *Arg = DagPat->getArg(K);
-    if (auto *DagArg = getDagWithSpecificOperator(*Arg, "MIFlags")) {
-      if (!parseInstructionPatternMIFlags(*Pat, DagArg))
-        return nullptr;
-      continue;
-    }
-
-    if (!parseInstructionPatternOperand(*Pat, Arg, DagPat->getArgName(K)))
-      return nullptr;
-  }
-
-  if (!Pat->checkSemantics(RuleDef.getLoc()))
-    return nullptr;
-
-  return std::move(Pat);
-}
-
-std::unique_ptr<Pattern>
-CombineRuleBuilder::parseWipMatchOpcodeMatcher(const Init &Arg,
-                                               StringRef Name) const {
-  const DagInit *Matcher = getDagWithSpecificOperator(Arg, "wip_match_opcode");
-  if (!Matcher)
-    return nullptr;
-
-  if (Matcher->getNumArgs() == 0) {
-    PrintError("Empty wip_match_opcode");
-    return nullptr;
-  }
-
-  // Each argument is an opcode that can match.
-  auto Result = std::make_unique<AnyOpcodePattern>(insertStrRef(Name));
-  for (const auto &Arg : Matcher->getArgs()) {
-    Record *OpcodeDef = getDefOfSubClass(*Arg, "Instruction");
-    if (OpcodeDef) {
-      Result->addOpcode(&CGT.getInstruction(OpcodeDef));
-      continue;
-    }
-
-    PrintError("Arguments to wip_match_opcode must be instructions");
-    return nullptr;
-  }
-
-  return std::move(Result);
-}
-
-bool CombineRuleBuilder::parseInstructionPatternOperand(
-    InstructionPattern &IP, const Init *OpInit,
-    const StringInit *OpName) const {
-  const auto ParseErr = [&]() {
-    PrintError("cannot parse operand '" + OpInit->getAsUnquotedString() + "' ");
-    if (OpName)
-      PrintNote("operand name is '" + OpName->getAsUnquotedString() + "'");
-    return false;
-  };
-
-  // untyped immediate, e.g. 0
-  if (const auto *IntImm = dyn_cast<IntInit>(OpInit)) {
-    std::string Name = OpName ? OpName->getAsUnquotedString() : "";
-    IP.addOperand(IntImm->getValue(), insertStrRef(Name), PatternType());
-    return true;
-  }
-
-  // typed immediate, e.g. (i32 0)
-  if (const auto *DagOp = dyn_cast<DagInit>(OpInit)) {
-    if (DagOp->getNumArgs() != 1)
-      return ParseErr();
-
-    const Record *TyDef = DagOp->getOperatorAsDef(RuleDef.getLoc());
-    auto ImmTy = PatternType::get(RuleDef.getLoc(), TyDef,
-                                  "cannot parse immediate '" +
-                                      DagOp->getAsUnquotedString() + "'");
-    if (!ImmTy)
-      return false;
-
-    if (!IP.hasAllDefs()) {
-      PrintError("out operand of '" + IP.getInstName() +
-                 "' cannot be an immediate");
-      return false;
-    }
-
-    const auto *Val = dyn_cast<IntInit>(DagOp->getArg(0));
-    if (!Val)
-      return ParseErr();
-
-    std::string Name = OpName ? OpName->getAsUnquotedString() : "";
-    IP.addOperand(Val->getValue(), insertStrRef(Name), *ImmTy);
-    return true;
-  }
-
-  // Typed operand e.g. $x/$z in (G_FNEG $x, $z)
-  if (auto *DefI = dyn_cast<DefInit>(OpInit)) {
-    if (!OpName) {
-      PrintError("expected an operand name after '" + OpInit->getAsString() +
-                 "'");
-      return false;
-    }
-    const Record *Def = DefI->getDef();
-    auto Ty =
-        PatternType::get(RuleDef.getLoc(), Def, "cannot parse operand type");
-    if (!Ty)
-      return false;
-    IP.addOperand(insertStrRef(OpName->getAsUnquotedString()), *Ty);
-    return true;
-  }
-
-  // Untyped operand e.g. $x/$z in (G_FNEG $x, $z)
-  if (isa<UnsetInit>(OpInit)) {
-    assert(OpName && "Unset w/ no OpName?");
-    IP.addOperand(insertStrRef(OpName->getAsUnquotedString()), PatternType());
-    return true;
-  }
-
-  return ParseErr();
-}
-
-bool CombineRuleBuilder::parseInstructionPatternMIFlags(
-    InstructionPattern &IP, const DagInit *Op) const {
-  auto *CGIP = dyn_cast<CodeGenInstructionPattern>(&IP);
-  if (!CGIP) {
-    PrintError("matching/writing MIFlags is only allowed on CodeGenInstruction "
-               "patterns");
-    return false;
-  }
-
-  const auto CheckFlagEnum = [&](const Record *R) {
-    if (!R->isSubClassOf(MIFlagsEnumClassName)) {
-      PrintError("'" + R->getName() + "' is not a subclass of '" +
-                 MIFlagsEnumClassName + "'");
-      return false;
-    }
-
-    return true;
-  };
-
-  if (CGIP->getMIFlagsInfo()) {
-    PrintError("MIFlags can only be present once on an instruction");
-    return false;
-  }
-
-  auto &FI = CGIP->getOrCreateMIFlagsInfo();
-  for (unsigned K = 0; K < Op->getNumArgs(); ++K) {
-    const Init *Arg = Op->getArg(K);
-
-    // Match/set a flag: (MIFlags FmNoNans)
-    if (const auto *Def = dyn_cast<DefInit>(Arg)) {
-      const Record *R = Def->getDef();
-      if (!CheckFlagEnum(R))
-        return false;
-
-      FI.addSetFlag(R);
-      continue;
-    }
-
-    // Do not match a flag/unset a flag: (MIFlags (not FmNoNans))
-    if (const DagInit *NotDag = getDagWithSpecificOperator(*Arg, "not")) {
-      for (const Init *NotArg : NotDag->getArgs()) {
-        const DefInit *DefArg = dyn_cast<DefInit>(NotArg);
-        if (!DefArg) {
-          PrintError("cannot parse '" + NotArg->getAsUnquotedString() +
-                     "': expected a '" + MIFlagsEnumClassName + "'");
-          return false;
-        }
-
-        const Record *R = DefArg->getDef();
-        if (!CheckFlagEnum(R))
-          return false;
-
-        FI.addUnsetFlag(R);
-        continue;
-      }
-
-      continue;
-    }
-
-    // Copy flags from a matched instruction: (MIFlags $mi)
-    if (isa<UnsetInit>(Arg)) {
-      FI.addCopyFlag(insertStrRef(Op->getArgName(K)->getAsUnquotedString()));
-      continue;
-    }
-  }
-
-  return true;
-}
-
-std::unique_ptr<PatFrag>
-CombineRuleBuilder::parsePatFragImpl(const Record *Def) const {
-  auto StackTrace = PrettyStackTraceParse(*Def);
-  if (!Def->isSubClassOf(PatFrag::ClassName))
-    return nullptr;
-
-  const DagInit *Ins = Def->getValueAsDag("InOperands");
-  if (Ins->getOperatorAsDef(Def->getLoc())->getName() != "ins") {
-    ::PrintError(Def, "expected 'ins' operator for " + PatFrag::ClassName +
-                          " in operands list");
-    return nullptr;
-  }
-
-  const DagInit *Outs = Def->getValueAsDag("OutOperands");
-  if (Outs->getOperatorAsDef(Def->getLoc())->getName() != "outs") {
-    ::PrintError(Def, "expected 'outs' operator for " + PatFrag::ClassName +
-                          " out operands list");
-    return nullptr;
-  }
-
-  auto Result = std::make_unique<PatFrag>(*Def);
-  if (!parsePatFragParamList(Def->getLoc(), *Outs,
-                             [&](StringRef Name, PatFrag::ParamKind Kind) {
-                               Result->addOutParam(insertStrRef(Name), Kind);
-                               return true;
-                             }))
-    return nullptr;
-
-  if (!parsePatFragParamList(Def->getLoc(), *Ins,
-                             [&](StringRef Name, PatFrag::ParamKind Kind) {
-                               Result->addInParam(insertStrRef(Name), Kind);
-                               return true;
-                             }))
-    return nullptr;
-
-  const ListInit *Alts = Def->getValueAsListInit("Alternatives");
-  unsigned AltIdx = 0;
-  for (const Init *Alt : *Alts) {
-    const auto *PatDag = dyn_cast<DagInit>(Alt);
-    if (!PatDag) {
-      ::PrintError(Def, "expected dag init for PatFrag pattern alternative");
-      return nullptr;
-    }
-
-    PatFrag::Alternative &A = Result->addAlternative();
-    const auto AddPat = [&](std::unique_ptr<Pattern> Pat) {
-      A.Pats.push_back(std::move(Pat));
-      return true;
-    };
-
-    if (!parsePatternList(
-            *PatDag, AddPat, "pattern", Def->getLoc(),
-            /*AnonPatPrefix*/
-            (Def->getName() + "_alt" + Twine(AltIdx++) + "_pattern").str()))
-      return nullptr;
-  }
-
-  if (!Result->buildOperandsTables() || !Result->checkSemantics())
-    return nullptr;
-
-  return Result;
-}
-
-bool CombineRuleBuilder::parsePatFragParamList(
-    ArrayRef<SMLoc> DiagLoc, const DagInit &OpsList,
-    function_ref<bool(StringRef, PatFrag::ParamKind)> ParseAction) const {
-  for (unsigned K = 0; K < OpsList.getNumArgs(); ++K) {
-    const StringInit *Name = OpsList.getArgName(K);
-    const Init *Ty = OpsList.getArg(K);
-
-    if (!Name) {
-      ::PrintError(DiagLoc, "all operands must be named'");
-      return false;
-    }
-    const std::string NameStr = Name->getAsUnquotedString();
-
-    PatFrag::ParamKind OpKind;
-    if (isSpecificDef(*Ty, "gi_imm"))
-      OpKind = PatFrag::PK_Imm;
-    else if (isSpecificDef(*Ty, "root"))
-      OpKind = PatFrag::PK_Root;
-    else if (isa<UnsetInit>(Ty) ||
-             isSpecificDef(*Ty, "gi_mo")) // no type = gi_mo.
-      OpKind = PatFrag::PK_MachineOperand;
-    else {
-      ::PrintError(
-          DiagLoc,
-          "'" + NameStr +
-              "' operand type was expected to be 'root', 'gi_imm' or 'gi_mo'");
-      return false;
-    }
-
-    if (!ParseAction(NameStr, OpKind))
-      return false;
-  }
-
-  return true;
-}
-
-const PatFrag *CombineRuleBuilder::parsePatFrag(const Record *Def) const {
-  // Cache already parsed PatFrags to avoid doing extra work.
-  static DenseMap<const Record *, std::unique_ptr<PatFrag>> ParsedPatFrags;
-
-  auto It = ParsedPatFrags.find(Def);
-  if (It != ParsedPatFrags.end()) {
-    SeenPatFrags.insert(It->second.get());
-    return It->second.get();
-  }
-
-  std::unique_ptr<PatFrag> NewPatFrag = parsePatFragImpl(Def);
-  if (!NewPatFrag) {
-    ::PrintError(Def, "Could not parse " + PatFrag::ClassName + " '" +
-                          Def->getName() + "'");
-    // Put a nullptr in the map so we don't attempt parsing this again.
-    ParsedPatFrags[Def] = nullptr;
-    return nullptr;
-  }
-
-  const auto *Res = NewPatFrag.get();
-  ParsedPatFrags[Def] = std::move(NewPatFrag);
-  SeenPatFrags.insert(Res);
-  return Res;
-}
-
 bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE,
                                           const PatternAlternatives &Alts,
                                           const InstructionPattern &IP) {
@@ -2956,8 +2505,8 @@ GICombinerEmitter::buildMatchTable(MutableArrayRef<RuleMatcher> Rules) {
                                                const Matcher *B) {
     auto *L = static_cast<const RuleMatcher *>(A);
     auto *R = static_cast<const RuleMatcher *>(B);
-    return std::tuple(OpcodeOrder[L->getOpcode()], L->getNumOperands()) <
-           std::tuple(OpcodeOrder[R->getOpcode()], R->getNumOperands());
+    return std::make_tuple(OpcodeOrder[L->getOpcode()], L->getNumOperands()) <
+           std::make_tuple(OpcodeOrder[R->getOpcode()], R->getNumOperands());
   });
 
   for (Matcher *Rule : InputRules)
-- 
GitLab


From 26464f2662d13c7c6ef9f8180b1653c046cd60a7 Mon Sep 17 00:00:00 2001
From: Justin Cady <desk@justincady.com>
Date: Wed, 27 Mar 2024 09:03:46 -0400
Subject: [PATCH 402/404] [FreeBSD] Mark __stack_chk_guard dso_local except for
 PPC64 (#86665)

Adjust logic of 1cb9f37a17ab to match freebsd/freebsd-src@9a4d48a645a7a.

D113443 is the original attempt to bring this FreeBSD patch to
llvm-project,
but it never landed. This change is required to build FreeBSD kernel
modules
with -fstack-protector using a standard LLVM toolchain. The FreeBSD
kernel
loader does not handle R_X86_64_REX_GOTPCRELX relocations.

Fixes #50932.
---
 llvm/lib/CodeGen/TargetLoweringBase.cpp  | 3 ++-
 llvm/test/CodeGen/X86/stack-protector.ll | 9 +++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 9990556f89ed..b16e78daf586 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -2073,7 +2073,8 @@ void TargetLoweringBase::insertSSPDeclarations(Module &M) const {
     // FreeBSD has "__stack_chk_guard" defined externally on libc.so
     if (M.getDirectAccessExternalData() &&
         !TM.getTargetTriple().isWindowsGNUEnvironment() &&
-        !TM.getTargetTriple().isOSFreeBSD() &&
+        !(TM.getTargetTriple().isPPC64() &&
+          TM.getTargetTriple().isOSFreeBSD()) &&
         (!TM.getTargetTriple().isOSDarwin() ||
          TM.getRelocationModel() == Reloc::Static))
       GV->setDSOLocal(true);
diff --git a/llvm/test/CodeGen/X86/stack-protector.ll b/llvm/test/CodeGen/X86/stack-protector.ll
index a277f9f862ab..f4f3ae4f55f2 100644
--- a/llvm/test/CodeGen/X86/stack-protector.ll
+++ b/llvm/test/CodeGen/X86/stack-protector.ll
@@ -1,6 +1,7 @@
 ; RUN: llc -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck --check-prefix=LINUX-I386 %s
 ; RUN: llc -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck --check-prefix=LINUX-X64 %s
 ; RUN: llc -code-model=kernel -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck --check-prefix=LINUX-KERNEL-X64 %s
+; RUN: llc -code-model=kernel -mtriple=x86_64-unknown-freebsd < %s -o - | FileCheck --check-prefix=FREEBSD-KERNEL-X64 %s
 ; RUN: llc -mtriple=x86_64-apple-darwin < %s -o - | FileCheck --check-prefix=DARWIN-X64 %s
 ; RUN: llc -mtriple=amd64-pc-openbsd < %s -o - | FileCheck --check-prefix=OPENBSD-AMD64 %s
 ; RUN: llc -mtriple=i386-pc-windows-msvc < %s -o - | FileCheck -check-prefix=MSVC-I386 %s
@@ -75,6 +76,10 @@ entry:
 ; LINUX-X64: mov{{l|q}} %fs:
 ; LINUX-X64: callq __stack_chk_fail
 
+; FREEBSD-KERNEL-X64-LABEL: test1b:
+; FREEBSD-KERNEL-X64-NOT: mov{{l|q}} __stack_chk_guard@GOTPCREL
+; FREEBSD-KERNEL-X64: callq __stack_chk_fail
+
 ; LINUX-KERNEL-X64-LABEL: test1b:
 ; LINUX-KERNEL-X64: mov{{l|q}} %gs:
 ; LINUX-KERNEL-X64: callq __stack_chk_fail
@@ -118,6 +123,10 @@ entry:
 ; LINUX-X64: mov{{l|q}} %fs:
 ; LINUX-X64: callq __stack_chk_fail
 
+; FREEBSD-KERNEL-X64-LABEL: test1c:
+; FREEBSD-KERNEL-X64: mov{{l|q}} __stack_chk_guard(%rip)
+; FREEBSD-KERNEL-X64: callq __stack_chk_fail
+
 ; LINUX-KERNEL-X64-LABEL: test1c:
 ; LINUX-KERNEL-X64: mov{{l|q}} %gs:
 ; LINUX-KERNEL-X64: callq __stack_chk_fail
-- 
GitLab


From 932949dbb517b089af28fdc480a16a738ee5db78 Mon Sep 17 00:00:00 2001
From: Egor Zhdan <e_zhdan@apple.com>
Date: Wed, 27 Mar 2024 13:13:06 +0000
Subject: [PATCH 403/404] [APINotes] Upstream the remaining API Notes fixes and
 tests

This upstreams the last bits of Clang API Notes functionality that is
currently implemented in the Apple fork:
https://github.com/apple/llvm-project/tree/next/clang/lib/APINotes
---
 clang/lib/Sema/SemaAPINotes.cpp               |  41 ++--
 clang/lib/Sema/SemaObjCProperty.cpp           |   4 +-
 .../Inputs/APINotes/SomeOtherKit.apinotes     |   8 +
 .../Inputs/BrokenHeaders/APINotes.apinotes    |   5 +
 .../Inputs/BrokenHeaders/SomeBrokenLib.h      |   6 +
 .../Inputs/BrokenHeaders2/APINotes.apinotes   |   7 +
 .../Inputs/BrokenHeaders2/SomeBrokenLib.h     |   6 +
 .../FrameworkWithActualPrivateModule.h        |   1 +
 .../Modules/module.modulemap                  |   5 +
 .../Modules/module.private.modulemap          |   5 +
 ...rkWithActualPrivateModule_Private.apinotes |   1 +
 ...FrameworkWithActualPrivateModule_Private.h |   2 +
 .../Headers/FrameworkWithWrongCase.h          |   1 +
 .../Modules/module.modulemap                  |   5 +
 .../FrameworkWithWrongCase_Private.apinotes   |   1 +
 .../Headers/FrameworkWithWrongCasePrivate.h   |   1 +
 .../Modules/module.modulemap                  |   5 +
 .../Modules/module.private.modulemap          |   1 +
 ...eworkWithWrongCasePrivate_Private.apinotes |   1 +
 .../LayeredKit.framework/Headers/LayeredKit.h |  11 ++
 .../Modules/module.modulemap                  |   5 +
 .../Headers/LayeredKitImpl.apinotes           |   9 +
 .../Headers/LayeredKitImpl.h                  |   7 +
 .../Modules/module.modulemap                  |   5 +
 .../Modules/module.modulemap                  |   5 +
 .../APINotes/SomeKit.apinotes                 |  74 +++++++
 .../APINotes/SomeKit_private.apinotes         |  15 ++
 .../Headers/SomeKitForNullAnnotation.h        |  55 ++++++
 .../Modules/module.modulemap                  |   5 +
 .../Modules/module.private.modulemap          |   8 +
 .../Modules/module_private.modulemap          |   8 +
 .../PrivateHeaders/SomeKit_Private.h          |  16 ++
 .../SomeKit_PrivateForNullAnnotation.h        |  17 ++
 .../PrivateHeaders/SomeKit_private.apinotes   |  15 ++
 .../APINotes/SomeOtherKit.apinotes            |   8 +
 .../Headers/SomeOtherKit.apinotes             |   8 +
 .../Headers/SomeOtherKit.h                    |   9 +
 .../Modules/module.modulemap                  |   5 +
 .../Headers/TopLevelPrivateKit.h              |   1 +
 .../TopLevelPrivateKit_Private.apinotes       |   1 +
 .../Modules/module.modulemap                  |   5 +
 .../Modules/module.private.modulemap          |   5 +
 .../TopLevelPrivateKit.apinotes               |   1 +
 .../TopLevelPrivateKit_Private.apinotes       |   4 +
 .../TopLevelPrivateKit_Private.h              |   1 +
 ...opLevelPrivateKit_Private_private.apinotes |   1 +
 .../Headers/VersionedKit.apinotes             | 156 +++++++++++++++
 .../Headers/VersionedKit.h                    | 137 +++++++++++++
 .../Modules/module.modulemap                  |   5 +
 .../APINotes/Inputs/Headers/APINotes.apinotes |  18 ++
 .../Inputs/Headers/BrokenTypes.apinotes       |  10 +
 .../APINotes/Inputs/Headers/BrokenTypes.h     |   8 +
 .../Inputs/Headers/ExternCtx.apinotes         |  15 ++
 .../test/APINotes/Inputs/Headers/ExternCtx.h  |  11 ++
 .../Inputs/Headers/HeaderLib.apinotes         |  37 ++++
 .../test/APINotes/Inputs/Headers/HeaderLib.h  |  19 ++
 .../Headers/InstancetypeModule.apinotes       |  10 +
 .../Inputs/Headers/InstancetypeModule.h       |  10 +
 .../Inputs/Headers/ModuleWithWrongCase.h      |   1 +
 .../Headers/ModuleWithWrongCasePrivate.h      |   1 +
 ...oduleWithWrongCasePrivate_Private.apinotes |   1 +
 .../ModuleWithWrongCase_Private.apinotes      |   1 +
 .../Inputs/Headers/Namespaces.apinotes        |  53 +++++
 .../test/APINotes/Inputs/Headers/Namespaces.h |  39 ++++
 .../Inputs/Headers/PrivateLib.apinotes        |   4 +
 .../test/APINotes/Inputs/Headers/PrivateLib.h |   1 +
 .../Headers/PrivateLib_private.apinotes       |   1 +
 .../Inputs/Headers/SwiftImportAs.apinotes     |   9 +
 .../APINotes/Inputs/Headers/SwiftImportAs.h   |   6 +
 .../APINotes/Inputs/Headers/module.modulemap  |  31 +++
 .../Inputs/Headers/module.private.modulemap   |   5 +
 .../Inputs/yaml-reader-errors/UIKit.apinotes  |  65 ++++++
 .../Inputs/yaml-reader-errors/UIKit.h         |   1 +
 .../yaml-reader-errors/module.modulemap       |   3 +
 clang/test/APINotes/availability.m            |  48 +++++
 clang/test/APINotes/broken_types.m            |  19 ++
 .../APINotes/case-for-private-apinotes-file.c |  22 +++
 clang/test/APINotes/extern-context.cpp        |  23 +++
 clang/test/APINotes/instancetype.m            |   9 +
 clang/test/APINotes/module-cache.m            |  65 ++++++
 clang/test/APINotes/namespaces.cpp            |  69 +++++++
 clang/test/APINotes/nullability.c             |  21 ++
 clang/test/APINotes/nullability.m             |  46 +++++
 .../test/APINotes/objc-forward-declarations.m |  12 ++
 clang/test/APINotes/objc_designated_inits.m   |  17 ++
 clang/test/APINotes/properties.m              |  42 ++++
 clang/test/APINotes/retain-count-convention.m |  38 ++++
 clang/test/APINotes/search-order.m            |  25 +++
 clang/test/APINotes/swift-import-as.cpp       |  16 ++
 .../test/APINotes/top-level-private-modules.c |   8 +
 clang/test/APINotes/types.m                   |  28 +++
 clang/test/APINotes/versioned-multi.c         |  69 +++++++
 clang/test/APINotes/versioned.m               | 187 ++++++++++++++++++
 clang/test/APINotes/yaml-convert-diags.c      |   6 +
 clang/test/APINotes/yaml-parse-diags.c        |   6 +
 clang/test/APINotes/yaml-reader-errors.m      |   5 +
 96 files changed, 1829 insertions(+), 20 deletions(-)
 create mode 100644 clang/test/APINotes/Inputs/APINotes/SomeOtherKit.apinotes
 create mode 100644 clang/test/APINotes/Inputs/BrokenHeaders/APINotes.apinotes
 create mode 100644 clang/test/APINotes/Inputs/BrokenHeaders/SomeBrokenLib.h
 create mode 100644 clang/test/APINotes/Inputs/BrokenHeaders2/APINotes.apinotes
 create mode 100644 clang/test/APINotes/Inputs/BrokenHeaders2/SomeBrokenLib.h
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/Headers/FrameworkWithActualPrivateModule.h
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/Modules/module.modulemap
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/Modules/module.private.modulemap
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/PrivateHeaders/FrameworkWithActualPrivateModule_Private.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/PrivateHeaders/FrameworkWithActualPrivateModule_Private.h
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCase.framework/Headers/FrameworkWithWrongCase.h
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCase.framework/Modules/module.modulemap
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCase.framework/PrivateHeaders/FrameworkWithWrongCase_Private.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCasePrivate.framework/Headers/FrameworkWithWrongCasePrivate.h
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCasePrivate.framework/Modules/module.modulemap
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCasePrivate.framework/Modules/module.private.modulemap
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCasePrivate.framework/PrivateHeaders/FrameworkWithWrongCasePrivate_Private.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/LayeredKit.framework/Headers/LayeredKit.h
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/LayeredKit.framework/Modules/module.modulemap
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/LayeredKitImpl.framework/Headers/LayeredKitImpl.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/LayeredKitImpl.framework/Headers/LayeredKitImpl.h
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/LayeredKitImpl.framework/Modules/module.modulemap
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/SimpleKit.framework/Modules/module.modulemap
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/APINotes/SomeKit.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/APINotes/SomeKit_private.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/Headers/SomeKitForNullAnnotation.h
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/Modules/module.modulemap
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/Modules/module.private.modulemap
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/Modules/module_private.modulemap
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/PrivateHeaders/SomeKit_Private.h
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/PrivateHeaders/SomeKit_PrivateForNullAnnotation.h
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/PrivateHeaders/SomeKit_private.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/SomeOtherKit.framework/APINotes/SomeOtherKit.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/SomeOtherKit.framework/Headers/SomeOtherKit.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/SomeOtherKit.framework/Headers/SomeOtherKit.h
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/SomeOtherKit.framework/Modules/module.modulemap
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/Headers/TopLevelPrivateKit.h
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/Headers/TopLevelPrivateKit_Private.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/Modules/module.modulemap
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/Modules/module.private.modulemap
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/PrivateHeaders/TopLevelPrivateKit.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/PrivateHeaders/TopLevelPrivateKit_Private.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/PrivateHeaders/TopLevelPrivateKit_Private.h
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/PrivateHeaders/TopLevelPrivateKit_Private_private.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/VersionedKit.framework/Headers/VersionedKit.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/VersionedKit.framework/Headers/VersionedKit.h
 create mode 100644 clang/test/APINotes/Inputs/Frameworks/VersionedKit.framework/Modules/module.modulemap
 create mode 100644 clang/test/APINotes/Inputs/Headers/APINotes.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Headers/BrokenTypes.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Headers/BrokenTypes.h
 create mode 100644 clang/test/APINotes/Inputs/Headers/ExternCtx.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Headers/ExternCtx.h
 create mode 100644 clang/test/APINotes/Inputs/Headers/HeaderLib.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Headers/HeaderLib.h
 create mode 100644 clang/test/APINotes/Inputs/Headers/InstancetypeModule.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Headers/InstancetypeModule.h
 create mode 100644 clang/test/APINotes/Inputs/Headers/ModuleWithWrongCase.h
 create mode 100644 clang/test/APINotes/Inputs/Headers/ModuleWithWrongCasePrivate.h
 create mode 100644 clang/test/APINotes/Inputs/Headers/ModuleWithWrongCasePrivate_Private.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Headers/ModuleWithWrongCase_Private.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Headers/Namespaces.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Headers/Namespaces.h
 create mode 100644 clang/test/APINotes/Inputs/Headers/PrivateLib.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Headers/PrivateLib.h
 create mode 100644 clang/test/APINotes/Inputs/Headers/PrivateLib_private.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Headers/SwiftImportAs.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Headers/SwiftImportAs.h
 create mode 100644 clang/test/APINotes/Inputs/Headers/module.modulemap
 create mode 100644 clang/test/APINotes/Inputs/Headers/module.private.modulemap
 create mode 100644 clang/test/APINotes/Inputs/yaml-reader-errors/UIKit.apinotes
 create mode 100644 clang/test/APINotes/Inputs/yaml-reader-errors/UIKit.h
 create mode 100644 clang/test/APINotes/Inputs/yaml-reader-errors/module.modulemap
 create mode 100644 clang/test/APINotes/availability.m
 create mode 100644 clang/test/APINotes/broken_types.m
 create mode 100644 clang/test/APINotes/case-for-private-apinotes-file.c
 create mode 100644 clang/test/APINotes/extern-context.cpp
 create mode 100644 clang/test/APINotes/instancetype.m
 create mode 100644 clang/test/APINotes/module-cache.m
 create mode 100644 clang/test/APINotes/namespaces.cpp
 create mode 100644 clang/test/APINotes/nullability.c
 create mode 100644 clang/test/APINotes/nullability.m
 create mode 100644 clang/test/APINotes/objc-forward-declarations.m
 create mode 100644 clang/test/APINotes/objc_designated_inits.m
 create mode 100644 clang/test/APINotes/properties.m
 create mode 100644 clang/test/APINotes/retain-count-convention.m
 create mode 100644 clang/test/APINotes/search-order.m
 create mode 100644 clang/test/APINotes/swift-import-as.cpp
 create mode 100644 clang/test/APINotes/top-level-private-modules.c
 create mode 100644 clang/test/APINotes/types.m
 create mode 100644 clang/test/APINotes/versioned-multi.c
 create mode 100644 clang/test/APINotes/versioned.m
 create mode 100644 clang/test/APINotes/yaml-convert-diags.c
 create mode 100644 clang/test/APINotes/yaml-parse-diags.c
 create mode 100644 clang/test/APINotes/yaml-reader-errors.m

diff --git a/clang/lib/Sema/SemaAPINotes.cpp b/clang/lib/Sema/SemaAPINotes.cpp
index 836c633e9d20..a3128306c664 100644
--- a/clang/lib/Sema/SemaAPINotes.cpp
+++ b/clang/lib/Sema/SemaAPINotes.cpp
@@ -52,49 +52,54 @@ static void applyNullability(Sema &S, Decl *D, NullabilityKind Nullability,
   if (!Metadata.IsActive)
     return;
 
-  auto IsModified = [&](Decl *D, QualType QT,
-                        NullabilityKind Nullability) -> bool {
+  auto GetModified =
+      [&](Decl *D, QualType QT,
+          NullabilityKind Nullability) -> std::optional<QualType> {
     QualType Original = QT;
     S.CheckImplicitNullabilityTypeSpecifier(QT, Nullability, D->getLocation(),
                                             isa<ParmVarDecl>(D),
                                             /*OverrideExisting=*/true);
-    return QT.getTypePtr() != Original.getTypePtr();
+    return (QT.getTypePtr() != Original.getTypePtr()) ? std::optional(QT)
+                                                      : std::nullopt;
   };
 
   if (auto Function = dyn_cast<FunctionDecl>(D)) {
-    if (IsModified(D, Function->getReturnType(), Nullability)) {
-      QualType FnType = Function->getType();
-      Function->setType(FnType);
+    if (auto Modified =
+            GetModified(D, Function->getReturnType(), Nullability)) {
+      const FunctionType *FnType = Function->getType()->castAs<FunctionType>();
+      if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(FnType))
+        Function->setType(S.Context.getFunctionType(
+            *Modified, proto->getParamTypes(), proto->getExtProtoInfo()));
+      else
+        Function->setType(
+            S.Context.getFunctionNoProtoType(*Modified, FnType->getExtInfo()));
     }
   } else if (auto Method = dyn_cast<ObjCMethodDecl>(D)) {
-    QualType Type = Method->getReturnType();
-    if (IsModified(D, Type, Nullability)) {
-      Method->setReturnType(Type);
+    if (auto Modified = GetModified(D, Method->getReturnType(), Nullability)) {
+      Method->setReturnType(*Modified);
 
       // Make it a context-sensitive keyword if we can.
-      if (!isIndirectPointerType(Type))
+      if (!isIndirectPointerType(*Modified))
         Method->setObjCDeclQualifier(Decl::ObjCDeclQualifier(
             Method->getObjCDeclQualifier() | Decl::OBJC_TQ_CSNullability));
     }
   } else if (auto Value = dyn_cast<ValueDecl>(D)) {
-    QualType Type = Value->getType();
-    if (IsModified(D, Type, Nullability)) {
-      Value->setType(Type);
+    if (auto Modified = GetModified(D, Value->getType(), Nullability)) {
+      Value->setType(*Modified);
 
       // Make it a context-sensitive keyword if we can.
       if (auto Parm = dyn_cast<ParmVarDecl>(D)) {
-        if (Parm->isObjCMethodParameter() && !isIndirectPointerType(Type))
+        if (Parm->isObjCMethodParameter() && !isIndirectPointerType(*Modified))
           Parm->setObjCDeclQualifier(Decl::ObjCDeclQualifier(
               Parm->getObjCDeclQualifier() | Decl::OBJC_TQ_CSNullability));
       }
     }
   } else if (auto Property = dyn_cast<ObjCPropertyDecl>(D)) {
-    QualType Type = Property->getType();
-    if (IsModified(D, Type, Nullability)) {
-      Property->setType(Type, Property->getTypeSourceInfo());
+    if (auto Modified = GetModified(D, Property->getType(), Nullability)) {
+      Property->setType(*Modified, Property->getTypeSourceInfo());
 
       // Make it a property attribute if we can.
-      if (!isIndirectPointerType(Type))
+      if (!isIndirectPointerType(*Modified))
         Property->setPropertyAttributes(
             ObjCPropertyAttribute::kind_null_resettable);
     }
diff --git a/clang/lib/Sema/SemaObjCProperty.cpp b/clang/lib/Sema/SemaObjCProperty.cpp
index 4636d89ebf2b..f9e1ad0121e2 100644
--- a/clang/lib/Sema/SemaObjCProperty.cpp
+++ b/clang/lib/Sema/SemaObjCProperty.cpp
@@ -638,8 +638,6 @@ ObjCPropertyDecl *Sema::CreatePropertyDecl(Scope *S,
     PDecl->setInvalidDecl();
   }
 
-  ProcessDeclAttributes(S, PDecl, FD.D);
-
   // Regardless of setter/getter attribute, we save the default getter/setter
   // selector names in anticipation of declaration of setter/getter methods.
   PDecl->setGetterName(GetterSel, GetterNameLoc);
@@ -647,6 +645,8 @@ ObjCPropertyDecl *Sema::CreatePropertyDecl(Scope *S,
   PDecl->setPropertyAttributesAsWritten(
                           makePropertyAttributesAsWritten(AttributesAsWritten));
 
+  ProcessDeclAttributes(S, PDecl, FD.D);
+
   if (Attributes & ObjCPropertyAttribute::kind_readonly)
     PDecl->setPropertyAttributes(ObjCPropertyAttribute::kind_readonly);
 
diff --git a/clang/test/APINotes/Inputs/APINotes/SomeOtherKit.apinotes b/clang/test/APINotes/Inputs/APINotes/SomeOtherKit.apinotes
new file mode 100644
index 000000000000..ccdc4e15d34d
--- /dev/null
+++ b/clang/test/APINotes/Inputs/APINotes/SomeOtherKit.apinotes
@@ -0,0 +1,8 @@
+Name: SomeOtherKit
+Classes:
+  - Name: A
+    Methods:
+      - Selector:        "methodB"
+        MethodKind:      Instance
+        Availability:    none
+        AvailabilityMsg: "anything but this"
diff --git a/clang/test/APINotes/Inputs/BrokenHeaders/APINotes.apinotes b/clang/test/APINotes/Inputs/BrokenHeaders/APINotes.apinotes
new file mode 100644
index 000000000000..cd5475b13423
--- /dev/null
+++ b/clang/test/APINotes/Inputs/BrokenHeaders/APINotes.apinotes
@@ -0,0 +1,5 @@
+Name: SomeBrokenLib
+Functions:
+  - Name: do_something_with_pointers
+    Nu llabilityOfRet: O
+    # the space is intentional, to make sure we don't crash on malformed API Notes
diff --git a/clang/test/APINotes/Inputs/BrokenHeaders/SomeBrokenLib.h b/clang/test/APINotes/Inputs/BrokenHeaders/SomeBrokenLib.h
new file mode 100644
index 000000000000..b09c6f63eae0
--- /dev/null
+++ b/clang/test/APINotes/Inputs/BrokenHeaders/SomeBrokenLib.h
@@ -0,0 +1,6 @@
+#ifndef SOME_BROKEN_LIB_H
+#define SOME_BROKEN_LIB_H
+
+void do_something_with_pointers(int *ptr1, int *ptr2);
+
+#endif // SOME_BROKEN_LIB_H
diff --git a/clang/test/APINotes/Inputs/BrokenHeaders2/APINotes.apinotes b/clang/test/APINotes/Inputs/BrokenHeaders2/APINotes.apinotes
new file mode 100644
index 000000000000..33eeaaada999
--- /dev/null
+++ b/clang/test/APINotes/Inputs/BrokenHeaders2/APINotes.apinotes
@@ -0,0 +1,7 @@
+Name: SomeBrokenLib
+Functions:
+  - Name: do_something_with_pointers
+    NullabilityOfRet: O
+  - Name: do_something_with_pointers
+    NullabilityOfRet: O
+    
diff --git a/clang/test/APINotes/Inputs/BrokenHeaders2/SomeBrokenLib.h b/clang/test/APINotes/Inputs/BrokenHeaders2/SomeBrokenLib.h
new file mode 100644
index 000000000000..b09c6f63eae0
--- /dev/null
+++ b/clang/test/APINotes/Inputs/BrokenHeaders2/SomeBrokenLib.h
@@ -0,0 +1,6 @@
+#ifndef SOME_BROKEN_LIB_H
+#define SOME_BROKEN_LIB_H
+
+void do_something_with_pointers(int *ptr1, int *ptr2);
+
+#endif // SOME_BROKEN_LIB_H
diff --git a/clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/Headers/FrameworkWithActualPrivateModule.h b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/Headers/FrameworkWithActualPrivateModule.h
new file mode 100644
index 000000000000..523de4f7ce08
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/Headers/FrameworkWithActualPrivateModule.h
@@ -0,0 +1 @@
+extern int FrameworkWithActualPrivateModule;
diff --git a/clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/Modules/module.modulemap b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/Modules/module.modulemap
new file mode 100644
index 000000000000..859d723716be
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/Modules/module.modulemap
@@ -0,0 +1,5 @@
+framework module FrameworkWithActualPrivateModule {
+  umbrella header "FrameworkWithActualPrivateModule.h"
+  export *
+  module * { export * }
+}
diff --git a/clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/Modules/module.private.modulemap b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/Modules/module.private.modulemap
new file mode 100644
index 000000000000..e7fafe3bcbb1
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/Modules/module.private.modulemap
@@ -0,0 +1,5 @@
+framework module FrameworkWithActualPrivateModule_Private {
+  umbrella header "FrameworkWithActualPrivateModule_Private.h"
+  export *
+  module * { export * }
+}
diff --git a/clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/PrivateHeaders/FrameworkWithActualPrivateModule_Private.apinotes b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/PrivateHeaders/FrameworkWithActualPrivateModule_Private.apinotes
new file mode 100644
index 000000000000..831cf1e93d35
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/PrivateHeaders/FrameworkWithActualPrivateModule_Private.apinotes
@@ -0,0 +1 @@
+Name: FrameworkWithActualPrivateModule_Private
diff --git a/clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/PrivateHeaders/FrameworkWithActualPrivateModule_Private.h b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/PrivateHeaders/FrameworkWithActualPrivateModule_Private.h
new file mode 100644
index 000000000000..c07a3e95d740
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithActualPrivateModule.framework/PrivateHeaders/FrameworkWithActualPrivateModule_Private.h
@@ -0,0 +1,2 @@
+#include <FrameworkWithActualPrivateModule/FrameworkWithActualPrivateModule.h>
+extern int FrameworkWithActualPrivateModule_Private;
diff --git a/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCase.framework/Headers/FrameworkWithWrongCase.h b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCase.framework/Headers/FrameworkWithWrongCase.h
new file mode 100644
index 000000000000..4f3b631c27e3
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCase.framework/Headers/FrameworkWithWrongCase.h
@@ -0,0 +1 @@
+extern int FrameworkWithWrongCase;
diff --git a/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCase.framework/Modules/module.modulemap b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCase.framework/Modules/module.modulemap
new file mode 100644
index 000000000000..e97d361039a1
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCase.framework/Modules/module.modulemap
@@ -0,0 +1,5 @@
+framework module FrameworkWithWrongCase {
+  umbrella header "FrameworkWithWrongCase.h"
+  export *
+  module * { export * }
+}
diff --git a/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCase.framework/PrivateHeaders/FrameworkWithWrongCase_Private.apinotes b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCase.framework/PrivateHeaders/FrameworkWithWrongCase_Private.apinotes
new file mode 100644
index 000000000000..ae5447c61e33
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCase.framework/PrivateHeaders/FrameworkWithWrongCase_Private.apinotes
@@ -0,0 +1 @@
+Name: FrameworkWithWrongCase
diff --git a/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCasePrivate.framework/Headers/FrameworkWithWrongCasePrivate.h b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCasePrivate.framework/Headers/FrameworkWithWrongCasePrivate.h
new file mode 100644
index 000000000000..d3d61483191c
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCasePrivate.framework/Headers/FrameworkWithWrongCasePrivate.h
@@ -0,0 +1 @@
+extern int FrameworkWithWrongCasePrivate;
diff --git a/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCasePrivate.framework/Modules/module.modulemap b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCasePrivate.framework/Modules/module.modulemap
new file mode 100644
index 000000000000..04b96adbbfeb
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCasePrivate.framework/Modules/module.modulemap
@@ -0,0 +1,5 @@
+framework module FrameworkWithWrongCasePrivate {
+  umbrella header "FrameworkWithWrongCasePrivate.h"
+  export *
+  module * { export * }
+}
diff --git a/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCasePrivate.framework/Modules/module.private.modulemap b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCasePrivate.framework/Modules/module.private.modulemap
new file mode 100644
index 000000000000..d6ad53cdc717
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCasePrivate.framework/Modules/module.private.modulemap
@@ -0,0 +1 @@
+module FrameworkWithWrongCasePrivate.Inner {}
diff --git a/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCasePrivate.framework/PrivateHeaders/FrameworkWithWrongCasePrivate_Private.apinotes b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCasePrivate.framework/PrivateHeaders/FrameworkWithWrongCasePrivate_Private.apinotes
new file mode 100644
index 000000000000..d7af293e8125
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/FrameworkWithWrongCasePrivate.framework/PrivateHeaders/FrameworkWithWrongCasePrivate_Private.apinotes
@@ -0,0 +1 @@
+Name: FrameworkWithWrongCasePrivate
diff --git a/clang/test/APINotes/Inputs/Frameworks/LayeredKit.framework/Headers/LayeredKit.h b/clang/test/APINotes/Inputs/Frameworks/LayeredKit.framework/Headers/LayeredKit.h
new file mode 100644
index 000000000000..a95d19ecbe9a
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/LayeredKit.framework/Headers/LayeredKit.h
@@ -0,0 +1,11 @@
+@import LayeredKitImpl;
+
+// @interface declarations already don't inherit attributes from forward 
+// declarations, so in order to test this properly we have to /not/ define
+// UpwardClass anywhere.
+
+// @interface UpwardClass
+// @end
+
+@protocol UpwardProto
+@end
diff --git a/clang/test/APINotes/Inputs/Frameworks/LayeredKit.framework/Modules/module.modulemap b/clang/test/APINotes/Inputs/Frameworks/LayeredKit.framework/Modules/module.modulemap
new file mode 100644
index 000000000000..04bbe72a2b6e
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/LayeredKit.framework/Modules/module.modulemap
@@ -0,0 +1,5 @@
+framework module LayeredKit {
+  umbrella header "LayeredKit.h"
+  export *
+  module * { export * }
+}
diff --git a/clang/test/APINotes/Inputs/Frameworks/LayeredKitImpl.framework/Headers/LayeredKitImpl.apinotes b/clang/test/APINotes/Inputs/Frameworks/LayeredKitImpl.framework/Headers/LayeredKitImpl.apinotes
new file mode 100644
index 000000000000..bece28cfe605
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/LayeredKitImpl.framework/Headers/LayeredKitImpl.apinotes
@@ -0,0 +1,9 @@
+Name: LayeredKitImpl
+Classes:
+- Name: PerfectlyNormalClass
+  Availability: none
+- Name: UpwardClass
+  Availability: none
+Protocols:
+- Name: UpwardProto
+  Availability: none
diff --git a/clang/test/APINotes/Inputs/Frameworks/LayeredKitImpl.framework/Headers/LayeredKitImpl.h b/clang/test/APINotes/Inputs/Frameworks/LayeredKitImpl.framework/Headers/LayeredKitImpl.h
new file mode 100644
index 000000000000..99591d35803a
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/LayeredKitImpl.framework/Headers/LayeredKitImpl.h
@@ -0,0 +1,7 @@
+@protocol UpwardProto;
+@class UpwardClass;
+
+@interface PerfectlyNormalClass
+@end
+
+void doImplementationThings(UpwardClass *first, id <UpwardProto> second) __attribute((unavailable));
diff --git a/clang/test/APINotes/Inputs/Frameworks/LayeredKitImpl.framework/Modules/module.modulemap b/clang/test/APINotes/Inputs/Frameworks/LayeredKitImpl.framework/Modules/module.modulemap
new file mode 100644
index 000000000000..58a6e55c1067
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/LayeredKitImpl.framework/Modules/module.modulemap
@@ -0,0 +1,5 @@
+framework module LayeredKitImpl {
+  umbrella header "LayeredKitImpl.h"
+  export *
+  module * { export * }
+}
diff --git a/clang/test/APINotes/Inputs/Frameworks/SimpleKit.framework/Modules/module.modulemap b/clang/test/APINotes/Inputs/Frameworks/SimpleKit.framework/Modules/module.modulemap
new file mode 100644
index 000000000000..2d07e76c0a14
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/SimpleKit.framework/Modules/module.modulemap
@@ -0,0 +1,5 @@
+framework module SimpleKit {
+  umbrella header "SimpleKit.h"
+  export *
+  module * { export * }
+}
diff --git a/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/APINotes/SomeKit.apinotes b/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/APINotes/SomeKit.apinotes
new file mode 100644
index 000000000000..817af123fc77
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/APINotes/SomeKit.apinotes
@@ -0,0 +1,74 @@
+Name: SomeKit
+Classes:
+  - Name: A
+    Methods:
+      - Selector:        "transform:"
+        MethodKind:      Instance
+        Availability:    none
+        AvailabilityMsg: "anything but this"
+      - Selector: "transform:integer:"
+        MethodKind:      Instance
+        NullabilityOfRet: N
+        Nullability:      [ N, S ]
+    Properties:
+      - Name: intValue
+        PropertyKind:    Instance
+        Availability: none
+        AvailabilityMsg: "wouldn't work anyway"
+      - Name: nonnullAInstance
+        PropertyKind:    Instance
+        Nullability:     N
+      - Name: nonnullAClass
+        PropertyKind:    Class
+        Nullability:     N
+      - Name: nonnullABoth
+        Nullability:     N
+  - Name: B
+    Availability: none
+    AvailabilityMsg: "just don't"
+  - Name: C
+    Methods:
+      - Selector: "initWithA:"
+        MethodKind: Instance
+        DesignatedInit: true
+  - Name: OverriddenTypes
+    Methods:
+      - Selector: "methodToMangle:second:"
+        MethodKind: Instance
+        ResultType: 'char *'
+        Parameters:
+          - Position: 0
+            Type: 'SOMEKIT_DOUBLE *'
+          - Position: 1
+            Type: 'float *'
+    Properties:
+      - Name: intPropertyToMangle
+        PropertyKind: Instance
+        Type: 'double *'
+Functions:
+  - Name: global_int_fun
+    ResultType: 'char *'
+    Parameters:
+      - Position: 0
+        Type: 'double *'
+      - Position: 1
+        Type: 'float *'
+Globals:
+  - Name: global_int_ptr
+    Type: 'double *'
+SwiftVersions:
+  - Version: 3.0
+    Classes:
+      - Name: A
+        Methods:
+          - Selector: "transform:integer:"
+            MethodKind:      Instance
+            NullabilityOfRet: O
+            Nullability:      [ O, S ]
+        Properties:
+          - Name: explicitNonnullInstance
+            PropertyKind:    Instance
+            Nullability:     O
+          - Name: explicitNullableInstance
+            PropertyKind:    Instance
+            Nullability:     N
diff --git a/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/APINotes/SomeKit_private.apinotes b/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/APINotes/SomeKit_private.apinotes
new file mode 100644
index 000000000000..28ede9dfa25c
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/APINotes/SomeKit_private.apinotes
@@ -0,0 +1,15 @@
+Name: SomeKit
+Classes:
+  - Name: A
+    Methods:         
+      - Selector: "privateTransform:input:"
+        MethodKind:      Instance
+        NullabilityOfRet: N
+        Nullability:      [ N, S ]
+    Properties:
+      - Name: internalProperty
+        Nullability: N
+Protocols:
+  - Name: InternalProtocol
+    Availability: none
+    AvailabilityMsg: "not for you"
diff --git a/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/Headers/SomeKitForNullAnnotation.h b/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/Headers/SomeKitForNullAnnotation.h
new file mode 100644
index 000000000000..bc0c5da8848e
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/Headers/SomeKitForNullAnnotation.h
@@ -0,0 +1,55 @@
+#ifndef SOMEKIT_H
+#define SOMEKIT_H
+
+#define ROOT_CLASS __attribute__((objc_root_class))
+
+ROOT_CLASS
+@interface A
+-(A*)transform:(A*)input;
+-(A*)transform:(A*)input integer:(int)integer;
+
+@property (nonatomic, readonly, retain) A* someA;
+@property (nonatomic, retain) A* someOtherA;
+
+@property (nonatomic) int intValue;
+@end
+
+@interface B : A
+@end
+
+@interface C : A
+- (instancetype)init;
+- (instancetype)initWithA:(A*)a;
+@end
+
+
+@interface MyClass : A
+- Inst;
++ Clas;
+@end
+
+struct CGRect {
+  float origin;
+  float size;
+};
+typedef struct CGRect NSRect;
+
+@interface I
+- (void) Meth : (NSRect[4])exposedRects;
+- (void) Meth1 : (const  I*)exposedRects;
+- (void) Meth2 : (const I*)exposedRects;
+- (void) Meth3 : (I*)exposedRects;
+- (const I*) Meth4;
+- (const I*) Meth5 : (int) Arg1 : (const I*)Arg2 : (double)Arg3 :   (const I*) Arg4 :(const  volatile id) Arg5;
+- (volatile const I*) Meth6 : (const char *)Arg1 : (const char *)Arg2 : (double)Arg3 :   (const I*) Arg4 :(const  volatile id) Arg5;
+@end
+
+@class NSURL, NSArray, NSError;
+@interface INTF_BLOCKS
+  + (void)getNonLocalVersionsOfItemAtURL:(NSURL *)url completionHandler:(void (^)(NSArray *nonLocalFileVersions, NSError *error))completionHandler;
+  + (void *)getNonLocalVersionsOfItemAtURL2:(NSURL *)url completionHandler:(void (^)(NSArray *nonLocalFileVersions, NSError *error))completionHandler;
+  + (NSError **)getNonLocalVersionsOfItemAtURL3:(int)url completionHandler:(void (^)(NSArray *nonLocalFileVersions, NSError *error))completionHandler;
+  + (id)getNonLocalVersionsOfItemAtURL4:(NSURL *)url completionHandler:(void (^)(int nonLocalFileVersions, NSError *error, NSURL*))completionHandler;
+@end
+
+#endif
diff --git a/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/Modules/module.modulemap b/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/Modules/module.modulemap
new file mode 100644
index 000000000000..3abee2df0be1
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/Modules/module.modulemap
@@ -0,0 +1,5 @@
+framework module SomeKit {
+  umbrella header "SomeKit.h"
+  export *
+  module * { export * }
+}
diff --git a/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/Modules/module.private.modulemap b/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/Modules/module.private.modulemap
new file mode 100644
index 000000000000..bbda9d08e399
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/Modules/module.private.modulemap
@@ -0,0 +1,8 @@
+module SomeKit.Private {
+  header "SomeKit_Private.h"
+  export *
+
+  explicit module NullAnnotation {
+    header "SomeKit_PrivateForNullAnnotation.h"
+  }
+}
diff --git a/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/Modules/module_private.modulemap b/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/Modules/module_private.modulemap
new file mode 100644
index 000000000000..e31034317cb8
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/Modules/module_private.modulemap
@@ -0,0 +1,8 @@
+explicit framework module SomeKit.Private {
+  header "SomeKit_Private.h"
+  explicit NullAnnotation { header "SomeKit_PrivateForNullAnnotation.h" }
+  export *
+  module * { export * }
+syntax error
+
+}
diff --git a/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/PrivateHeaders/SomeKit_Private.h b/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/PrivateHeaders/SomeKit_Private.h
new file mode 100644
index 000000000000..c7611123e4ad
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/PrivateHeaders/SomeKit_Private.h
@@ -0,0 +1,16 @@
+#ifndef SOMEKIT_PRIVATE_H
+#define SOMEKIT_PRIVATE_H
+
+#import <SomeKit/SomeKit.h>
+
+@interface A(Private)
+-(A*)privateTransform:(A*)input;
+
+@property (nonatomic) A* internalProperty;
+@end
+
+@protocol InternalProtocol
+@end
+
+#endif
+
diff --git a/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/PrivateHeaders/SomeKit_PrivateForNullAnnotation.h b/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/PrivateHeaders/SomeKit_PrivateForNullAnnotation.h
new file mode 100644
index 000000000000..bae4456b4080
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/PrivateHeaders/SomeKit_PrivateForNullAnnotation.h
@@ -0,0 +1,17 @@
+#ifndef SOMEKIT_PRIVATE_H
+#define SOMEKIT_PRIVATE_H
+
+#import <SomeKit/SomeKitForNullAnnotation.h>
+
+@interface A(Private)
+-(A*)privateTransform:(A*)input;
+
+@property (nonatomic) A* internalProperty;
+@end
+
+@protocol InternalProtocol
+- (id) MomeMethod;
+@end
+
+#endif
+
diff --git a/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/PrivateHeaders/SomeKit_private.apinotes b/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/PrivateHeaders/SomeKit_private.apinotes
new file mode 100644
index 000000000000..28ede9dfa25c
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/SomeKit.framework/PrivateHeaders/SomeKit_private.apinotes
@@ -0,0 +1,15 @@
+Name: SomeKit
+Classes:
+  - Name: A
+    Methods:         
+      - Selector: "privateTransform:input:"
+        MethodKind:      Instance
+        NullabilityOfRet: N
+        Nullability:      [ N, S ]
+    Properties:
+      - Name: internalProperty
+        Nullability: N
+Protocols:
+  - Name: InternalProtocol
+    Availability: none
+    AvailabilityMsg: "not for you"
diff --git a/clang/test/APINotes/Inputs/Frameworks/SomeOtherKit.framework/APINotes/SomeOtherKit.apinotes b/clang/test/APINotes/Inputs/Frameworks/SomeOtherKit.framework/APINotes/SomeOtherKit.apinotes
new file mode 100644
index 000000000000..2ad546b8f8bc
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/SomeOtherKit.framework/APINotes/SomeOtherKit.apinotes
@@ -0,0 +1,8 @@
+Name: SomeOtherKit
+Classes:
+  - Name: A
+    Methods:
+      - Selector:        "methodA"
+        MethodKind:      Instance
+        Availability:    none
+        AvailabilityMsg: "anything but this"
diff --git a/clang/test/APINotes/Inputs/Frameworks/SomeOtherKit.framework/Headers/SomeOtherKit.apinotes b/clang/test/APINotes/Inputs/Frameworks/SomeOtherKit.framework/Headers/SomeOtherKit.apinotes
new file mode 100644
index 000000000000..2ad546b8f8bc
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/SomeOtherKit.framework/Headers/SomeOtherKit.apinotes
@@ -0,0 +1,8 @@
+Name: SomeOtherKit
+Classes:
+  - Name: A
+    Methods:
+      - Selector:        "methodA"
+        MethodKind:      Instance
+        Availability:    none
+        AvailabilityMsg: "anything but this"
diff --git a/clang/test/APINotes/Inputs/Frameworks/SomeOtherKit.framework/Headers/SomeOtherKit.h b/clang/test/APINotes/Inputs/Frameworks/SomeOtherKit.framework/Headers/SomeOtherKit.h
new file mode 100644
index 000000000000..3911d765230c
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/SomeOtherKit.framework/Headers/SomeOtherKit.h
@@ -0,0 +1,9 @@
+#ifndef SOME_OTHER_KIT_H
+
+__attribute__((objc_root_class))
+@interface A
+-(void)methodA;
+-(void)methodB;
+@end
+
+#endif
diff --git a/clang/test/APINotes/Inputs/Frameworks/SomeOtherKit.framework/Modules/module.modulemap b/clang/test/APINotes/Inputs/Frameworks/SomeOtherKit.framework/Modules/module.modulemap
new file mode 100644
index 000000000000..0aaad92e041c
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/SomeOtherKit.framework/Modules/module.modulemap
@@ -0,0 +1,5 @@
+framework module SomeOtherKit {
+  umbrella header "SomeOtherKit.h"
+  export *
+  module * { export * }
+}
diff --git a/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/Headers/TopLevelPrivateKit.h b/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/Headers/TopLevelPrivateKit.h
new file mode 100644
index 000000000000..d3376f1dac5d
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/Headers/TopLevelPrivateKit.h
@@ -0,0 +1 @@
+extern int TopLevelPrivateKit_Public;
diff --git a/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/Headers/TopLevelPrivateKit_Private.apinotes b/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/Headers/TopLevelPrivateKit_Private.apinotes
new file mode 100644
index 000000000000..ece1dd220adf
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/Headers/TopLevelPrivateKit_Private.apinotes
@@ -0,0 +1 @@
+garbage here because this file shouldn't get read
\ No newline at end of file
diff --git a/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/Modules/module.modulemap b/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/Modules/module.modulemap
new file mode 100644
index 000000000000..70faa54e8347
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/Modules/module.modulemap
@@ -0,0 +1,5 @@
+framework module TopLevelPrivateKit {
+  umbrella header "TopLevelPrivateKit.h"
+  export *
+  module * { export * }
+}
diff --git a/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/Modules/module.private.modulemap b/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/Modules/module.private.modulemap
new file mode 100644
index 000000000000..0958a14d6710
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/Modules/module.private.modulemap
@@ -0,0 +1,5 @@
+framework module TopLevelPrivateKit_Private {
+  umbrella header "TopLevelPrivateKit_Private.h"
+  export *
+  module * { export * }
+}
diff --git a/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/PrivateHeaders/TopLevelPrivateKit.apinotes b/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/PrivateHeaders/TopLevelPrivateKit.apinotes
new file mode 100644
index 000000000000..908dae0e3b0b
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/PrivateHeaders/TopLevelPrivateKit.apinotes
@@ -0,0 +1 @@
+garbage here because this file shouldn't get read
diff --git a/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/PrivateHeaders/TopLevelPrivateKit_Private.apinotes b/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/PrivateHeaders/TopLevelPrivateKit_Private.apinotes
new file mode 100644
index 000000000000..43323621588b
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/PrivateHeaders/TopLevelPrivateKit_Private.apinotes
@@ -0,0 +1,4 @@
+Name: TopLevelPrivateKit_Private
+Globals:
+- Name: TopLevelPrivateKit_Private
+  Type: float
diff --git a/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/PrivateHeaders/TopLevelPrivateKit_Private.h b/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/PrivateHeaders/TopLevelPrivateKit_Private.h
new file mode 100644
index 000000000000..39cbfe6e9918
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/PrivateHeaders/TopLevelPrivateKit_Private.h
@@ -0,0 +1 @@
+extern int TopLevelPrivateKit_Private;
diff --git a/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/PrivateHeaders/TopLevelPrivateKit_Private_private.apinotes b/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/PrivateHeaders/TopLevelPrivateKit_Private_private.apinotes
new file mode 100644
index 000000000000..ece1dd220adf
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/TopLevelPrivateKit.framework/PrivateHeaders/TopLevelPrivateKit_Private_private.apinotes
@@ -0,0 +1 @@
+garbage here because this file shouldn't get read
\ No newline at end of file
diff --git a/clang/test/APINotes/Inputs/Frameworks/VersionedKit.framework/Headers/VersionedKit.apinotes b/clang/test/APINotes/Inputs/Frameworks/VersionedKit.framework/Headers/VersionedKit.apinotes
new file mode 100644
index 000000000000..572c714b3d61
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/VersionedKit.framework/Headers/VersionedKit.apinotes
@@ -0,0 +1,156 @@
+Name: VersionedKit
+Classes:
+  - Name: TestProperties
+    SwiftObjCMembers: true
+    Properties:
+      - Name: accessorsOnly
+        PropertyKind:    Instance
+        SwiftImportAsAccessors: true
+      - Name: accessorsOnlyForClass
+        PropertyKind:    Class
+        SwiftImportAsAccessors: true
+      - Name: accessorsOnlyExceptInVersion3
+        PropertyKind:    Instance
+        SwiftImportAsAccessors: true
+      - Name: accessorsOnlyForClassExceptInVersion3
+        PropertyKind:    Class
+        SwiftImportAsAccessors: true
+Functions:
+  - Name: unversionedRenameDUMP
+    SwiftName: 'unversionedRename_NOTES()'
+Tags:
+  - Name: APINotedFlagEnum
+    FlagEnum: true
+  - Name: APINotedOpenEnum
+    EnumExtensibility: open
+  - Name: APINotedClosedEnum
+    EnumExtensibility: closed
+  - Name: SoonToBeCFEnum
+    EnumKind: CFEnum
+  - Name: SoonToBeNSEnum
+    EnumKind: NSEnum
+  - Name: SoonToBeCFOptions
+    EnumKind: CFOptions
+  - Name: SoonToBeNSOptions
+    EnumKind: NSOptions
+  - Name: SoonToBeCFClosedEnum
+    EnumKind: CFClosedEnum
+  - Name: SoonToBeNSClosedEnum
+    EnumKind: NSClosedEnum
+  - Name: UndoAllThatHasBeenDoneToMe
+    EnumKind: none
+Typedefs:
+  - Name: MultiVersionedTypedef34Notes
+    SwiftName: MultiVersionedTypedef34Notes_NEW
+  - Name: MultiVersionedTypedef345Notes
+    SwiftName: MultiVersionedTypedef345Notes_NEW
+  - Name: MultiVersionedTypedef4Notes
+    SwiftName: MultiVersionedTypedef4Notes_NEW
+  - Name: MultiVersionedTypedef45Notes
+    SwiftName: MultiVersionedTypedef45Notes_NEW
+SwiftVersions:
+  - Version: 3.0
+    Classes:
+      - Name: MyReferenceType
+        SwiftBridge: ''
+      - Name: TestGenericDUMP
+        SwiftImportAsNonGeneric: true
+      - Name: TestProperties
+        SwiftObjCMembers: false
+        Properties:
+          - Name: accessorsOnlyInVersion3
+            PropertyKind:    Instance
+            SwiftImportAsAccessors: true
+          - Name: accessorsOnlyForClassInVersion3
+            PropertyKind:    Class
+            SwiftImportAsAccessors: true
+          - Name: accessorsOnlyExceptInVersion3
+            PropertyKind:    Instance
+            SwiftImportAsAccessors: false
+          - Name: accessorsOnlyForClassExceptInVersion3
+            PropertyKind:    Class
+            SwiftImportAsAccessors: false
+      - Name: Swift3RenamedOnlyDUMP
+        SwiftName: SpecialSwift3Name
+      - Name: Swift3RenamedAlsoDUMP
+        SwiftName: SpecialSwift3Also
+    Functions:
+      - Name: moveToPointDUMP
+        SwiftName: 'moveTo(a:b:)'
+      - Name: acceptClosure
+        Parameters:      
+          - Position:        0
+            NoEscape:        false
+      - Name: privateFunc
+        SwiftPrivate: false
+    Tags:
+      - Name: MyErrorCode
+        NSErrorDomain: ''
+      - Name: NewlyFlagEnum
+        FlagEnum: false
+      - Name: OpenToClosedEnum
+        EnumExtensibility: open
+      - Name: ClosedToOpenEnum
+        EnumExtensibility: closed
+      - Name: NewlyClosedEnum
+        EnumExtensibility: none
+      - Name: NewlyOpenEnum
+        EnumExtensibility: none
+    Typedefs:
+      - Name: MyDoubleWrapper
+        SwiftWrapper: none
+      - Name: MultiVersionedTypedef34
+        SwiftName: MultiVersionedTypedef34_3
+      - Name: MultiVersionedTypedef34Header
+        SwiftName: MultiVersionedTypedef34Header_3
+      - Name: MultiVersionedTypedef34Notes
+        SwiftName: MultiVersionedTypedef34Notes_3
+      - Name: MultiVersionedTypedef345
+        SwiftName: MultiVersionedTypedef345_3
+      - Name: MultiVersionedTypedef345Header
+        SwiftName: MultiVersionedTypedef345Header_3
+      - Name: MultiVersionedTypedef345Notes
+        SwiftName: MultiVersionedTypedef345Notes_3
+  - Version: 5
+    Typedefs:
+      - Name: MultiVersionedTypedef345
+        SwiftName: MultiVersionedTypedef345_5
+      - Name: MultiVersionedTypedef345Header
+        SwiftName: MultiVersionedTypedef345Header_5
+      - Name: MultiVersionedTypedef345Notes
+        SwiftName: MultiVersionedTypedef345Notes_5
+      - Name: MultiVersionedTypedef45
+        SwiftName: MultiVersionedTypedef45_5
+      - Name: MultiVersionedTypedef45Header
+        SwiftName: MultiVersionedTypedef45Header_5
+      - Name: MultiVersionedTypedef45Notes
+        SwiftName: MultiVersionedTypedef45Notes_5
+  - Version: 4 # Versions are deliberately ordered as "3, 5, 4" to catch bugs.
+    Classes:
+      - Name: Swift4RenamedDUMP
+        SwiftName: SpecialSwift4Name
+    Typedefs:
+      - Name: MultiVersionedTypedef34
+        SwiftName: MultiVersionedTypedef34_4
+      - Name: MultiVersionedTypedef34Header
+        SwiftName: MultiVersionedTypedef34Header_4
+      - Name: MultiVersionedTypedef34Notes
+        SwiftName: MultiVersionedTypedef34Notes_4
+      - Name: MultiVersionedTypedef345
+        SwiftName: MultiVersionedTypedef345_4
+      - Name: MultiVersionedTypedef345Header
+        SwiftName: MultiVersionedTypedef345Header_4
+      - Name: MultiVersionedTypedef345Notes
+        SwiftName: MultiVersionedTypedef345Notes_4
+      - Name: MultiVersionedTypedef4
+        SwiftName: MultiVersionedTypedef4_4
+      - Name: MultiVersionedTypedef4Header
+        SwiftName: MultiVersionedTypedef4Header_4
+      - Name: MultiVersionedTypedef4Notes
+        SwiftName: MultiVersionedTypedef4Notes_4
+      - Name: MultiVersionedTypedef45
+        SwiftName: MultiVersionedTypedef45_4
+      - Name: MultiVersionedTypedef45Header
+        SwiftName: MultiVersionedTypedef45Header_4
+      - Name: MultiVersionedTypedef45Notes
+        SwiftName: MultiVersionedTypedef45Notes_4
diff --git a/clang/test/APINotes/Inputs/Frameworks/VersionedKit.framework/Headers/VersionedKit.h b/clang/test/APINotes/Inputs/Frameworks/VersionedKit.framework/Headers/VersionedKit.h
new file mode 100644
index 000000000000..9ce95633c523
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/VersionedKit.framework/Headers/VersionedKit.h
@@ -0,0 +1,137 @@
+void moveToPointDUMP(double x, double y) __attribute__((swift_name("moveTo(x:y:)")));
+
+void unversionedRenameDUMP(void) __attribute__((swift_name("unversionedRename_HEADER()")));
+
+void acceptClosure(void (^ __attribute__((noescape)) block)(void));
+
+void privateFunc(void) __attribute__((swift_private));
+
+typedef double MyDoubleWrapper __attribute__((swift_wrapper(struct)));
+
+#if __OBJC__
+@class NSString;
+
+extern NSString *MyErrorDomain;
+
+enum __attribute__((ns_error_domain(MyErrorDomain))) MyErrorCode {
+  MyErrorCodeFailed = 1
+};
+
+__attribute__((swift_bridge("MyValueType")))
+@interface MyReferenceType
+@end
+
+@interface TestProperties
+@property (nonatomic, readwrite, retain) id accessorsOnly;
+@property (nonatomic, readwrite, retain, class) id accessorsOnlyForClass;
+
+@property (nonatomic, readwrite, retain) id accessorsOnlyInVersion3;
+@property (nonatomic, readwrite, retain, class) id accessorsOnlyForClassInVersion3;
+
+@property (nonatomic, readwrite, retain) id accessorsOnlyExceptInVersion3;
+@property (nonatomic, readwrite, retain, class) id accessorsOnlyForClassExceptInVersion3;
+@end
+
+@interface Base
+@end
+
+@interface TestGenericDUMP<Element> : Base
+- (Element)element;
+@end
+
+@interface Swift3RenamedOnlyDUMP
+@end
+
+__attribute__((swift_name("Swift4Name")))
+@interface Swift3RenamedAlsoDUMP
+@end
+
+@interface Swift4RenamedDUMP
+@end
+
+#endif
+
+
+enum __attribute__((flag_enum)) FlagEnum {
+  FlagEnumA = 1,
+  FlagEnumB = 2
+};
+
+enum __attribute__((flag_enum)) NewlyFlagEnum {
+  NewlyFlagEnumA = 1,
+  NewlyFlagEnumB = 2
+};
+
+enum APINotedFlagEnum {
+  APINotedFlagEnumA = 1,
+  APINotedFlagEnumB = 2
+};
+
+
+enum __attribute__((enum_extensibility(open))) OpenEnum {
+  OpenEnumA = 1,
+};
+
+enum __attribute__((enum_extensibility(open))) NewlyOpenEnum {
+  NewlyOpenEnumA = 1,
+};
+
+enum __attribute__((enum_extensibility(closed))) NewlyClosedEnum {
+  NewlyClosedEnumA = 1,
+};
+
+enum __attribute__((enum_extensibility(open))) ClosedToOpenEnum {
+  ClosedToOpenEnumA = 1,
+};
+
+enum __attribute__((enum_extensibility(closed))) OpenToClosedEnum {
+  OpenToClosedEnumA = 1,
+};
+
+enum APINotedOpenEnum {
+  APINotedOpenEnumA = 1,
+};
+
+enum APINotedClosedEnum {
+  APINotedClosedEnumA = 1,
+};
+
+
+enum SoonToBeCFEnum {
+  SoonToBeCFEnumA = 1
+};
+enum SoonToBeNSEnum {
+  SoonToBeNSEnumA = 1
+};
+enum SoonToBeCFOptions {
+  SoonToBeCFOptionsA = 1
+};
+enum SoonToBeNSOptions {
+  SoonToBeNSOptionsA = 1
+};
+enum SoonToBeCFClosedEnum {
+  SoonToBeCFClosedEnumA = 1
+};
+enum SoonToBeNSClosedEnum {
+  SoonToBeNSClosedEnumA = 1
+};
+enum UndoAllThatHasBeenDoneToMe {
+  UndoAllThatHasBeenDoneToMeA = 1
+} __attribute__((flag_enum)) __attribute__((enum_extensibility(closed)));
+
+
+typedef int MultiVersionedTypedef4;
+typedef int MultiVersionedTypedef4Notes;
+typedef int MultiVersionedTypedef4Header __attribute__((swift_name("MultiVersionedTypedef4Header_NEW")));
+
+typedef int MultiVersionedTypedef34;
+typedef int MultiVersionedTypedef34Notes;
+typedef int MultiVersionedTypedef34Header __attribute__((swift_name("MultiVersionedTypedef34Header_NEW")));
+
+typedef int MultiVersionedTypedef45;
+typedef int MultiVersionedTypedef45Notes;
+typedef int MultiVersionedTypedef45Header __attribute__((swift_name("MultiVersionedTypedef45Header_NEW")));
+
+typedef int MultiVersionedTypedef345;
+typedef int MultiVersionedTypedef345Notes;
+typedef int MultiVersionedTypedef345Header __attribute__((swift_name("MultiVersionedTypedef345Header_NEW")));
diff --git a/clang/test/APINotes/Inputs/Frameworks/VersionedKit.framework/Modules/module.modulemap b/clang/test/APINotes/Inputs/Frameworks/VersionedKit.framework/Modules/module.modulemap
new file mode 100644
index 000000000000..6d957fd68009
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Frameworks/VersionedKit.framework/Modules/module.modulemap
@@ -0,0 +1,5 @@
+framework module VersionedKit {
+  umbrella header "VersionedKit.h"
+  export *
+  module * { export * }
+}
diff --git a/clang/test/APINotes/Inputs/Headers/APINotes.apinotes b/clang/test/APINotes/Inputs/Headers/APINotes.apinotes
new file mode 100644
index 000000000000..08210fc70565
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/APINotes.apinotes
@@ -0,0 +1,18 @@
+Name: HeaderLib
+SwiftInferImportAsMember: true
+Functions:
+  - Name: custom_realloc
+    NullabilityOfRet: N
+    Nullability: [ N, S ]
+  - Name: unavailable_function
+    Availability: none
+    AvailabilityMsg: "I beg you not to use this"
+  - Name: do_something_with_pointers
+    NullabilityOfRet: O
+    Nullability: [ N, O ]
+    
+Globals:
+  - Name: global_int
+    Nullability: N
+  - Name: unavailable_global_int
+    Availability: none
diff --git a/clang/test/APINotes/Inputs/Headers/BrokenTypes.apinotes b/clang/test/APINotes/Inputs/Headers/BrokenTypes.apinotes
new file mode 100644
index 000000000000..00f7b5074e98
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/BrokenTypes.apinotes
@@ -0,0 +1,10 @@
+Name: BrokenTypes
+Functions:
+  - Name: break_me_function
+    ResultType: 'int * with extra junk'
+    Parameters:
+      - Position: 0
+        Type: 'not_a_type'
+Globals:
+  - Name: break_me_variable
+    Type: 'double'
diff --git a/clang/test/APINotes/Inputs/Headers/BrokenTypes.h b/clang/test/APINotes/Inputs/Headers/BrokenTypes.h
new file mode 100644
index 000000000000..fee054b74cf7
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/BrokenTypes.h
@@ -0,0 +1,8 @@
+#ifndef BROKEN_TYPES_H
+#define BROKEN_TYPES_H
+
+char break_me_function(void *ptr);
+
+extern char break_me_variable;
+
+#endif // BROKEN_TYPES_H
diff --git a/clang/test/APINotes/Inputs/Headers/ExternCtx.apinotes b/clang/test/APINotes/Inputs/Headers/ExternCtx.apinotes
new file mode 100644
index 000000000000..0f47ac6deea8
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/ExternCtx.apinotes
@@ -0,0 +1,15 @@
+Name: ExternCtx
+Globals:
+  - Name: globalInExternC
+    Availability: none
+    AvailabilityMsg: "oh no"
+  - Name: globalInExternCXX
+    Availability: none
+    AvailabilityMsg: "oh no #2"
+Functions:
+  - Name: globalFuncInExternC
+    Availability: none
+    AvailabilityMsg: "oh no #3"
+  - Name: globalFuncInExternCXX
+    Availability: none
+    AvailabilityMsg: "oh no #4"
diff --git a/clang/test/APINotes/Inputs/Headers/ExternCtx.h b/clang/test/APINotes/Inputs/Headers/ExternCtx.h
new file mode 100644
index 000000000000..669d443f60ec
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/ExternCtx.h
@@ -0,0 +1,11 @@
+extern "C" {
+  static int globalInExternC = 1;
+
+  static void globalFuncInExternC() {}
+}
+
+extern "C++" {
+  static int globalInExternCXX = 2;
+
+  static void globalFuncInExternCXX() {}
+}
diff --git a/clang/test/APINotes/Inputs/Headers/HeaderLib.apinotes b/clang/test/APINotes/Inputs/Headers/HeaderLib.apinotes
new file mode 100644
index 000000000000..7dcb22476a1d
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/HeaderLib.apinotes
@@ -0,0 +1,37 @@
+Name: HeaderLib
+SwiftInferImportAsMember: true
+Functions:
+  - Name: custom_realloc
+    NullabilityOfRet: N
+    Nullability: [ N, S ]
+  - Name: unavailable_function
+    Availability: none
+    AvailabilityMsg: "I beg you not to use this"
+  - Name: do_something_with_pointers
+    NullabilityOfRet: O
+    Nullability: [ N, O ]
+  - Name: do_something_with_arrays
+    Parameters:
+      - Position: 0
+        Nullability: N
+      - Position: 1
+        Nullability: N
+  - Name: take_pointer_and_int
+    Parameters:
+      - Position: 0
+        Nullability: N
+        NoEscape: true
+      - Position: 1
+        NoEscape: true
+Globals:
+  - Name: global_int
+    Nullability: N
+  - Name: unavailable_global_int
+    Availability: none
+Tags:
+  - Name: unavailable_struct
+    Availability: none
+
+Typedefs:
+  - Name: unavailable_typedef
+    Availability: none
diff --git a/clang/test/APINotes/Inputs/Headers/HeaderLib.h b/clang/test/APINotes/Inputs/Headers/HeaderLib.h
new file mode 100644
index 000000000000..806524960785
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/HeaderLib.h
@@ -0,0 +1,19 @@
+#ifndef HEADER_LIB_H
+#define HEADER_LIB_H
+
+void *custom_realloc(void *member, unsigned size);
+
+int *global_int;
+
+int unavailable_function(void);
+int unavailable_global_int;
+
+void do_something_with_pointers(int *ptr1, int *ptr2);
+void do_something_with_arrays(int simple[], int nested[][2]);
+
+typedef int unavailable_typedef;
+struct unavailable_struct { int x, y, z; };
+
+void take_pointer_and_int(int *ptr1, int value);
+
+#endif
diff --git a/clang/test/APINotes/Inputs/Headers/InstancetypeModule.apinotes b/clang/test/APINotes/Inputs/Headers/InstancetypeModule.apinotes
new file mode 100644
index 000000000000..813eb506f39a
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/InstancetypeModule.apinotes
@@ -0,0 +1,10 @@
+Name: InstancetypeModule
+Classes:
+- Name: SomeBaseClass
+  Methods:
+  - Selector: instancetypeFactoryMethod
+    MethodKind: Class
+    ResultType: SomeBaseClass * _Nonnull
+  - Selector: staticFactoryMethod
+    MethodKind: Class
+    ResultType: SomeBaseClass * _Nonnull
diff --git a/clang/test/APINotes/Inputs/Headers/InstancetypeModule.h b/clang/test/APINotes/Inputs/Headers/InstancetypeModule.h
new file mode 100644
index 000000000000..767f201d9faf
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/InstancetypeModule.h
@@ -0,0 +1,10 @@
+@interface Object
+@end
+
+@interface SomeBaseClass : Object
++ (nullable instancetype)instancetypeFactoryMethod;
++ (nullable SomeBaseClass *)staticFactoryMethod;
+@end
+
+@interface SomeSubclass : SomeBaseClass
+@end
diff --git a/clang/test/APINotes/Inputs/Headers/ModuleWithWrongCase.h b/clang/test/APINotes/Inputs/Headers/ModuleWithWrongCase.h
new file mode 100644
index 000000000000..867a15cae9a6
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/ModuleWithWrongCase.h
@@ -0,0 +1 @@
+extern int ModuleWithWrongCase;
diff --git a/clang/test/APINotes/Inputs/Headers/ModuleWithWrongCasePrivate.h b/clang/test/APINotes/Inputs/Headers/ModuleWithWrongCasePrivate.h
new file mode 100644
index 000000000000..aa014296ca7d
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/ModuleWithWrongCasePrivate.h
@@ -0,0 +1 @@
+extern int ModuleWithWrongCasePrivate;
diff --git a/clang/test/APINotes/Inputs/Headers/ModuleWithWrongCasePrivate_Private.apinotes b/clang/test/APINotes/Inputs/Headers/ModuleWithWrongCasePrivate_Private.apinotes
new file mode 100644
index 000000000000..dc6dc50bab6e
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/ModuleWithWrongCasePrivate_Private.apinotes
@@ -0,0 +1 @@
+Name: ModuleWithWrongCasePrivate
diff --git a/clang/test/APINotes/Inputs/Headers/ModuleWithWrongCase_Private.apinotes b/clang/test/APINotes/Inputs/Headers/ModuleWithWrongCase_Private.apinotes
new file mode 100644
index 000000000000..dc6dc50bab6e
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/ModuleWithWrongCase_Private.apinotes
@@ -0,0 +1 @@
+Name: ModuleWithWrongCasePrivate
diff --git a/clang/test/APINotes/Inputs/Headers/Namespaces.apinotes b/clang/test/APINotes/Inputs/Headers/Namespaces.apinotes
new file mode 100644
index 000000000000..e9da36787b63
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/Namespaces.apinotes
@@ -0,0 +1,53 @@
+---
+Name: Namespaces
+Globals:
+  - Name: varInInlineNamespace
+    SwiftName: swiftVarInInlineNamespace
+Functions:
+  - Name: funcInNamespace
+    SwiftName: inWrongContext()
+  - Name: funcInInlineNamespace
+    SwiftName: swiftFuncInInlineNamespace()
+Tags:
+  - Name: char_box
+    SwiftName: InWrongContext
+Namespaces:
+  - Name: Namespace1
+    Typedefs:
+      - Name: my_typedef
+        SwiftName: SwiftTypedef
+      - Name: my_using_decl
+        SwiftName: SwiftUsingDecl
+    Globals:
+      - Name: varInNamespace
+        SwiftName: swiftVarInNamespace
+    Functions:
+      - Name: funcInNamespace
+        SwiftName: swiftFuncInNamespace()
+    Tags:
+      - Name: char_box
+        SwiftName: CharBox
+    Namespaces:
+      - Name: Nested1
+        Globals:
+          - Name: varInNestedNamespace
+            SwiftName: swiftVarInNestedNamespace
+        Functions:
+          - Name: funcInNestedNamespace
+            SwiftName: swiftFuncInNestedNamespace(_:)
+        Tags:
+          - Name: char_box
+            SwiftName: NestedCharBox
+        Namespaces:
+          - Name: Namespace1
+            Tags:
+              - Name: char_box
+                SwiftName: DeepNestedCharBox
+      - Name: Nested2
+        Globals:
+          - Name: varInNestedNamespace
+            SwiftName: swiftAnotherVarInNestedNamespace
+  - Name: InlineNamespace1
+    Functions:
+      - Name: funcInInlineNamespace
+        SwiftName: shouldNotSpellOutInlineNamespaces()
diff --git a/clang/test/APINotes/Inputs/Headers/Namespaces.h b/clang/test/APINotes/Inputs/Headers/Namespaces.h
new file mode 100644
index 000000000000..6a79e996be86
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/Namespaces.h
@@ -0,0 +1,39 @@
+namespace Namespace1 { namespace Nested1 {} }
+
+namespace Namespace1 {
+static int varInNamespace = 1;
+struct char_box { char c; };
+void funcInNamespace();
+
+namespace Nested1 {
+void funcInNestedNamespace(int i);
+struct char_box {
+  char c;
+};
+}
+
+namespace Nested1 {
+static int varInNestedNamespace = 1;
+void funcInNestedNamespace(int i);
+
+namespace Namespace1 {
+struct char_box { char c; };
+} // namespace Namespace1
+} // namespace Nested1
+
+namespace Nested2 {
+static int varInNestedNamespace = 2;
+} // namespace Nested2
+
+namespace Nested1 { namespace Namespace1 {} }
+} // namespace Namespace1
+
+namespace Namespace1 {
+typedef int my_typedef;
+using my_using_decl = int;
+}
+
+inline namespace InlineNamespace1 {
+static int varInInlineNamespace = 3;
+void funcInInlineNamespace();
+}
diff --git a/clang/test/APINotes/Inputs/Headers/PrivateLib.apinotes b/clang/test/APINotes/Inputs/Headers/PrivateLib.apinotes
new file mode 100644
index 000000000000..5f62284aadca
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/PrivateLib.apinotes
@@ -0,0 +1,4 @@
+Name: HeaderLib
+Globals:
+- Name: PrivateLib
+  Type: float
diff --git a/clang/test/APINotes/Inputs/Headers/PrivateLib.h b/clang/test/APINotes/Inputs/Headers/PrivateLib.h
new file mode 100644
index 000000000000..59aeef09bdd3
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/PrivateLib.h
@@ -0,0 +1 @@
+extern int PrivateLib;
diff --git a/clang/test/APINotes/Inputs/Headers/PrivateLib_private.apinotes b/clang/test/APINotes/Inputs/Headers/PrivateLib_private.apinotes
new file mode 100644
index 000000000000..908dae0e3b0b
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/PrivateLib_private.apinotes
@@ -0,0 +1 @@
+garbage here because this file shouldn't get read
diff --git a/clang/test/APINotes/Inputs/Headers/SwiftImportAs.apinotes b/clang/test/APINotes/Inputs/Headers/SwiftImportAs.apinotes
new file mode 100644
index 000000000000..5dbb83cab86b
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/SwiftImportAs.apinotes
@@ -0,0 +1,9 @@
+---
+Name: SwiftImportAs
+Tags:
+- Name: ImmortalRefType
+  SwiftImportAs: reference
+- Name: RefCountedType
+  SwiftImportAs: reference
+  SwiftReleaseOp: RCRelease
+  SwiftRetainOp: RCRetain
diff --git a/clang/test/APINotes/Inputs/Headers/SwiftImportAs.h b/clang/test/APINotes/Inputs/Headers/SwiftImportAs.h
new file mode 100644
index 000000000000..82b8a6749c4f
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/SwiftImportAs.h
@@ -0,0 +1,6 @@
+struct ImmortalRefType {};
+
+struct RefCountedType { int value; };
+
+inline void RCRetain(RefCountedType *x) { x->value++; }
+inline void RCRelease(RefCountedType *x) { x->value--; }
diff --git a/clang/test/APINotes/Inputs/Headers/module.modulemap b/clang/test/APINotes/Inputs/Headers/module.modulemap
new file mode 100644
index 000000000000..98b4ee3e96cf
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/module.modulemap
@@ -0,0 +1,31 @@
+module ExternCtx {
+  header "ExternCtx.h"
+}
+
+module HeaderLib {
+  header "HeaderLib.h"
+}
+
+module InstancetypeModule {
+  header "InstancetypeModule.h"
+}
+
+module BrokenTypes {
+  header "BrokenTypes.h"
+}
+
+module ModuleWithWrongCase {
+  header "ModuleWithWrongCase.h"
+}
+
+module ModuleWithWrongCasePrivate {
+  header "ModuleWithWrongCasePrivate.h"
+}
+
+module Namespaces {
+  header "Namespaces.h"
+}
+
+module SwiftImportAs {
+  header "SwiftImportAs.h"
+}
diff --git a/clang/test/APINotes/Inputs/Headers/module.private.modulemap b/clang/test/APINotes/Inputs/Headers/module.private.modulemap
new file mode 100644
index 000000000000..2ecf322ed18d
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/module.private.modulemap
@@ -0,0 +1,5 @@
+module PrivateLib {
+  header "PrivateLib.h"
+}
+
+module ModuleWithWrongCasePrivate.Inner {}
diff --git a/clang/test/APINotes/Inputs/yaml-reader-errors/UIKit.apinotes b/clang/test/APINotes/Inputs/yaml-reader-errors/UIKit.apinotes
new file mode 100644
index 000000000000..77db84400899
--- /dev/null
+++ b/clang/test/APINotes/Inputs/yaml-reader-errors/UIKit.apinotes
@@ -0,0 +1,65 @@
+---
+Name:            UIKit
+Classes:
+  - Name:            UIFont
+    Methods:
+      - Selector:        'fontWithName:size:'
+        MethodKind:      Instance
+        Nullability:     [ N ]
+        NullabilityOfRet: O
+        DesignatedInit:  true
+# CHECK: duplicate definition of method '-[UIFont fontWithName:size:]'
+      - Selector:        'fontWithName:size:'
+        MethodKind:      Instance
+        Nullability:     [ N ]
+        NullabilityOfRet: O
+        DesignatedInit:  true
+    Properties:
+      - Name:            familyName
+        Nullability:     N
+      - Name:            fontName
+        Nullability:     N
+# CHECK: duplicate definition of instance property 'UIFont.familyName'
+      - Name:            familyName
+        Nullability:     N
+# CHECK: multiple definitions of class 'UIFont'
+  - Name:            UIFont
+Protocols:
+  - Name:            MyProto
+    AuditedForNullability: true
+# CHECK: multiple definitions of protocol 'MyProto'
+  - Name:            MyProto
+    AuditedForNullability: true
+Functions:
+  - Name:        'globalFoo'
+    Nullability:     [ N, N, O, S ]
+    NullabilityOfRet: O
+  - Name:        'globalFoo2'
+    Nullability:     [ N, N, O, S ]
+    NullabilityOfRet: O
+Globals:
+  - Name:            globalVar
+    Nullability:     O
+  - Name:            globalVar2
+    Nullability:     O
+Tags:
+# CHECK: cannot mix EnumKind and FlagEnum (for FlagAndEnumKind)
+  - Name: FlagAndEnumKind
+    FlagEnum: true
+    EnumKind: CFOptions
+# CHECK: cannot mix EnumKind and FlagEnum (for FlagAndEnumKind2)
+  - Name: FlagAndEnumKind2
+    EnumKind: CFOptions
+    FlagEnum: false
+# CHECK: cannot mix EnumKind and EnumExtensibility (for ExtensibilityAndEnumKind)
+  - Name: ExtensibilityAndEnumKind
+    EnumExtensibility: open
+    EnumKind: CFOptions
+# CHECK: cannot mix EnumKind and EnumExtensibility (for ExtensibilityAndEnumKind2)
+  - Name: ExtensibilityAndEnumKind2
+    EnumKind: CFOptions
+    EnumExtensibility: closed
+# CHECK: cannot mix EnumKind and EnumExtensibility (for ExtensibilityAndEnumKind3)
+  - Name: ExtensibilityAndEnumKind3
+    EnumKind: none
+    EnumExtensibility: none
diff --git a/clang/test/APINotes/Inputs/yaml-reader-errors/UIKit.h b/clang/test/APINotes/Inputs/yaml-reader-errors/UIKit.h
new file mode 100644
index 000000000000..55313ae260ae
--- /dev/null
+++ b/clang/test/APINotes/Inputs/yaml-reader-errors/UIKit.h
@@ -0,0 +1 @@
+extern int yesOfCourseThisIsWhatUIKitLooksLike;
diff --git a/clang/test/APINotes/Inputs/yaml-reader-errors/module.modulemap b/clang/test/APINotes/Inputs/yaml-reader-errors/module.modulemap
new file mode 100644
index 000000000000..3d683d705cac
--- /dev/null
+++ b/clang/test/APINotes/Inputs/yaml-reader-errors/module.modulemap
@@ -0,0 +1,3 @@
+module UIKit {
+  header "UIKit.h"
+}
diff --git a/clang/test/APINotes/availability.m b/clang/test/APINotes/availability.m
new file mode 100644
index 000000000000..2ddc2a73da80
--- /dev/null
+++ b/clang/test/APINotes/availability.m
@@ -0,0 +1,48 @@
+// RUN: rm -rf %t
+// RUN: %clang_cc1 -fmodules -Wno-private-module -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -verify
+
+#include "HeaderLib.h"
+#import <SomeKit/SomeKit.h>
+#import <SomeKit/SomeKit_Private.h>
+
+int main() {
+  int i;
+  i = unavailable_function(); // expected-error{{'unavailable_function' is unavailable: I beg you not to use this}}
+  // expected-note@HeaderLib.h:8{{'unavailable_function' has been explicitly marked unavailable here}}
+  i = unavailable_global_int; // expected-error{{'unavailable_global_int' is unavailable}}
+  // expected-note@HeaderLib.h:9{{'unavailable_global_int' has been explicitly marked unavailable here}}
+
+  unavailable_typedef t; // expected-error{{'unavailable_typedef' is unavailable}}
+  // expected-note@HeaderLib.h:14{{'unavailable_typedef' has been explicitly marked unavailable here}}
+
+  struct unavailable_struct s; // expected-error{{'unavailable_struct' is unavailable}}
+  // expected-note@HeaderLib.h:15{{'unavailable_struct' has been explicitly marked unavailable here}}
+
+  B *b = 0; // expected-error{{'B' is unavailable: just don't}}
+  // expected-note@SomeKit/SomeKit.h:15{{'B' has been explicitly marked unavailable here}}
+
+  id<InternalProtocol> proto = 0; // expected-error{{'InternalProtocol' is unavailable: not for you}}
+  // expected-note@SomeKit/SomeKit_Private.h:12{{'InternalProtocol' has been explicitly marked unavailable here}}
+
+  A *a = 0;
+  i = a.intValue; // expected-error{{intValue' is unavailable: wouldn't work anyway}}
+  // expected-note@SomeKit/SomeKit.h:12{{'intValue' has been explicitly marked unavailable here}}
+
+  [a transform:a]; // expected-error{{'transform:' is unavailable: anything but this}}
+  // expected-note@SomeKit/SomeKit.h:6{{'transform:' has been explicitly marked unavailable here}}
+
+  [a implicitGetOnlyInstance]; // expected-error{{'implicitGetOnlyInstance' is unavailable: getter gone}}
+  // expected-note@SomeKit/SomeKit.h:53{{'implicitGetOnlyInstance' has been explicitly marked unavailable here}}
+  [A implicitGetOnlyClass]; // expected-error{{'implicitGetOnlyClass' is unavailable: getter gone}}
+  // expected-note@SomeKit/SomeKit.h:54{{'implicitGetOnlyClass' has been explicitly marked unavailable here}}
+  [a implicitGetSetInstance]; // expected-error{{'implicitGetSetInstance' is unavailable: getter gone}}
+  // expected-note@SomeKit/SomeKit.h:56{{'implicitGetSetInstance' has been explicitly marked unavailable here}}
+  [a setImplicitGetSetInstance: a];  // expected-error{{'setImplicitGetSetInstance:' is unavailable: setter gone}}
+  // expected-note@SomeKit/SomeKit.h:56{{'setImplicitGetSetInstance:' has been explicitly marked unavailable here}}
+  [A implicitGetSetClass]; // expected-error{{'implicitGetSetClass' is unavailable: getter gone}}
+  // expected-note@SomeKit/SomeKit.h:57{{'implicitGetSetClass' has been explicitly marked unavailable here}}
+  [A setImplicitGetSetClass: a];  // expected-error{{'setImplicitGetSetClass:' is unavailable: setter gone}}
+  // expected-note@SomeKit/SomeKit.h:57{{'setImplicitGetSetClass:' has been explicitly marked unavailable here}}
+  return 0;
+}
+
diff --git a/clang/test/APINotes/broken_types.m b/clang/test/APINotes/broken_types.m
new file mode 100644
index 000000000000..ee33ff7c4b4b
--- /dev/null
+++ b/clang/test/APINotes/broken_types.m
@@ -0,0 +1,19 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: not %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s 2> %t.err
+// RUN: FileCheck %s < %t.err
+
+#include "BrokenTypes.h"
+
+// CHECK: <API Notes>:1:1: error: unknown type name 'not_a_type'
+// CHECK-NEXT: not_a_type
+// CHECK-NEXT: ^
+
+// CHECK: <API Notes>:1:7: error: unparsed tokens following type
+// CHECK-NEXT: int * with extra junk
+// CHECK-NEXT:       ^
+
+// CHECK: BrokenTypes.h:4:6: error: API notes replacement type 'int *' has a different size from original type 'char'
+
+// CHECK: BrokenTypes.h:6:13: error: API notes replacement type 'double' has a different size from original type 'char'
+
+// CHECK: 5 errors generated.
diff --git a/clang/test/APINotes/case-for-private-apinotes-file.c b/clang/test/APINotes/case-for-private-apinotes-file.c
new file mode 100644
index 000000000000..6aff3db54918
--- /dev/null
+++ b/clang/test/APINotes/case-for-private-apinotes-file.c
@@ -0,0 +1,22 @@
+// REQUIRES: case-insensitive-filesystem
+
+// RUN: rm -rf %t
+// RUN: %clang_cc1 -fsyntax-only -fmodules -fapinotes-modules -fimplicit-module-maps -fmodules-cache-path=%t -F %S/Inputs/Frameworks -I %S/Inputs/Headers %s 2>&1 | FileCheck %s
+
+// RUN: rm -rf %t
+// RUN: %clang_cc1 -fsyntax-only -fmodules -fapinotes-modules -fimplicit-module-maps -fmodules-cache-path=%t -iframework %S/Inputs/Frameworks -isystem %S/Inputs/Headers %s -Werror
+
+// RUN: rm -rf %t
+// RUN: %clang_cc1 -fsyntax-only -fmodules -fapinotes-modules -fimplicit-module-maps -fmodules-cache-path=%t -iframework %S/Inputs/Frameworks -isystem %S/Inputs/Headers %s -Wnonportable-private-system-apinotes-path 2>&1 | FileCheck %s
+
+#include <ModuleWithWrongCase.h>
+#include <ModuleWithWrongCasePrivate.h>
+#include <FrameworkWithWrongCase/FrameworkWithWrongCase.h>
+#include <FrameworkWithWrongCasePrivate/FrameworkWithWrongCasePrivate.h>
+#include <FrameworkWithActualPrivateModule/FrameworkWithActualPrivateModule_Private.h>
+
+// CHECK-NOT: warning:
+// CHECK: warning: private API notes file for module 'ModuleWithWrongCasePrivate' should be named 'ModuleWithWrongCasePrivate_private.apinotes', not 'ModuleWithWrongCasePrivate_Private.apinotes'
+// CHECK-NOT: warning:
+// CHECK: warning: private API notes file for module 'FrameworkWithWrongCasePrivate' should be named 'FrameworkWithWrongCasePrivate_private.apinotes', not 'FrameworkWithWrongCasePrivate_Private.apinotes'
+// CHECK-NOT: warning:
diff --git a/clang/test/APINotes/extern-context.cpp b/clang/test/APINotes/extern-context.cpp
new file mode 100644
index 000000000000..331dee002361
--- /dev/null
+++ b/clang/test/APINotes/extern-context.cpp
@@ -0,0 +1,23 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers %s -ast-dump -ast-dump-filter globalInExternC -x c++ | FileCheck -check-prefix=CHECK-EXTERN-C %s
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers %s -ast-dump -ast-dump-filter globalInExternCXX -x c++ | FileCheck -check-prefix=CHECK-EXTERN-CXX %s
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers %s -ast-dump -ast-dump-filter globalFuncInExternC -x c++ | FileCheck -check-prefix=CHECK-FUNC-EXTERN-C %s
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers %s -ast-dump -ast-dump-filter globalFuncInExternCXX -x c++ | FileCheck -check-prefix=CHECK-FUNC-EXTERN-CXX %s
+
+#include "ExternCtx.h"
+
+// CHECK-EXTERN-C: Dumping globalInExternC:
+// CHECK-EXTERN-C: VarDecl {{.+}} imported in ExternCtx globalInExternC 'int'
+// CHECK-EXTERN-C: UnavailableAttr {{.+}} <<invalid sloc>> "oh no"
+
+// CHECK-EXTERN-CXX: Dumping globalInExternCXX:
+// CHECK-EXTERN-CXX: VarDecl {{.+}} imported in ExternCtx globalInExternCXX 'int'
+// CHECK-EXTERN-CXX: UnavailableAttr {{.+}} <<invalid sloc>> "oh no #2"
+
+// CHECK-FUNC-EXTERN-C: Dumping globalFuncInExternC:
+// CHECK-FUNC-EXTERN-C: FunctionDecl {{.+}} imported in ExternCtx globalFuncInExternC 'void ()'
+// CHECK-FUNC-EXTERN-C: UnavailableAttr {{.+}} <<invalid sloc>> "oh no #3"
+
+// CHECK-FUNC-EXTERN-CXX: Dumping globalFuncInExternCXX:
+// CHECK-FUNC-EXTERN-CXX: FunctionDecl {{.+}} imported in ExternCtx globalFuncInExternCXX 'void ()'
+// CHECK-FUNC-EXTERN-CXX: UnavailableAttr {{.+}} <<invalid sloc>> "oh no #4"
diff --git a/clang/test/APINotes/instancetype.m b/clang/test/APINotes/instancetype.m
new file mode 100644
index 000000000000..30339e5386f6
--- /dev/null
+++ b/clang/test/APINotes/instancetype.m
@@ -0,0 +1,9 @@
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -verify %s
+
+@import InstancetypeModule;
+
+void test() {
+  // The nullability is here to verify that the API notes were applied.
+  int good = [SomeSubclass instancetypeFactoryMethod]; // expected-error {{initializing 'int' with an expression of type 'SomeSubclass * _Nonnull'}}
+  int bad = [SomeSubclass staticFactoryMethod]; // expected-error {{initializing 'int' with an expression of type 'SomeBaseClass * _Nonnull'}}
+}
diff --git a/clang/test/APINotes/module-cache.m b/clang/test/APINotes/module-cache.m
new file mode 100644
index 000000000000..5dcaf1181f9d
--- /dev/null
+++ b/clang/test/APINotes/module-cache.m
@@ -0,0 +1,65 @@
+// RUN: rm -rf %t
+
+// Set up directories
+// RUN: mkdir -p %t/APINotes
+// RUN: cp %S/Inputs/APINotes/SomeOtherKit.apinotes %t/APINotes/SomeOtherKit.apinotes
+// RUN: mkdir -p %t/Frameworks
+// RUN: cp -r %S/Inputs/Frameworks/SomeOtherKit.framework %t/Frameworks
+
+// First build: check that 'methodB' is unavailable but 'methodA' is available.
+// RUN: not %clang_cc1 -fmodules -fimplicit-module-maps -Rmodule-build -fmodules-cache-path=%t/ModulesCache -iapinotes-modules %t/APINotes  -F %t/Frameworks %s > %t/before.log 2>&1
+// RUN: FileCheck -check-prefix=CHECK-METHODB %s < %t/before.log
+// RUN: FileCheck -check-prefix=CHECK-REBUILD %s < %t/before.log
+// RUN: FileCheck -check-prefix=CHECK-ONE-ERROR %s < %t/before.log
+
+// Do it again; now we're using caches.
+// RUN: not %clang_cc1 -fmodules -fimplicit-module-maps -Rmodule-build -fmodules-cache-path=%t/ModulesCache -iapinotes-modules %t/APINotes  -F %t/Frameworks %s > %t/before.log 2>&1
+// RUN: FileCheck -check-prefix=CHECK-METHODB %s < %t/before.log
+// RUN: FileCheck -check-prefix=CHECK-WITHOUT-REBUILD %s < %t/before.log
+// RUN: FileCheck -check-prefix=CHECK-ONE-ERROR %s < %t/before.log
+
+// Add a blank line to the header to force the module to rebuild, without
+// (yet) changing API notes.
+// RUN: echo >> %t/Frameworks/SomeOtherKit.framework/Headers/SomeOtherKit.h
+// RUN: not %clang_cc1 -fmodules -fimplicit-module-maps -Rmodule-build -fmodules-cache-path=%t/ModulesCache -iapinotes-modules %t/APINotes  -F %t/Frameworks %s > %t/before.log 2>&1
+// RUN: FileCheck -check-prefix=CHECK-METHODB %s < %t/before.log
+// RUN: FileCheck -check-prefix=CHECK-REBUILD %s < %t/before.log
+// RUN: FileCheck -check-prefix=CHECK-ONE-ERROR %s < %t/before.log
+
+// Change the API notes file, after the module has rebuilt once.
+// RUN: echo '      - Selector: "methodA"' >> %t/APINotes/SomeOtherKit.apinotes
+// RUN: echo '        MethodKind: Instance' >> %t/APINotes/SomeOtherKit.apinotes
+// RUN: echo '        Availability: none' >> %t/APINotes/SomeOtherKit.apinotes
+// RUN: echo '        AvailabilityMsg: "not here either"' >> %t/APINotes/SomeOtherKit.apinotes
+
+// Build again: check that both methods are now unavailable and that the module rebuilt.
+// RUN: not %clang_cc1 -fmodules -fimplicit-module-maps -Rmodule-build -fmodules-cache-path=%t/ModulesCache -iapinotes-modules %t/APINotes  -F %t/Frameworks %s > %t/after.log 2>&1
+// RUN: FileCheck -check-prefix=CHECK-METHODA %s < %t/after.log
+// RUN: FileCheck -check-prefix=CHECK-METHODB %s < %t/after.log
+// RUN: FileCheck -check-prefix=CHECK-REBUILD %s < %t/after.log
+// RUN: FileCheck -check-prefix=CHECK-TWO-ERRORS %s < %t/after.log
+
+// Run the build again: check that both methods are now unavailable
+// RUN: not %clang_cc1 -fmodules -fimplicit-module-maps -Rmodule-build -fmodules-cache-path=%t/ModulesCache -iapinotes-modules %t/APINotes  -F %t/Frameworks %s > %t/after.log 2>&1
+// RUN: FileCheck -check-prefix=CHECK-METHODA %s < %t/after.log
+// RUN: FileCheck -check-prefix=CHECK-METHODB %s < %t/after.log
+// RUN: FileCheck -check-prefix=CHECK-WITHOUT-REBUILD %s < %t/after.log
+// RUN: FileCheck -check-prefix=CHECK-TWO-ERRORS %s < %t/after.log
+
+@import SomeOtherKit;
+
+void test(A *a) {
+  // CHECK-METHODA: error: 'methodA' is unavailable: not here either
+  [a methodA];
+
+  // CHECK-METHODB: error: 'methodB' is unavailable: anything but this
+  [a methodB];
+}
+
+// CHECK-REBUILD: remark: building module{{.*}}SomeOtherKit
+
+// CHECK-WITHOUT-REBUILD-NOT: remark: building module{{.*}}SomeOtherKit
+
+// CHECK-ONE-ERROR: 1 error generated.
+// CHECK-TWO-ERRORS: 2 errors generated.
+
diff --git a/clang/test/APINotes/namespaces.cpp b/clang/test/APINotes/namespaces.cpp
new file mode 100644
index 000000000000..2f9d93c2ea0e
--- /dev/null
+++ b/clang/test/APINotes/namespaces.cpp
@@ -0,0 +1,69 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -x objective-c++
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter Namespace1::my_typedef -x objective-c++ | FileCheck -check-prefix=CHECK-TYPEDEF-IN-NAMESPACE %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter Namespace1::my_using_decl -x objective-c++ | FileCheck -check-prefix=CHECK-USING-DECL-IN-NAMESPACE %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter Namespace1::varInNamespace -x objective-c++ | FileCheck -check-prefix=CHECK-GLOBAL-IN-NAMESPACE %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter Namespace1::funcInNamespace -x objective-c++ | FileCheck -check-prefix=CHECK-FUNC-IN-NAMESPACE %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter Namespace1::char_box -x objective-c++ | FileCheck -check-prefix=CHECK-STRUCT-IN-NAMESPACE %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter Namespace1::Nested1::varInNestedNamespace -x objective-c++ | FileCheck -check-prefix=CHECK-GLOBAL-IN-NESTED-NAMESPACE %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter Namespace1::Nested2::varInNestedNamespace -x objective-c++ | FileCheck -check-prefix=CHECK-ANOTHER-GLOBAL-IN-NESTED-NAMESPACE %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter Namespace1::Nested1::char_box -x objective-c++ | FileCheck -check-prefix=CHECK-STRUCT-IN-NESTED-NAMESPACE %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter Namespace1::Nested1::funcInNestedNamespace -x objective-c++ | FileCheck -check-prefix=CHECK-FUNC-IN-NESTED-NAMESPACE %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter Namespace1::Nested1::Namespace1::char_box -x objective-c++ | FileCheck -check-prefix=CHECK-STRUCT-IN-DEEP-NESTED-NAMESPACE %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter varInInlineNamespace -x objective-c++ | FileCheck -check-prefix=CHECK-GLOBAL-IN-INLINE-NAMESPACE %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/CxxInterop -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter funcInInlineNamespace -x objective-c++ | FileCheck -check-prefix=CHECK-FUNC-IN-INLINE-NAMESPACE %s
+
+#import <Namespaces.h>
+
+// CHECK-TYPEDEF-IN-NAMESPACE: Dumping Namespace1::my_typedef:
+// CHECK-TYPEDEF-IN-NAMESPACE-NEXT: TypedefDecl {{.+}} imported in Namespaces my_typedef 'int'
+// CHECK-TYPEDEF-IN-NAMESPACE: SwiftNameAttr {{.+}} <<invalid sloc>> "SwiftTypedef"
+
+// CHECK-USING-DECL-IN-NAMESPACE: Dumping Namespace1::my_using_decl:
+// CHECK-USING-DECL-IN-NAMESPACE-NEXT: TypeAliasDecl {{.+}} imported in Namespaces my_using_decl 'int'
+// CHECK-USING-DECL-IN-NAMESPACE: SwiftNameAttr {{.+}} <<invalid sloc>> "SwiftUsingDecl"
+
+// CHECK-GLOBAL-IN-NAMESPACE: Dumping Namespace1::varInNamespace:
+// CHECK-GLOBAL-IN-NAMESPACE-NEXT: VarDecl {{.+}} imported in Namespaces varInNamespace 'int' static cinit
+// CHECK-GLOBAL-IN-NAMESPACE-NEXT: IntegerLiteral {{.+}} 'int' 1
+// CHECK-GLOBAL-IN-NAMESPACE-NEXT: SwiftNameAttr {{.+}} <<invalid sloc>> "swiftVarInNamespace"
+
+// CHECK-FUNC-IN-NAMESPACE: Dumping Namespace1::funcInNamespace:
+// CHECK-FUNC-IN-NAMESPACE-NEXT: FunctionDecl {{.+}} imported in Namespaces funcInNamespace 'void ()'
+// CHECK-FUNC-IN-NAMESPACE-NEXT: SwiftNameAttr {{.+}} <<invalid sloc>> "swiftFuncInNamespace()"
+
+// CHECK-STRUCT-IN-NAMESPACE: Dumping Namespace1::char_box:
+// CHECK-STRUCT-IN-NAMESPACE-NEXT: CXXRecordDecl {{.+}} imported in Namespaces <undeserialized declarations> struct char_box
+// CHECK-STRUCT-IN-NAMESPACE: SwiftNameAttr {{.+}} <<invalid sloc>> "CharBox"
+
+// CHECK-GLOBAL-IN-NESTED-NAMESPACE: Dumping Namespace1::Nested1::varInNestedNamespace:
+// CHECK-GLOBAL-IN-NESTED-NAMESPACE-NEXT: VarDecl {{.+}} imported in Namespaces varInNestedNamespace 'int' static cinit
+// CHECK-GLOBAL-IN-NESTED-NAMESPACE-NEXT: IntegerLiteral {{.+}} 'int' 1
+// CHECK-GLOBAL-IN-NESTED-NAMESPACE-NEXT: SwiftNameAttr {{.+}} <<invalid sloc>> "swiftVarInNestedNamespace"
+
+// CHECK-ANOTHER-GLOBAL-IN-NESTED-NAMESPACE: Dumping Namespace1::Nested2::varInNestedNamespace:
+// CHECK-ANOTHER-GLOBAL-IN-NESTED-NAMESPACE-NEXT: VarDecl {{.+}} imported in Namespaces varInNestedNamespace 'int' static cinit
+// CHECK-ANOTHER-GLOBAL-IN-NESTED-NAMESPACE-NEXT: IntegerLiteral {{.+}} 'int' 2
+// CHECK-ANOTHER-GLOBAL-IN-NESTED-NAMESPACE-NEXT: SwiftNameAttr {{.+}} <<invalid sloc>> "swiftAnotherVarInNestedNamespace"
+
+// CHECK-FUNC-IN-NESTED-NAMESPACE: Dumping Namespace1::Nested1::funcInNestedNamespace:
+// CHECK-FUNC-IN-NESTED-NAMESPACE-NEXT: FunctionDecl {{.+}} imported in Namespaces funcInNestedNamespace 'void (int)'
+// CHECK-FUNC-IN-NESTED-NAMESPACE-NEXT: ParmVarDecl {{.+}} i 'int'
+// CHECK-FUNC-IN-NESTED-NAMESPACE-NEXT: SwiftNameAttr {{.+}} <<invalid sloc>> "swiftFuncInNestedNamespace(_:)"
+
+// CHECK-STRUCT-IN-NESTED-NAMESPACE: Dumping Namespace1::Nested1::char_box:
+// CHECK-STRUCT-IN-NESTED-NAMESPACE-NEXT: CXXRecordDecl {{.+}} imported in Namespaces <undeserialized declarations> struct char_box
+// CHECK-STRUCT-IN-NESTED-NAMESPACE: SwiftNameAttr {{.+}} <<invalid sloc>> "NestedCharBox"
+
+// CHECK-STRUCT-IN-DEEP-NESTED-NAMESPACE: Dumping Namespace1::Nested1::Namespace1::char_box:
+// CHECK-STRUCT-IN-DEEP-NESTED-NAMESPACE-NEXT: CXXRecordDecl {{.+}} imported in Namespaces <undeserialized declarations> struct char_box
+// CHECK-STRUCT-IN-DEEP-NESTED-NAMESPACE: SwiftNameAttr {{.+}} <<invalid sloc>> "DeepNestedCharBox"
+
+// CHECK-GLOBAL-IN-INLINE-NAMESPACE: Dumping varInInlineNamespace:
+// CHECK-GLOBAL-IN-INLINE-NAMESPACE-NEXT: VarDecl {{.+}} imported in Namespaces varInInlineNamespace 'int' static cinit
+// CHECK-GLOBAL-IN-INLINE-NAMESPACE-NEXT: IntegerLiteral {{.+}} 'int' 3
+// CHECK-GLOBAL-IN-INLINE-NAMESPACE-NEXT: SwiftNameAttr {{.+}} <<invalid sloc>> "swiftVarInInlineNamespace"
+
+// CHECK-FUNC-IN-INLINE-NAMESPACE: Dumping funcInInlineNamespace:
+// CHECK-FUNC-IN-INLINE-NAMESPACE-NEXT: FunctionDecl {{.+}} imported in Namespaces funcInInlineNamespace 'void ()'
+// CHECK-FUNC-IN-INLINE-NAMESPACE-NEXT: SwiftNameAttr {{.+}} <<invalid sloc>> "swiftFuncInInlineNamespace()"
diff --git a/clang/test/APINotes/nullability.c b/clang/test/APINotes/nullability.c
new file mode 100644
index 000000000000..e07fc2e5c117
--- /dev/null
+++ b/clang/test/APINotes/nullability.c
@@ -0,0 +1,21 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -verify
+
+#include "HeaderLib.h"
+
+int main() {
+  custom_realloc(0, 0); // expected-warning{{null passed to a callee that requires a non-null argument}}
+  int i = 0;
+  do_something_with_pointers(&i, 0);
+  do_something_with_pointers(0, &i); // expected-warning{{null passed to a callee that requires a non-null argument}}
+  
+  extern void *p;
+  do_something_with_arrays(0, p); // expected-warning{{null passed to a callee that requires a non-null argument}}
+  do_something_with_arrays(p, 0); // expected-warning{{null passed to a callee that requires a non-null argument}}
+
+  take_pointer_and_int(0, 0); // expected-warning{{null passed to a callee that requires a non-null argument}}
+
+  float *fp = global_int; // expected-warning{{incompatible pointer types initializing 'float *' with an expression of type 'int * _Nonnull'}}
+  return 0;
+}
+
diff --git a/clang/test/APINotes/nullability.m b/clang/test/APINotes/nullability.m
new file mode 100644
index 000000000000..21ec6680fa71
--- /dev/null
+++ b/clang/test/APINotes/nullability.m
@@ -0,0 +1,46 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fapinotes-modules -Wno-private-module -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -verify
+
+// Test with Swift version 3.0. This should only affect the few APIs that have an entry in the 3.0 tables.
+
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fapinotes-modules -Wno-private-module -fapinotes-swift-version=3.0 -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -verify -DSWIFT_VERSION_3_0 -fmodules-ignore-macro=SWIFT_VERSION_3_0
+
+#import <SomeKit/SomeKit.h>
+
+int main() {
+  A *a;
+
+#if SWIFT_VERSION_3_0
+  float *fp =  // expected-warning{{incompatible pointer types initializing 'float *' with an expression of type 'A * _Nullable'}}
+    [a transform: 0 integer: 0];
+#else
+  float *fp =  // expected-warning{{incompatible pointer types initializing 'float *' with an expression of type 'A *'}}
+    [a transform: 0 integer: 0]; // expected-warning{{null passed to a callee that requires a non-null argument}}
+#endif
+
+  [a setNonnullAInstance: 0]; // expected-warning{{null passed to a callee that requires a non-null argument}}
+  [A setNonnullAInstance: 0]; // no warning
+  a.nonnullAInstance = 0; // expected-warning{{null passed to a callee that requires a non-null argument}}
+  A* _Nonnull aPtr = a.nonnullAInstance; // no warning
+
+  [a setNonnullAClass: 0]; // no warning
+  [A setNonnullAClass: 0]; // expected-warning{{null passed to a callee that requires a non-null argument}}
+
+  [a setNonnullABoth: 0]; // expected-warning{{null passed to a callee that requires a non-null argument}}
+  [A setNonnullABoth: 0]; // expected-warning{{null passed to a callee that requires a non-null argument}}
+
+  [a setInternalProperty: 0]; // expected-warning{{null passed to a callee that requires a non-null argument}}
+
+#if SWIFT_VERSION_3_0
+  // Version 3 information overrides header information.
+  [a setExplicitNonnullInstance: 0]; //  okay
+  [a setExplicitNullableInstance: 0]; // expected-warning{{null passed to a callee that requires a non-null argument}}
+#else
+  // Header information overrides unversioned information.
+  [a setExplicitNonnullInstance: 0]; // expected-warning{{null passed to a callee that requires a non-null argument}}
+  [a setExplicitNullableInstance: 0]; // okay
+#endif
+
+  return 0;
+}
+
diff --git a/clang/test/APINotes/objc-forward-declarations.m b/clang/test/APINotes/objc-forward-declarations.m
new file mode 100644
index 000000000000..e82bed205550
--- /dev/null
+++ b/clang/test/APINotes/objc-forward-declarations.m
@@ -0,0 +1,12 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fapinotes-modules -fsyntax-only -F %S/Inputs/Frameworks %s -verify
+
+@import LayeredKit;
+
+void test(
+  UpwardClass *okayClass,
+  id <UpwardProto> okayProto,
+  PerfectlyNormalClass *badClass // expected-error {{'PerfectlyNormalClass' is unavailable}}
+) {
+  // expected-note@LayeredKitImpl/LayeredKitImpl.h:4 {{'PerfectlyNormalClass' has been explicitly marked unavailable here}}
+}
diff --git a/clang/test/APINotes/objc_designated_inits.m b/clang/test/APINotes/objc_designated_inits.m
new file mode 100644
index 000000000000..1f2b8ed534b7
--- /dev/null
+++ b/clang/test/APINotes/objc_designated_inits.m
@@ -0,0 +1,17 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fapinotes-modules -Wno-private-module -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -verify
+
+#include "HeaderLib.h"
+#import <SomeKit/SomeKit.h>
+
+@interface CSub : C
+-(instancetype)initWithA:(A*)a;
+@end
+
+@implementation CSub
+-(instancetype)initWithA:(A*)a { // expected-warning{{designated initializer missing a 'super' call to a designated initializer of the super class}}
+  // expected-note@SomeKit/SomeKit.h:20 2{{method marked as designated initializer of the class here}}
+  self = [super init]; // expected-warning{{designated initializer invoked a non-designated initializer}}
+  return self;
+}
+@end
diff --git a/clang/test/APINotes/properties.m b/clang/test/APINotes/properties.m
new file mode 100644
index 000000000000..f218092a66e1
--- /dev/null
+++ b/clang/test/APINotes/properties.m
@@ -0,0 +1,42 @@
+// RUN: rm -rf %t && mkdir -p %t
+
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fapinotes-modules  -fblocks -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter 'TestProperties::' | FileCheck -check-prefix=CHECK -check-prefix=CHECK-4 %s
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fapinotes-modules  -fblocks -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter 'TestProperties::' -fapinotes-swift-version=3 | FileCheck -check-prefix=CHECK -check-prefix=CHECK-3 %s
+
+@import VersionedKit;
+
+// CHECK-LABEL: ObjCPropertyDecl {{.+}} accessorsOnly 'id'
+// CHECK-NEXT: SwiftImportPropertyAsAccessorsAttr {{.+}} <<invalid sloc>>
+// CHECK-NOT: Attr
+
+// CHECK-LABEL: ObjCPropertyDecl {{.+}} accessorsOnlyForClass 'id'
+// CHECK-NEXT: SwiftImportPropertyAsAccessorsAttr {{.+}} <<invalid sloc>>
+// CHECK-NOT: Attr
+
+// CHECK-LABEL: ObjCPropertyDecl {{.+}} accessorsOnlyInVersion3 'id'
+// CHECK-3-NEXT: SwiftImportPropertyAsAccessorsAttr {{.+}} <<invalid sloc>>
+// CHECK-4-NEXT: SwiftVersionedAdditionAttr {{.+}} 3.0{{$}}
+// CHECK-4-NEXT: SwiftImportPropertyAsAccessorsAttr {{.+}} <<invalid sloc>>
+// CHECK-NOT: Attr
+
+// CHECK-LABEL: ObjCPropertyDecl {{.+}} accessorsOnlyForClassInVersion3 'id'
+// CHECK-3-NEXT: SwiftImportPropertyAsAccessorsAttr {{.+}} <<invalid sloc>>
+// CHECK-4-NEXT: SwiftVersionedAdditionAttr {{.+}} 3.0{{$}}
+// CHECK-4-NEXT: SwiftImportPropertyAsAccessorsAttr {{.+}} <<invalid sloc>>
+// CHECK-NOT: Attr
+
+// CHECK-LABEL: ObjCPropertyDecl {{.+}} accessorsOnlyExceptInVersion3 'id'
+// CHECK-3-NEXT: SwiftVersionedAdditionAttr {{.+}} Implicit 3.0 IsReplacedByActive{{$}}
+// CHECK-3-NEXT: SwiftImportPropertyAsAccessorsAttr {{.+}} <<invalid sloc>>
+// CHECK-4-NEXT: SwiftImportPropertyAsAccessorsAttr {{.+}} <<invalid sloc>>
+// CHECK-4-NEXT: SwiftVersionedRemovalAttr {{.+}} Implicit 3.0 {{[0-9]+}}
+// CHECK-NOT: Attr
+
+// CHECK-LABEL: ObjCPropertyDecl {{.+}} accessorsOnlyForClassExceptInVersion3 'id'
+// CHECK-3-NEXT: SwiftVersionedAdditionAttr {{.+}} Implicit 3.0 IsReplacedByActive{{$}}
+// CHECK-3-NEXT: SwiftImportPropertyAsAccessorsAttr {{.+}} <<invalid sloc>>
+// CHECK-4-NEXT: SwiftImportPropertyAsAccessorsAttr {{.+}} <<invalid sloc>>
+// CHECK-4-NEXT: SwiftVersionedRemovalAttr {{.+}} Implicit 3.0 {{[0-9]+}}
+// CHECK-NOT: Attr
+
+// CHECK-LABEL: Decl
diff --git a/clang/test/APINotes/retain-count-convention.m b/clang/test/APINotes/retain-count-convention.m
new file mode 100644
index 000000000000..4bf9610a352a
--- /dev/null
+++ b/clang/test/APINotes/retain-count-convention.m
@@ -0,0 +1,38 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fapinotes-modules  -fdisable-module-hash -fsyntax-only -F %S/Inputs/Frameworks %s
+// RUN: %clang_cc1 -ast-print %t/ModulesCache/SimpleKit.pcm | FileCheck %s
+// RUN: %clang_cc1 -ast-dump -ast-dump-filter 'DUMP' %t/ModulesCache/SimpleKit.pcm | FileCheck -check-prefix CHECK-DUMP %s
+
+#import <SimpleKit/SimpleKit.h>
+
+// CHECK: void *getCFOwnedToUnowned(void) __attribute__((cf_returns_not_retained));
+// CHECK: void *getCFUnownedToOwned(void) __attribute__((cf_returns_retained));
+// CHECK: void *getCFOwnedToNone(void) __attribute__((cf_unknown_transfer));
+// CHECK: id getObjCOwnedToUnowned(void) __attribute__((ns_returns_not_retained));
+// CHECK: id getObjCUnownedToOwned(void) __attribute__((ns_returns_retained));
+// CHECK: int indirectGetCFOwnedToUnowned(void * _Nullable *out __attribute__((cf_returns_not_retained)));
+// CHECK: int indirectGetCFUnownedToOwned(void * _Nullable *out __attribute__((cf_returns_retained)));
+// CHECK: int indirectGetCFOwnedToNone(void * _Nullable *out);
+// CHECK: int indirectGetCFNoneToOwned(void **out __attribute__((cf_returns_not_retained)));
+
+// CHECK-LABEL: @interface MethodTest
+// CHECK: - (id)getOwnedToUnowned __attribute__((ns_returns_not_retained));
+// CHECK: - (id)getUnownedToOwned __attribute__((ns_returns_retained));
+// CHECK: @end
+
+// CHECK-DUMP-LABEL: Dumping getCFAuditedToUnowned_DUMP:
+// CHECK-DUMP-NEXT: FunctionDecl
+// CHECK-DUMP-NEXT: CFReturnsNotRetainedAttr
+// CHECK-DUMP-NEXT: CFAuditedTransferAttr
+// CHECK-DUMP-NOT: Attr
+
+// CHECK-DUMP-LABEL: Dumping getCFAuditedToOwned_DUMP:
+// CHECK-DUMP-NEXT: FunctionDecl
+// CHECK-DUMP-NEXT: CFReturnsRetainedAttr
+// CHECK-DUMP-NEXT: CFAuditedTransferAttr
+// CHECK-DUMP-NOT: Attr
+
+// CHECK-DUMP-LABEL: Dumping getCFAuditedToNone_DUMP:
+// CHECK-DUMP-NEXT: FunctionDecl
+// CHECK-DUMP-NEXT: CFUnknownTransferAttr
+// CHECK-DUMP-NOT: Attr
diff --git a/clang/test/APINotes/search-order.m b/clang/test/APINotes/search-order.m
new file mode 100644
index 000000000000..17e81d5eb2d6
--- /dev/null
+++ b/clang/test/APINotes/search-order.m
@@ -0,0 +1,25 @@
+// RUN: rm -rf %t && mkdir -p %t
+
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -DFROM_FRAMEWORK=1 -verify
+
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -iapinotes-modules %S/Inputs/APINotes  -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -DFROM_SEARCH_PATH=1 -verify
+
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fapinotes-modules -iapinotes-modules %S/Inputs/APINotes  -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -DFROM_FRAMEWORK=1 -verify
+
+@import SomeOtherKit;
+
+void test(A *a) {
+#if FROM_FRAMEWORK
+  [a methodA]; // expected-error{{unavailable}}
+  [a methodB];
+
+  // expected-note@SomeOtherKit/SomeOtherKit.h:5{{'methodA' has been explicitly marked unavailable here}}
+#elif FROM_SEARCH_PATH
+  [a methodA];
+  [a methodB]; // expected-error{{unavailable}}
+
+  // expected-note@SomeOtherKit/SomeOtherKit.h:6{{'methodB' has been explicitly marked unavailable here}}
+#else
+#  error Not something we need to test
+#endif
+}
diff --git a/clang/test/APINotes/swift-import-as.cpp b/clang/test/APINotes/swift-import-as.cpp
new file mode 100644
index 000000000000..904857e58593
--- /dev/null
+++ b/clang/test/APINotes/swift-import-as.cpp
@@ -0,0 +1,16 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers %s -x c++
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers %s -x c++ -ast-dump -ast-dump-filter ImmortalRefType | FileCheck -check-prefix=CHECK-IMMORTAL %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers %s -x c++ -ast-dump -ast-dump-filter RefCountedType | FileCheck -check-prefix=CHECK-REF-COUNTED %s
+
+#include <SwiftImportAs.h>
+
+// CHECK-IMMORTAL: Dumping ImmortalRefType:
+// CHECK-IMMORTAL-NEXT: CXXRecordDecl {{.+}} imported in SwiftImportAs {{.+}} struct ImmortalRefType
+// CHECK-IMMORTAL: SwiftAttrAttr {{.+}} <<invalid sloc>> "import_reference"
+
+// CHECK-REF-COUNTED: Dumping RefCountedType:
+// CHECK-REF-COUNTED-NEXT: CXXRecordDecl {{.+}} imported in SwiftImportAs {{.+}} struct RefCountedType
+// CHECK-REF-COUNTED: SwiftAttrAttr {{.+}} <<invalid sloc>> "import_reference"
+// CHECK-REF-COUNTED: SwiftAttrAttr {{.+}} <<invalid sloc>> "retain:RCRetain"
+// CHECK-REF-COUNTED: SwiftAttrAttr {{.+}} <<invalid sloc>> "release:RCRelease"
diff --git a/clang/test/APINotes/top-level-private-modules.c b/clang/test/APINotes/top-level-private-modules.c
new file mode 100644
index 000000000000..0da72b2e36f4
--- /dev/null
+++ b/clang/test/APINotes/top-level-private-modules.c
@@ -0,0 +1,8 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fapinotes-modules -Wno-private-module -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -verify
+
+#include <PrivateLib.h>
+#include <TopLevelPrivateKit/TopLevelPrivateKit_Private.h>
+
+void *testPlain = PrivateLib; // expected-error {{initializing 'void *' with an expression of incompatible type 'float'}}
+void *testFramework = TopLevelPrivateKit_Private; // expected-error {{initializing 'void *' with an expression of incompatible type 'float'}}
diff --git a/clang/test/APINotes/types.m b/clang/test/APINotes/types.m
new file mode 100644
index 000000000000..133d504713d7
--- /dev/null
+++ b/clang/test/APINotes/types.m
@@ -0,0 +1,28 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fapinotes-modules -Wno-private-module -fdisable-module-hash -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -verify
+// RUN: %clang_cc1 -ast-print %t/ModulesCache/SimpleKit.pcm | FileCheck %s
+
+#import <SomeKit/SomeKit.h>
+#import <SimpleKit/SimpleKit.h>
+
+// CHECK: struct __attribute__((swift_name("SuccessfullyRenamedA"))) RenamedAgainInAPINotesA {
+// CHECK: struct __attribute__((swift_name("SuccessfullyRenamedB"))) RenamedAgainInAPINotesB {
+
+void test(OverriddenTypes *overridden) {
+  int *ip1 = global_int_ptr; // expected-warning{{incompatible pointer types initializing 'int *' with an expression of type 'double (*)(int, int)'}}
+
+  int *ip2 = global_int_fun( // expected-warning{{incompatible pointer types initializing 'int *' with an expression of type 'char *'}}
+               ip2, // expected-warning{{incompatible pointer types passing 'int *' to parameter of type 'double *'}}
+               ip2); // expected-warning{{incompatible pointer types passing 'int *' to parameter of type 'float *'}}
+
+  int *ip3 = [overridden // expected-warning{{incompatible pointer types initializing 'int *' with an expression of type 'char *'}}
+                methodToMangle: ip3 // expected-warning{{incompatible pointer types sending 'int *' to parameter of type 'double *'}}
+                        second: ip3]; // expected-warning{{incompatible pointer types sending 'int *' to parameter of type 'float *'}}
+
+  int *ip4 = overridden.intPropertyToMangle; // expected-warning{{incompatible pointer types initializing 'int *' with an expression of type 'double *'}}
+}
+
+// expected-note@SomeKit/SomeKit.h:42{{passing argument to parameter 'ptr' here}}
+// expected-note@SomeKit/SomeKit.h:42{{passing argument to parameter 'ptr2' here}}
+// expected-note@SomeKit/SomeKit.h:48{{passing argument to parameter 'ptr1' here}}
+// expected-note@SomeKit/SomeKit.h:48{{passing argument to parameter 'ptr2' here}}
diff --git a/clang/test/APINotes/versioned-multi.c b/clang/test/APINotes/versioned-multi.c
new file mode 100644
index 000000000000..48c51fd932e1
--- /dev/null
+++ b/clang/test/APINotes/versioned-multi.c
@@ -0,0 +1,69 @@
+// RUN: rm -rf %t && mkdir -p %t
+
+// Build and check the unversioned module file.
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/Unversioned -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s
+// RUN: %clang_cc1 -ast-print %t/ModulesCache/Unversioned/VersionedKit.pcm | FileCheck -check-prefix=CHECK-UNVERSIONED %s
+
+// Build and check the various versions.
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/Versioned3 -fdisable-module-hash -fapinotes-modules -fapinotes-swift-version=3 -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s
+// RUN: %clang_cc1 -ast-print %t/ModulesCache/Versioned3/VersionedKit.pcm | FileCheck -check-prefix=CHECK-VERSIONED-3 %s
+
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/Versioned4 -fdisable-module-hash -fapinotes-modules -fapinotes-swift-version=4 -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s
+// RUN: %clang_cc1 -ast-print %t/ModulesCache/Versioned4/VersionedKit.pcm | FileCheck -check-prefix=CHECK-VERSIONED-4 %s
+
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/Versioned5 -fdisable-module-hash -fapinotes-modules -fapinotes-swift-version=5 -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s
+// RUN: %clang_cc1 -ast-print %t/ModulesCache/Versioned5/VersionedKit.pcm | FileCheck -check-prefix=CHECK-VERSIONED-5 %s
+
+#import <VersionedKit/VersionedKit.h>
+
+// CHECK-UNVERSIONED: typedef int MultiVersionedTypedef4;
+// CHECK-UNVERSIONED: typedef int MultiVersionedTypedef4Notes __attribute__((swift_name("MultiVersionedTypedef4Notes_NEW")));
+// CHECK-UNVERSIONED: typedef int MultiVersionedTypedef4Header __attribute__((swift_name("MultiVersionedTypedef4Header_NEW")));
+// CHECK-UNVERSIONED: typedef int MultiVersionedTypedef34;
+// CHECK-UNVERSIONED: typedef int MultiVersionedTypedef34Notes __attribute__((swift_name("MultiVersionedTypedef34Notes_NEW")));
+// CHECK-UNVERSIONED: typedef int MultiVersionedTypedef34Header __attribute__((swift_name("MultiVersionedTypedef34Header_NEW")));
+// CHECK-UNVERSIONED: typedef int MultiVersionedTypedef45;
+// CHECK-UNVERSIONED: typedef int MultiVersionedTypedef45Notes __attribute__((swift_name("MultiVersionedTypedef45Notes_NEW")));
+// CHECK-UNVERSIONED: typedef int MultiVersionedTypedef45Header __attribute__((swift_name("MultiVersionedTypedef45Header_NEW")));
+// CHECK-UNVERSIONED: typedef int MultiVersionedTypedef345;
+// CHECK-UNVERSIONED: typedef int MultiVersionedTypedef345Notes __attribute__((swift_name("MultiVersionedTypedef345Notes_NEW")));
+// CHECK-UNVERSIONED: typedef int MultiVersionedTypedef345Header __attribute__((swift_name("MultiVersionedTypedef345Header_NEW")));
+
+// CHECK-VERSIONED-3: typedef int MultiVersionedTypedef4 __attribute__((swift_name("MultiVersionedTypedef4_4")));
+// CHECK-VERSIONED-3: typedef int MultiVersionedTypedef4Notes __attribute__((swift_name("MultiVersionedTypedef4Notes_4")));
+// CHECK-VERSIONED-3: typedef int MultiVersionedTypedef4Header __attribute__((swift_name("MultiVersionedTypedef4Header_4")));
+// CHECK-VERSIONED-3: typedef int MultiVersionedTypedef34 __attribute__((swift_name("MultiVersionedTypedef34_3")));
+// CHECK-VERSIONED-3: typedef int MultiVersionedTypedef34Notes __attribute__((swift_name("MultiVersionedTypedef34Notes_3")));
+// CHECK-VERSIONED-3: typedef int MultiVersionedTypedef34Header __attribute__((swift_name("MultiVersionedTypedef34Header_3")));
+// CHECK-VERSIONED-3: typedef int MultiVersionedTypedef45 __attribute__((swift_name("MultiVersionedTypedef45_4")));
+// CHECK-VERSIONED-3: typedef int MultiVersionedTypedef45Notes __attribute__((swift_name("MultiVersionedTypedef45Notes_4")));
+// CHECK-VERSIONED-3: typedef int MultiVersionedTypedef45Header __attribute__((swift_name("MultiVersionedTypedef45Header_4")));
+// CHECK-VERSIONED-3: typedef int MultiVersionedTypedef345 __attribute__((swift_name("MultiVersionedTypedef345_3")));
+// CHECK-VERSIONED-3: typedef int MultiVersionedTypedef345Notes __attribute__((swift_name("MultiVersionedTypedef345Notes_3")));
+// CHECK-VERSIONED-3: typedef int MultiVersionedTypedef345Header __attribute__((swift_name("MultiVersionedTypedef345Header_3")));
+
+// CHECK-VERSIONED-4: typedef int MultiVersionedTypedef4 __attribute__((swift_name("MultiVersionedTypedef4_4")));
+// CHECK-VERSIONED-4: typedef int MultiVersionedTypedef4Notes __attribute__((swift_name("MultiVersionedTypedef4Notes_4")));
+// CHECK-VERSIONED-4: typedef int MultiVersionedTypedef4Header __attribute__((swift_name("MultiVersionedTypedef4Header_4")));
+// CHECK-VERSIONED-4: typedef int MultiVersionedTypedef34 __attribute__((swift_name("MultiVersionedTypedef34_4")));
+// CHECK-VERSIONED-4: typedef int MultiVersionedTypedef34Notes __attribute__((swift_name("MultiVersionedTypedef34Notes_4")));
+// CHECK-VERSIONED-4: typedef int MultiVersionedTypedef34Header __attribute__((swift_name("MultiVersionedTypedef34Header_4")));
+// CHECK-VERSIONED-4: typedef int MultiVersionedTypedef45 __attribute__((swift_name("MultiVersionedTypedef45_4")));
+// CHECK-VERSIONED-4: typedef int MultiVersionedTypedef45Notes __attribute__((swift_name("MultiVersionedTypedef45Notes_4")));
+// CHECK-VERSIONED-4: typedef int MultiVersionedTypedef45Header __attribute__((swift_name("MultiVersionedTypedef45Header_4")));
+// CHECK-VERSIONED-4: typedef int MultiVersionedTypedef345 __attribute__((swift_name("MultiVersionedTypedef345_4")));
+// CHECK-VERSIONED-4: typedef int MultiVersionedTypedef345Notes __attribute__((swift_name("MultiVersionedTypedef345Notes_4")));
+// CHECK-VERSIONED-4: typedef int MultiVersionedTypedef345Header __attribute__((swift_name("MultiVersionedTypedef345Header_4")));
+
+// CHECK-VERSIONED-5: typedef int MultiVersionedTypedef4;
+// CHECK-VERSIONED-5: typedef int MultiVersionedTypedef4Notes __attribute__((swift_name("MultiVersionedTypedef4Notes_NEW")));
+// CHECK-VERSIONED-5: typedef int MultiVersionedTypedef4Header __attribute__((swift_name("MultiVersionedTypedef4Header_NEW")));
+// CHECK-VERSIONED-5: typedef int MultiVersionedTypedef34;
+// CHECK-VERSIONED-5: typedef int MultiVersionedTypedef34Notes __attribute__((swift_name("MultiVersionedTypedef34Notes_NEW")));
+// CHECK-VERSIONED-5: typedef int MultiVersionedTypedef34Header __attribute__((swift_name("MultiVersionedTypedef34Header_NEW")));
+// CHECK-VERSIONED-5: typedef int MultiVersionedTypedef45 __attribute__((swift_name("MultiVersionedTypedef45_5")));
+// CHECK-VERSIONED-5: typedef int MultiVersionedTypedef45Notes __attribute__((swift_name("MultiVersionedTypedef45Notes_5")));
+// CHECK-VERSIONED-5: typedef int MultiVersionedTypedef45Header __attribute__((swift_name("MultiVersionedTypedef45Header_5")));
+// CHECK-VERSIONED-5: typedef int MultiVersionedTypedef345 __attribute__((swift_name("MultiVersionedTypedef345_5")));
+// CHECK-VERSIONED-5: typedef int MultiVersionedTypedef345Notes __attribute__((swift_name("MultiVersionedTypedef345Notes_5")));
+// CHECK-VERSIONED-5: typedef int MultiVersionedTypedef345Header __attribute__((swift_name("MultiVersionedTypedef345Header_5")));
diff --git a/clang/test/APINotes/versioned.m b/clang/test/APINotes/versioned.m
new file mode 100644
index 000000000000..61cc8c3f7c4d
--- /dev/null
+++ b/clang/test/APINotes/versioned.m
@@ -0,0 +1,187 @@
+// RUN: rm -rf %t && mkdir -p %t
+
+// Build and check the unversioned module file.
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/Unversioned -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s
+// RUN: %clang_cc1 -ast-print %t/ModulesCache/Unversioned/VersionedKit.pcm | FileCheck -check-prefix=CHECK-UNVERSIONED %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/Unversioned -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter 'DUMP' | FileCheck -check-prefix=CHECK-DUMP -check-prefix=CHECK-UNVERSIONED-DUMP %s
+
+// Build and check the versioned module file.
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/Versioned -fdisable-module-hash -fapinotes-modules -fapinotes-swift-version=3 -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s
+// RUN: %clang_cc1 -ast-print %t/ModulesCache/Versioned/VersionedKit.pcm | FileCheck -check-prefix=CHECK-VERSIONED %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/Versioned -fdisable-module-hash -fapinotes-modules -fapinotes-swift-version=3 -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter 'DUMP' | FileCheck -check-prefix=CHECK-DUMP -check-prefix=CHECK-VERSIONED-DUMP %s
+
+#import <VersionedKit/VersionedKit.h>
+
+// CHECK-UNVERSIONED: void moveToPointDUMP(double x, double y) __attribute__((swift_name("moveTo(x:y:)")));
+// CHECK-VERSIONED: void moveToPointDUMP(double x, double y) __attribute__((swift_name("moveTo(a:b:)")));
+
+// CHECK-DUMP-LABEL: Dumping moveToPointDUMP
+// CHECK-VERSIONED-DUMP: SwiftVersionedAdditionAttr {{.+}} Implicit 3.0 IsReplacedByActive{{$}}
+// CHECK-VERSIONED-DUMP-NEXT: SwiftNameAttr {{.+}} "moveTo(x:y:)"
+// CHECK-VERSIONED-DUMP-NEXT: SwiftNameAttr {{.+}} <<invalid sloc>> "moveTo(a:b:)"
+// CHECK-UNVERSIONED-DUMP: SwiftNameAttr {{.+}} "moveTo(x:y:)"
+// CHECK-UNVERSIONED-DUMP-NEXT: SwiftVersionedAdditionAttr {{.+}} Implicit 3.0{{$}}
+// CHECK-UNVERSIONED-DUMP-NEXT: SwiftNameAttr {{.+}} <<invalid sloc>> "moveTo(a:b:)"
+// CHECK-DUMP-NOT: Attr
+
+// CHECK-DUMP-LABEL: Dumping unversionedRenameDUMP
+// CHECK-DUMP: in VersionedKit unversionedRenameDUMP
+// CHECK-DUMP-NEXT: SwiftVersionedAdditionAttr {{.+}} Implicit 0 IsReplacedByActive{{$}}
+// CHECK-DUMP-NEXT: SwiftNameAttr {{.+}} "unversionedRename_HEADER()"
+// CHECK-DUMP-NEXT: SwiftNameAttr {{.+}} "unversionedRename_NOTES()"
+// CHECK-DUMP-NOT: Attr
+
+// CHECK-DUMP-LABEL: Dumping TestGenericDUMP
+// CHECK-VERSIONED-DUMP: SwiftImportAsNonGenericAttr {{.+}} <<invalid sloc>>
+// CHECK-UNVERSIONED-DUMP: SwiftVersionedAdditionAttr {{.+}} Implicit 3.0{{$}}
+// CHECK-UNVERSIONED-DUMP-NEXT: SwiftImportAsNonGenericAttr {{.+}} <<invalid sloc>>
+// CHECK-DUMP-NOT: Attr
+
+// CHECK-DUMP-LABEL: Dumping Swift3RenamedOnlyDUMP
+// CHECK-DUMP: in VersionedKit Swift3RenamedOnlyDUMP
+// CHECK-VERSIONED-DUMP-NEXT: SwiftVersionedRemovalAttr {{.+}} Implicit 3.0 {{[0-9]+}} IsReplacedByActive{{$}}
+// CHECK-VERSIONED-DUMP-NEXT: SwiftNameAttr {{.+}} "SpecialSwift3Name"
+// CHECK-UNVERSIONED-DUMP-NEXT: SwiftVersionedAdditionAttr {{.+}} Implicit 3.0{{$}}
+// CHECK-UNVERSIONED-DUMP-NEXT: SwiftNameAttr {{.+}} <<invalid sloc>> "SpecialSwift3Name"
+// CHECK-DUMP-NOT: Attr
+
+// CHECK-DUMP-LABEL: Dumping Swift3RenamedAlsoDUMP
+// CHECK-DUMP: in VersionedKit Swift3RenamedAlsoDUMP
+// CHECK-VERSIONED-DUMP-NEXT: SwiftVersionedAdditionAttr {{.+}} Implicit 3.0 IsReplacedByActive{{$}}
+// CHECK-VERSIONED-DUMP-NEXT: SwiftNameAttr {{.+}} <line:{{.+}}, col:{{.+}}> "Swift4Name"
+// CHECK-VERSIONED-DUMP-NEXT: SwiftNameAttr {{.+}} "SpecialSwift3Also"
+// CHECK-UNVERSIONED-DUMP-NEXT: SwiftNameAttr {{.+}} <line:{{.+}}, col:{{.+}}> "Swift4Name"
+// CHECK-UNVERSIONED-DUMP-NEXT: SwiftVersionedAdditionAttr {{.+}} Implicit 3.0{{$}}
+// CHECK-UNVERSIONED-DUMP-NEXT: SwiftNameAttr {{.+}} <<invalid sloc>> "SpecialSwift3Also"
+// CHECK-DUMP-NOT: Attr
+
+// CHECK-DUMP-LABEL: Dumping Swift4RenamedDUMP
+// CHECK-DUMP: in VersionedKit Swift4RenamedDUMP
+// CHECK-VERSIONED-DUMP-NEXT: SwiftVersionedRemovalAttr {{.+}} Implicit 4 {{[0-9]+}} IsReplacedByActive{{$}}
+// CHECK-VERSIONED-DUMP-NEXT: SwiftNameAttr {{.+}} "SpecialSwift4Name"
+// CHECK-UNVERSIONED-DUMP-NEXT: SwiftVersionedAdditionAttr {{.+}} Implicit 4{{$}}
+// CHECK-UNVERSIONED-DUMP-NEXT: SwiftNameAttr {{.+}} <<invalid sloc>> "SpecialSwift4Name"
+// CHECK-DUMP-NOT: Attr
+
+// CHECK-DUMP-NOT: Dumping
+
+// CHECK-UNVERSIONED: void acceptClosure(void (^block)(void) __attribute__((noescape)));
+// CHECK-VERSIONED: void acceptClosure(void (^block)(void));
+
+// CHECK-UNVERSIONED: void privateFunc(void) __attribute__((swift_private));
+
+// CHECK-UNVERSIONED: typedef double MyDoubleWrapper __attribute__((swift_wrapper("struct")));
+
+// CHECK-UNVERSIONED:      enum __attribute__((ns_error_domain(MyErrorDomain))) MyErrorCode {
+// CHECK-UNVERSIONED-NEXT:     MyErrorCodeFailed = 1
+// CHECK-UNVERSIONED-NEXT: };
+
+// CHECK-UNVERSIONED: __attribute__((swift_bridge("MyValueType")))
+// CHECK-UNVERSIONED: @interface MyReferenceType
+
+// CHECK-VERSIONED: void privateFunc(void);
+
+// CHECK-VERSIONED: typedef double MyDoubleWrapper;
+
+// CHECK-VERSIONED:      enum MyErrorCode {
+// CHECK-VERSIONED-NEXT:     MyErrorCodeFailed = 1
+// CHECK-VERSIONED-NEXT: };
+
+// CHECK-VERSIONED-NOT: __attribute__((swift_bridge("MyValueType")))
+// CHECK-VERSIONED: @interface MyReferenceType
+
+// CHECK-UNVERSIONED: __attribute__((swift_objc_members)
+// CHECK-UNVERSIONED-NEXT: @interface TestProperties
+// CHECK-VERSIONED-NOT: __attribute__((swift_objc_members)
+// CHECK-VERSIONED: @interface TestProperties
+
+// CHECK-UNVERSIONED-LABEL: enum __attribute__((flag_enum)) FlagEnum {
+// CHECK-UNVERSIONED-NEXT:     FlagEnumA = 1,
+// CHECK-UNVERSIONED-NEXT:     FlagEnumB = 2
+// CHECK-UNVERSIONED-NEXT: };
+// CHECK-UNVERSIONED-LABEL: enum __attribute__((flag_enum)) NewlyFlagEnum {
+// CHECK-UNVERSIONED-NEXT:     NewlyFlagEnumA = 1,
+// CHECK-UNVERSIONED-NEXT:     NewlyFlagEnumB = 2
+// CHECK-UNVERSIONED-NEXT: };
+// CHECK-UNVERSIONED-LABEL: enum __attribute__((flag_enum)) APINotedFlagEnum {
+// CHECK-UNVERSIONED-NEXT:     APINotedFlagEnumA = 1,
+// CHECK-UNVERSIONED-NEXT:     APINotedFlagEnumB = 2
+// CHECK-UNVERSIONED-NEXT: };
+// CHECK-UNVERSIONED-LABEL: enum  __attribute__((enum_extensibility("open"))) OpenEnum {
+// CHECK-UNVERSIONED-NEXT:     OpenEnumA = 1
+// CHECK-UNVERSIONED-NEXT: };
+// CHECK-UNVERSIONED-LABEL: enum  __attribute__((enum_extensibility("open"))) NewlyOpenEnum {
+// CHECK-UNVERSIONED-NEXT:     NewlyOpenEnumA = 1
+// CHECK-UNVERSIONED-NEXT: };
+// CHECK-UNVERSIONED-LABEL: enum  __attribute__((enum_extensibility("closed"))) NewlyClosedEnum {
+// CHECK-UNVERSIONED-NEXT:     NewlyClosedEnumA = 1
+// CHECK-UNVERSIONED-NEXT: };
+// CHECK-UNVERSIONED-LABEL: enum  __attribute__((enum_extensibility("open"))) ClosedToOpenEnum {
+// CHECK-UNVERSIONED-NEXT:     ClosedToOpenEnumA = 1
+// CHECK-UNVERSIONED-NEXT: };
+// CHECK-UNVERSIONED-LABEL: enum  __attribute__((enum_extensibility("closed"))) OpenToClosedEnum {
+// CHECK-UNVERSIONED-NEXT:     OpenToClosedEnumA = 1
+// CHECK-UNVERSIONED-NEXT: };
+// CHECK-UNVERSIONED-LABEL: enum  __attribute__((enum_extensibility("open"))) APINotedOpenEnum {
+// CHECK-UNVERSIONED-NEXT:     APINotedOpenEnumA = 1
+// CHECK-UNVERSIONED-NEXT: };
+// CHECK-UNVERSIONED-LABEL: enum  __attribute__((enum_extensibility("closed"))) APINotedClosedEnum {
+// CHECK-UNVERSIONED-NEXT:     APINotedClosedEnumA = 1
+// CHECK-UNVERSIONED-NEXT: };
+
+// CHECK-VERSIONED-LABEL: enum __attribute__((flag_enum)) FlagEnum {
+// CHECK-VERSIONED-NEXT:     FlagEnumA = 1,
+// CHECK-VERSIONED-NEXT:     FlagEnumB = 2
+// CHECK-VERSIONED-NEXT: };
+// CHECK-VERSIONED-LABEL: enum NewlyFlagEnum {
+// CHECK-VERSIONED-NEXT:     NewlyFlagEnumA = 1,
+// CHECK-VERSIONED-NEXT:     NewlyFlagEnumB = 2
+// CHECK-VERSIONED-NEXT: };
+// CHECK-VERSIONED-LABEL: enum __attribute__((flag_enum)) APINotedFlagEnum {
+// CHECK-VERSIONED-NEXT:     APINotedFlagEnumA = 1,
+// CHECK-VERSIONED-NEXT:     APINotedFlagEnumB = 2
+// CHECK-VERSIONED-NEXT: };
+// CHECK-VERSIONED-LABEL: enum __attribute__((enum_extensibility("open"))) OpenEnum {
+// CHECK-VERSIONED-NEXT:     OpenEnumA = 1
+// CHECK-VERSIONED-NEXT: };
+// CHECK-VERSIONED-LABEL: enum NewlyOpenEnum {
+// CHECK-VERSIONED-NEXT:     NewlyOpenEnumA = 1
+// CHECK-VERSIONED-NEXT: };
+// CHECK-VERSIONED-LABEL: enum NewlyClosedEnum {
+// CHECK-VERSIONED-NEXT:     NewlyClosedEnumA = 1
+// CHECK-VERSIONED-NEXT: };
+// CHECK-VERSIONED-LABEL: enum __attribute__((enum_extensibility("closed"))) ClosedToOpenEnum {
+// CHECK-VERSIONED-NEXT:     ClosedToOpenEnumA = 1
+// CHECK-VERSIONED-NEXT: };
+// CHECK-VERSIONED-LABEL: enum __attribute__((enum_extensibility("open"))) OpenToClosedEnum {
+// CHECK-VERSIONED-NEXT:     OpenToClosedEnumA = 1
+// CHECK-VERSIONED-NEXT: };
+// CHECK-VERSIONED-LABEL: enum __attribute__((enum_extensibility("open"))) APINotedOpenEnum {
+// CHECK-VERSIONED-NEXT:     APINotedOpenEnumA = 1
+// CHECK-VERSIONED-NEXT: };
+// CHECK-VERSIONED-LABEL: enum __attribute__((enum_extensibility("closed"))) APINotedClosedEnum {
+// CHECK-VERSIONED-NEXT:     APINotedClosedEnumA = 1
+// CHECK-VERSIONED-NEXT: };
+
+// These don't actually have versioned information, so we just check them once.
+// CHECK-UNVERSIONED-LABEL: enum  __attribute__((enum_extensibility("open"))) SoonToBeCFEnum {
+// CHECK-UNVERSIONED-NEXT:     SoonToBeCFEnumA = 1
+// CHECK-UNVERSIONED-NEXT: };
+// CHECK-UNVERSIONED-LABEL: enum  __attribute__((enum_extensibility("open"))) SoonToBeNSEnum {
+// CHECK-UNVERSIONED-NEXT:     SoonToBeNSEnumA = 1
+// CHECK-UNVERSIONED-NEXT: };
+// CHECK-UNVERSIONED-LABEL: enum  __attribute__((enum_extensibility("open"))) __attribute__((flag_enum)) SoonToBeCFOptions {
+// CHECK-UNVERSIONED-NEXT:     SoonToBeCFOptionsA = 1
+// CHECK-UNVERSIONED-NEXT: };
+// CHECK-UNVERSIONED-LABEL: enum  __attribute__((enum_extensibility("open"))) __attribute__((flag_enum)) SoonToBeNSOptions {
+// CHECK-UNVERSIONED-NEXT:     SoonToBeNSOptionsA = 1
+// CHECK-UNVERSIONED-NEXT: };
+// CHECK-UNVERSIONED-LABEL: enum  __attribute__((enum_extensibility("closed"))) SoonToBeCFClosedEnum {
+// CHECK-UNVERSIONED-NEXT:     SoonToBeCFClosedEnumA = 1
+// CHECK-UNVERSIONED-NEXT: };
+// CHECK-UNVERSIONED-LABEL: enum  __attribute__((enum_extensibility("closed"))) SoonToBeNSClosedEnum {
+// CHECK-UNVERSIONED-NEXT:     SoonToBeNSClosedEnumA = 1
+// CHECK-UNVERSIONED-NEXT: };
+// CHECK-UNVERSIONED-LABEL: enum UndoAllThatHasBeenDoneToMe {
+// CHECK-UNVERSIONED-NEXT:     UndoAllThatHasBeenDoneToMeA = 1
+// CHECK-UNVERSIONED-NEXT: };
diff --git a/clang/test/APINotes/yaml-convert-diags.c b/clang/test/APINotes/yaml-convert-diags.c
new file mode 100644
index 000000000000..1d352dc2c523
--- /dev/null
+++ b/clang/test/APINotes/yaml-convert-diags.c
@@ -0,0 +1,6 @@
+// RUN: rm -rf %t
+// RUN: not %clang_cc1 -fsyntax-only -fapinotes %s -I %S/Inputs/BrokenHeaders2 2>&1 | FileCheck %s
+
+#include "SomeBrokenLib.h"
+
+// CHECK: error: multiple definitions of global function 'do_something_with_pointers'
diff --git a/clang/test/APINotes/yaml-parse-diags.c b/clang/test/APINotes/yaml-parse-diags.c
new file mode 100644
index 000000000000..3ae39ccb301d
--- /dev/null
+++ b/clang/test/APINotes/yaml-parse-diags.c
@@ -0,0 +1,6 @@
+// RUN: rm -rf %t
+// RUN: %clang_cc1 -fsyntax-only -fapinotes %s -I %S/Inputs/BrokenHeaders -verify
+
+#include "SomeBrokenLib.h"
+
+// expected-error@APINotes.apinotes:4{{unknown key 'Nu llabilityOfRet'}}
diff --git a/clang/test/APINotes/yaml-reader-errors.m b/clang/test/APINotes/yaml-reader-errors.m
new file mode 100644
index 000000000000..9e5ee34c3e41
--- /dev/null
+++ b/clang/test/APINotes/yaml-reader-errors.m
@@ -0,0 +1,5 @@
+// RUN: rm -rf %t
+// RUN: not %clang_cc1 -fmodules -fimplicit-module-maps -fapinotes -fapinotes-modules -fmodules-cache-path=%t -I %S/Inputs/yaml-reader-errors/ -fsyntax-only %s > %t.err 2>&1
+// RUN: FileCheck %S/Inputs/yaml-reader-errors/UIKit.apinotes < %t.err
+
+@import UIKit;
-- 
GitLab


From b343b02a88821cab320e7d9976a05eabd0df29ec Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Wed, 27 Mar 2024 13:14:36 +0000
Subject: [PATCH 404/404] [gn build] Port 4f9aab2b500d

---
 llvm/utils/gn/secondary/llvm/utils/TableGen/Common/BUILD.gn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/utils/gn/secondary/llvm/utils/TableGen/Common/BUILD.gn b/llvm/utils/gn/secondary/llvm/utils/TableGen/Common/BUILD.gn
index c0ea62716fd2..daa3278d56d7 100644
--- a/llvm/utils/gn/secondary/llvm/utils/TableGen/Common/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/utils/TableGen/Common/BUILD.gn
@@ -18,9 +18,11 @@ static_library("Common") {
     "DAGISelMatcher.cpp",
     "GlobalISel/CXXPredicates.cpp",
     "GlobalISel/CodeExpander.cpp",
+    "GlobalISel/CombinerUtils.cpp",
     "GlobalISel/GlobalISelMatchTable.cpp",
     "GlobalISel/GlobalISelMatchTableExecutorEmitter.cpp",
     "GlobalISel/MatchDataInfo.cpp",
+    "GlobalISel/PatternParser.cpp",
     "GlobalISel/Patterns.cpp",
     "InfoByHwMode.cpp",
     "OptEmitter.cpp",
-- 
GitLab